# Unified rules for all allowed crawlers
User-agent: *
Disallow: /index2.html
Disallow: /index-test.html
Disallow: /pgn/
Disallow: /perl/nph-chesspgn
Disallow: /perl/pgndownload
Disallow: /perl/printgame
Disallow: /perl/rerolpxe
Disallow: /perl/chesscorrection.pl
Disallow: /perl/kibitzing
Disallow: /perl/storeadmin
Disallow: /perl/whistle
Disallow: /perl/chesslike
Disallow: /perl/chess.pl
Disallow: /perl/explorer
Disallow: /perl/articlenews
Disallow: /perl/articleversion
Disallow: /q/
Disallow: /shibuimoc.php
Disallow: /spidertrap.html
Disallow: /nodejs/
Disallow: /njs/
Disallow: /perl/ezsearch.pl
Disallow: /fen/
Disallow: /perl/exploregame.pl
Disallow: /perl/cn
Disallow: /perl/fen
Disallow: /perl/lookupfen
Disallow: /perl/lookupmoves

Crawl-delay: 5

User-agent: ClaudeBot
Disallow: /

User-agent: anthropic-ai
Disallow: /

User-agent: Google-Extended
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: OAI-SearchBot
Disallow: /

User-agent: Claude-Web
Disallow: /

User-agent: Meta-ExternalAgent
Disallow: /

User-agent: FacebookBot
Disallow: /

User-agent: Omgilibot
Disallow: /

User-agent: Diffbot
Disallow: /

User-agent: PerplexityBot
Disallow: /

User-agent: YouBot
Disallow: /

User-agent: Applebot-Extended
Disallow: /

User-agent: cohere-ai
Disallow: /

User-agent: Amazonbot
Disallow: /


User-agent: ChatGPT-User
Disallow: /

User-agent: GPTBot
Disallow: /

User-agent: Yandex
Disallow: /

User-agent: linguee
Disallow: /

User-agent: The Knowledge AI
Disallow: /

User-agent: Vegi bot
Disallow: /

User-agent: Vegi-bot
Disallow: /

# Block bots
User-agent: 008
Disallow: /

User-agent: A6-Indexer
Disallow: /

User-agent: AhrefsBot
Disallow: /

User-agent: AlphaSeoBot
Disallow: /

User-agent: AlphaSeoBot-SA
Disallow: /

# RDH, 08.19.19: temporarily blocking Applebot due to excessive crawling
User-agent: Applebot
Disallow: /

User-agent: AspiegelBot
Disallow: /

# RDH, 05.13.20: bing currently blocked
#User-agent: bingbot/2.0
#Disallow: /

User-agent: Blackboard Safeassign
Disallow: /

User-agent: BLEXBot/1.0
Disallow: /

User-agent: Bytespider
Disallow: /

User-agent: crawler4j
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: Gigabot
Disallow: /

User-agent: LieBaoFast
Disallow: /

User-agent: MauiBot
Disallow: /

User-agent: MauiBot (crawler.feedback+wc@gmail.com)
Disallow: /

User-agent: MegaIndex.ru/2.0
Disallow: /

User-agent: MqqBrowser
Disallow: /

User-agent: Nimbostratus-Bot/v1.3.2
Disallow: /

User-agent: Seekport Crawler
Disallow: /

User-agent: SemrushBot
Disallow: /

User-agent: SemrushBot-SA
Disallow: /

User-agent: SeznamBot
Disallow: /

User-agent: SiteExplorer
Disallow: /

User-agent: SputnikBot/2.3
Disallow: /

User-agent: TurnitinBot
Disallow: /

User-agent: UCBrowser
Disallow: /

User-agent: yacybot
Disallow: /

User-agent: Yeti
Disallow: /

User-agent: YisouSpider
Disallow: /