# Block training crawlers and AI answer engines, keep classic search — nginx # 52 user agents, generated from https://crawlcheck.io/ai-crawlers # A user agent string can be forged; this stops crawlers that name themselves. # Put the map in the http block and the if in each server block. map $http_user_agent $cc_block { default 0; "~*(img2dataset|TurnitinBot|archive\.org_bot|ia_archiver|omgili|cohere-training-data-crawler|PanguBot|Ai2Bot-Dolma|FriendlyCrawler|MyCentralAIScraperBot|DeepSeekBot|ICC-Crawler|GoogleOther|OAI-SearchBot|ChatGPT-User|Claude-SearchBot|Claude-User|PerplexityBot|Perplexity-User|Gemini-Deep-Research|MistralAI-User|DuckAssistBot|YouBot|PhindBot|Copilot-User|Meta-ExternalFetcher|Amazonbot|GPTBot|ClaudeBot|anthropic-ai|Claude-Web|CCBot|Bytespider|meta-externalagent|FacebookBot|cohere-ai|Diffbot|Omgilibot|ImagesiftBot|Timpibot|AI2Bot|Applebot-Extended-Ads|TikTokSpider|QuillBot|Webzio-Extended|ProRataInc|Amzn-User|MistralAI-Training|Google-Agent|Google-GeminiNotebook|GoogleOther-Image|GoogleOther-Video)" 1; } # server { # if ($cc_block) { return 403; } # }