# Block training and corpus crawlers, keep search and answer engines — cloudflare # 36 user agents, generated from https://crawlcheck.io/ai-crawlers # A user agent string can be forged; this stops crawlers that name themselves. # Security > WAF > Custom rules: paste as the expression, action Block. # Redirect rules run before custom rules: on a zone that redirects every request this never fires, so put it on the zone that serves the pages. (lower(http.user_agent) contains "img2dataset") or (lower(http.user_agent) contains "turnitinbot") or (lower(http.user_agent) contains "archive.org_bot") or (lower(http.user_agent) contains "ia_archiver") or (lower(http.user_agent) contains "omgili") or (lower(http.user_agent) contains "cohere-training-data-crawler") or (lower(http.user_agent) contains "pangubot") or (lower(http.user_agent) contains "ai2bot-dolma") or (lower(http.user_agent) contains "friendlycrawler") or (lower(http.user_agent) contains "mycentralaiscraperbot") or (lower(http.user_agent) contains "deepseekbot") or (lower(http.user_agent) contains "icc-crawler") or (lower(http.user_agent) contains "googleother") or (lower(http.user_agent) contains "amazonbot") or (lower(http.user_agent) contains "gptbot") or (lower(http.user_agent) contains "claudebot") or (lower(http.user_agent) contains "anthropic-ai") or (lower(http.user_agent) contains "claude-web") or (lower(http.user_agent) contains "ccbot") or (lower(http.user_agent) contains "bytespider") or (lower(http.user_agent) contains "meta-externalagent") or (lower(http.user_agent) contains "facebookbot") or (lower(http.user_agent) contains "cohere-ai") or (lower(http.user_agent) contains "diffbot") or (lower(http.user_agent) contains "omgilibot") or (lower(http.user_agent) contains "imagesiftbot") or (lower(http.user_agent) contains "timpibot") or (lower(http.user_agent) contains "ai2bot") or (lower(http.user_agent) contains "applebot-extended-ads") or (lower(http.user_agent) contains "tiktokspider") or (lower(http.user_agent) contains "quillbot") or (lower(http.user_agent) contains "webzio-extended") or (lower(http.user_agent) contains "proratainc") or (lower(http.user_agent) contains "mistralai-training") or (lower(http.user_agent) contains "googleother-image") or (lower(http.user_agent) contains "googleother-video")