{"source":"https://crawlcheck.io/robots-policies","catalogue":"https://crawlcheck.io/ai-crawlers.json","changes":"https://crawlcheck.io/ai-crawlers/changes.json","ip_ranges":"https://crawlcheck.io/ai-crawlers/ip-ranges.json","note":"Block-only: a robots.txt group that names a crawler replaces the * rules for it, so these files only ever add Disallow: /.","policies":[{"id":"block-ai-training","what":"Block training and corpus crawlers, keep search and answer engines","user_agents":36,"robots_txt":"https://crawlcheck.io/robots/block-ai-training.txt","server_rules":{"nginx":"https://crawlcheck.io/robots/block-ai-training.nginx.txt","apache":"https://crawlcheck.io/robots/block-ai-training.apache.txt","caddy":"https://crawlcheck.io/robots/block-ai-training.caddy.txt","cloudflare":"https://crawlcheck.io/robots/block-ai-training.cloudflare.txt"}},{"id":"block-all-ai","what":"Block training crawlers and AI answer engines, keep classic search","user_agents":52,"robots_txt":"https://crawlcheck.io/robots/block-all-ai.txt","server_rules":{"nginx":"https://crawlcheck.io/robots/block-all-ai.nginx.txt","apache":"https://crawlcheck.io/robots/block-all-ai.apache.txt","caddy":"https://crawlcheck.io/robots/block-all-ai.caddy.txt","cloudflare":"https://crawlcheck.io/robots/block-all-ai.cloudflare.txt"}},{"id":"block-seo-tools","what":"Block SEO, advertising and monitoring crawlers","user_agents":27,"robots_txt":"https://crawlcheck.io/robots/block-seo-tools.txt","server_rules":{"nginx":"https://crawlcheck.io/robots/block-seo-tools.nginx.txt","apache":"https://crawlcheck.io/robots/block-seo-tools.apache.txt","caddy":"https://crawlcheck.io/robots/block-seo-tools.caddy.txt","cloudflare":"https://crawlcheck.io/robots/block-seo-tools.cloudflare.txt"}}]}