{"source":"https://crawlcheck.io/ai-crawlers","retrieved":"2026-10-13T08:52:20.122Z","license":"https://creativecommons.org/licenses/by/4.0/","count":114,"sent_by_scanner":12,"identity_types":{"answer_engine_crawler":14,"training_crawler":44,"user_triggered_fetch":3,"search_index_crawler":31,"control_token":2,"link_preview_fetcher":6,"tool_or_developer_fetch":14},"verification_methods":{"user_agent_string_only":83,"published_ip_feed":22,"operator_documented":2,"none_control_token":2,"asn_range":1,"reverse_dns":4},"schema":{"identity_type":["training_crawler","search_index_crawler","answer_engine_crawler","user_triggered_fetch","link_preview_fetcher","tool_or_developer_fetch","control_token"],"verification_method":["published_ip_feed","asn_range","reverse_dns","operator_published_ranges","operator_documented","user_agent_string_only","none_control_token"],"note":"identity_type and verification_method are derived from the operator facts on record, never typed per row; a user_agent_string_only agent cannot be authenticated in a log."},"agents":[{"token":"Meta-ExternalFetcher","operator":null,"description":"Meta AI live fetch","purpose":"answer","identity_type":"answer_engine_crawler","identity_label":"answer-engine crawler","verification_method":"user_agent_string_only","verification_source":null,"verifiable":false,"sent_by_scanner":false},{"token":"MistralAI-User","operator":"Mistral","description":"Le Chat live fetch","purpose":"answer","identity_type":"answer_engine_crawler","identity_label":"answer-engine crawler","verification_method":"published_ip_feed","verification_source":"https://mistral.ai/mistralai-user-ips.json","verifiable":true,"sent_by_scanner":false},{"token":"DuckAssistBot","operator":null,"description":"DuckDuckGo DuckAssist","purpose":"answer","identity_type":"answer_engine_crawler","identity_label":"answer-engine crawler","verification_method":"operator_documented","verification_source":"published IP ranges","verifiable":true,"sent_by_scanner":false},{"token":"YouBot","operator":null,"description":"You.com index","purpose":"answer","identity_type":"answer_engine_crawler","identity_label":"answer-engine crawler","verification_method":"user_agent_string_only","verification_source":null,"verifiable":false,"sent_by_scanner":false},{"token":"cohere-ai","operator":null,"description":"Cohere","purpose":"train","identity_type":"training_crawler","identity_label":"training crawler","verification_method":"user_agent_string_only","verification_source":null,"verifiable":false,"sent_by_scanner":false},{"token":"Diffbot","operator":null,"description":"Diffbot knowledge graph","purpose":"train","identity_type":"training_crawler","identity_label":"training crawler","verification_method":"user_agent_string_only","verification_source":null,"verifiable":false,"sent_by_scanner":false},{"token":"Timpibot","operator":null,"description":"Timpi index","purpose":"train","identity_type":"training_crawler","identity_label":"training crawler","verification_method":"user_agent_string_only","verification_source":null,"verifiable":false,"sent_by_scanner":false},{"token":"Google-CloudVertexBot","operator":null,"description":"Vertex AI grounding","purpose":"train","identity_type":"training_crawler","identity_label":"training crawler","verification_method":"user_agent_string_only","verification_source":null,"verifiable":false,"sent_by_scanner":false},{"token":"OAI-SearchBot","operator":"OpenAI","description":"ChatGPT search index","purpose":"answer","identity_type":"answer_engine_crawler","identity_label":"answer-engine crawler","verification_method":"published_ip_feed","verification_source":"https://openai.com/searchbot.json","verifiable":true,"sent_by_scanner":true},{"token":"ChatGPT-User","operator":"OpenAI","description":"ChatGPT live fetch","purpose":"answer","identity_type":"user_triggered_fetch","identity_label":"user-triggered fetch","verification_method":"published_ip_feed","verification_source":"https://openai.com/chatgpt-user.json","verifiable":true,"sent_by_scanner":false},{"token":"Claude-SearchBot","operator":"Anthropic","description":"Claude search index","purpose":"answer","identity_type":"answer_engine_crawler","identity_label":"answer-engine crawler","verification_method":"published_ip_feed","verification_source":"https://claude.com/crawling/bots.json","verifiable":true,"sent_by_scanner":true},{"token":"Claude-User","operator":"Anthropic","description":"Claude live fetch","purpose":"answer","identity_type":"answer_engine_crawler","identity_label":"answer-engine crawler","verification_method":"published_ip_feed","verification_source":"https://claude.com/crawling/bots.json","verifiable":true,"sent_by_scanner":false},{"token":"PerplexityBot","operator":"Perplexity","description":"Perplexity index","purpose":"answer","identity_type":"answer_engine_crawler","identity_label":"answer-engine crawler","verification_method":"published_ip_feed","verification_source":"https://www.perplexity.ai/perplexitybot.json","verifiable":true,"sent_by_scanner":true},{"token":"Perplexity-User","operator":"Perplexity","description":"Perplexity live fetch","purpose":"answer","identity_type":"user_triggered_fetch","identity_label":"user-triggered fetch","verification_method":"published_ip_feed","verification_source":"https://www.perplexity.ai/perplexity-user.json","verifiable":true,"sent_by_scanner":false},{"token":"Googlebot","operator":"Google","description":"Google Search / AI Overviews","purpose":"search","identity_type":"search_index_crawler","identity_label":"search index crawler","verification_method":"published_ip_feed","verification_source":"https://developers.google.com/static/crawling/ipranges/common-crawlers.json","verifiable":true,"sent_by_scanner":true},{"token":"bingbot","operator":"Microsoft","description":"Bing / Copilot","purpose":"search","identity_type":"search_index_crawler","identity_label":"search index crawler","verification_method":"published_ip_feed","verification_source":"https://www.bing.com/toolbox/bingbot.json","verifiable":true,"sent_by_scanner":true},{"token":"Applebot","operator":"Apple","description":"Apple / Siri","purpose":"search","identity_type":"search_index_crawler","identity_label":"search index crawler","verification_method":"published_ip_feed","verification_source":"https://search.developer.apple.com/applebot.json","verifiable":true,"sent_by_scanner":true},{"token":"Amazonbot","operator":"Amazon","description":"Amazon","purpose":"search","identity_type":"search_index_crawler","identity_label":"search index crawler","verification_method":"published_ip_feed","verification_source":"https://developer.amazon.com/amazonbot/ip-addresses/","verifiable":true,"sent_by_scanner":true},{"token":"GPTBot","operator":"OpenAI","description":"OpenAI training + index","purpose":"train","identity_type":"training_crawler","identity_label":"training crawler","verification_method":"published_ip_feed","verification_source":"https://openai.com/gptbot.json","verifiable":true,"sent_by_scanner":true},{"token":"ClaudeBot","operator":"Anthropic","description":"Anthropic training","purpose":"train","identity_type":"training_crawler","identity_label":"training crawler","verification_method":"published_ip_feed","verification_source":"https://claude.com/crawling/bots.json","verifiable":true,"sent_by_scanner":true},{"token":"Google-Extended","operator":"Google","description":"Gemini training","purpose":"train","identity_type":"control_token","identity_label":"robots.txt control token (never sent)","verification_method":"none_control_token","verification_source":null,"verifiable":false,"sent_by_scanner":false},{"token":"Applebot-Extended","operator":"Apple","description":"Apple training","purpose":"train","identity_type":"control_token","identity_label":"robots.txt control token (never sent)","verification_method":"none_control_token","verification_source":null,"verifiable":false,"sent_by_scanner":false},{"token":"CCBot","operator":"Common Crawl","description":"Common Crawl","purpose":"train","identity_type":"training_crawler","identity_label":"training crawler","verification_method":"published_ip_feed","verification_source":"https://index.commoncrawl.org/ccbot.json","verifiable":true,"sent_by_scanner":true},{"token":"Bytespider","operator":"ByteDance","description":"ByteDance","purpose":"train","identity_type":"training_crawler","identity_label":"training crawler","verification_method":"user_agent_string_only","verification_source":"no published ranges","verifiable":false,"sent_by_scanner":true},{"token":"meta-externalagent","operator":"Meta","description":"Meta AI","purpose":"train","identity_type":"training_crawler","identity_label":"training crawler","verification_method":"asn_range","verification_source":"no JSON feed — Meta publishes its crawler ranges as ASN AS32934 (whois -h whois.radb.net -- '-i origin AS32934')","verifiable":true,"sent_by_scanner":true},{"token":"Googlebot-Image","operator":null,"description":"Google image crawler","purpose":"search","identity_type":"search_index_crawler","identity_label":"search index crawler","verification_method":"reverse_dns","verification_source":"Google common-crawler ranges (shared feed) + reverse DNS","verifiable":true,"sent_by_scanner":false},{"token":"Googlebot-News","operator":null,"description":"Google News","purpose":"search","identity_type":"search_index_crawler","identity_label":"search index crawler","verification_method":"user_agent_string_only","verification_source":null,"verifiable":false,"sent_by_scanner":false},{"token":"Googlebot-Video","operator":null,"description":"Google video crawler","purpose":"search","identity_type":"search_index_crawler","identity_label":"search index crawler","verification_method":"reverse_dns","verification_source":"Google common-crawler ranges (shared feed) + reverse DNS","verifiable":true,"sent_by_scanner":false},{"token":"Storebot-Google","operator":null,"description":"Google Shopping","purpose":"search","identity_type":"search_index_crawler","identity_label":"search index crawler","verification_method":"reverse_dns","verification_source":"Google special-crawler ranges (shared feed) + reverse DNS","verifiable":true,"sent_by_scanner":false},{"token":"Mediapartners-Google","operator":null,"description":"Google AdSense","purpose":"search","identity_type":"search_index_crawler","identity_label":"search index crawler","verification_method":"user_agent_string_only","verification_source":null,"verifiable":false,"sent_by_scanner":false},{"token":"APIs-Google","operator":null,"description":"Google push delivery","purpose":"search","identity_type":"search_index_crawler","identity_label":"search index crawler","verification_method":"user_agent_string_only","verification_source":null,"verifiable":false,"sent_by_scanner":false},{"token":"BingPreview","operator":null,"description":"Bing page preview","purpose":"search","identity_type":"search_index_crawler","identity_label":"search index crawler","verification_method":"user_agent_string_only","verification_source":null,"verifiable":false,"sent_by_scanner":false},{"token":"msnbot","operator":null,"description":"Microsoft legacy crawler","purpose":"search","identity_type":"search_index_crawler","identity_label":"search index crawler","verification_method":"user_agent_string_only","verification_source":null,"verifiable":false,"sent_by_scanner":false},{"token":"Slurp","operator":null,"description":"Yahoo Slurp","purpose":"search","identity_type":"search_index_crawler","identity_label":"search index crawler","verification_method":"user_agent_string_only","verification_source":null,"verifiable":false,"sent_by_scanner":false},{"token":"Yandex","operator":null,"description":"Yandex (umbrella token)","purpose":"search","identity_type":"search_index_crawler","identity_label":"search index crawler","verification_method":"user_agent_string_only","verification_source":null,"verifiable":false,"sent_by_scanner":false},{"token":"coccocbot-web","operator":null,"description":"Coc Coc","purpose":"search","identity_type":"search_index_crawler","identity_label":"search index crawler","verification_method":"user_agent_string_only","verification_source":null,"verifiable":false,"sent_by_scanner":false},{"token":"Facebot","operator":null,"description":"Meta link crawler","purpose":"social","identity_type":"link_preview_fetcher","identity_label":"link-preview fetcher","verification_method":"user_agent_string_only","verification_source":null,"verifiable":false,"sent_by_scanner":false},{"token":"Magpie-crawler","operator":null,"description":"Magpie AI","purpose":"train","identity_type":"training_crawler","identity_label":"training crawler","verification_method":"user_agent_string_only","verification_source":null,"verifiable":false,"sent_by_scanner":false},{"token":"img2dataset","operator":null,"description":"img2dataset image corpus","purpose":"train","identity_type":"training_crawler","identity_label":"training crawler","verification_method":"user_agent_string_only","verification_source":null,"verifiable":false,"sent_by_scanner":false},{"token":"AwarioRssBot","operator":null,"description":"Awario RSS","purpose":"train","identity_type":"training_crawler","identity_label":"training crawler","verification_method":"user_agent_string_only","verification_source":null,"verifiable":false,"sent_by_scanner":false},{"token":"AwarioSmartBot","operator":null,"description":"Awario smart","purpose":"train","identity_type":"training_crawler","identity_label":"training crawler","verification_method":"user_agent_string_only","verification_source":null,"verifiable":false,"sent_by_scanner":false},{"token":"TurnitinBot","operator":null,"description":"Turnitin","purpose":"train","identity_type":"training_crawler","identity_label":"training crawler","verification_method":"user_agent_string_only","verification_source":null,"verifiable":false,"sent_by_scanner":false},{"token":"archive.org_bot","operator":null,"description":"Internet Archive","purpose":"train","identity_type":"training_crawler","identity_label":"training crawler","verification_method":"user_agent_string_only","verification_source":null,"verifiable":false,"sent_by_scanner":false},{"token":"ia_archiver","operator":null,"description":"Internet Archive (legacy)","purpose":"train","identity_type":"training_crawler","identity_label":"training crawler","verification_method":"user_agent_string_only","verification_source":null,"verifiable":false,"sent_by_scanner":false},{"token":"AhrefsBot","operator":null,"description":"Ahrefs link index","purpose":"tool","identity_type":"tool_or_developer_fetch","identity_label":"tool or developer fetch","verification_method":"user_agent_string_only","verification_source":null,"verifiable":false,"sent_by_scanner":false},{"token":"SemrushBot","operator":null,"description":"Semrush crawler","purpose":"tool","identity_type":"tool_or_developer_fetch","identity_label":"tool or developer fetch","verification_method":"user_agent_string_only","verification_source":null,"verifiable":false,"sent_by_scanner":false},{"token":"MJ12bot","operator":null,"description":"Majestic link index","purpose":"tool","identity_type":"tool_or_developer_fetch","identity_label":"tool or developer fetch","verification_method":"user_agent_string_only","verification_source":null,"verifiable":false,"sent_by_scanner":false},{"token":"DotBot","operator":null,"description":"Moz link index","purpose":"tool","identity_type":"tool_or_developer_fetch","identity_label":"tool or developer fetch","verification_method":"user_agent_string_only","verification_source":null,"verifiable":false,"sent_by_scanner":false},{"token":"rogerbot","operator":null,"description":"Moz site crawler","purpose":"tool","identity_type":"tool_or_developer_fetch","identity_label":"tool or developer fetch","verification_method":"user_agent_string_only","verification_source":null,"verifiable":false,"sent_by_scanner":false},{"token":"DataForSeoBot","operator":null,"description":"DataForSEO","purpose":"tool","identity_type":"tool_or_developer_fetch","identity_label":"tool or developer fetch","verification_method":"user_agent_string_only","verification_source":null,"verifiable":false,"sent_by_scanner":false},{"token":"BLEXBot","operator":null,"description":"WebMeUp link index","purpose":"tool","identity_type":"tool_or_developer_fetch","identity_label":"tool or developer fetch","verification_method":"user_agent_string_only","verification_source":null,"verifiable":false,"sent_by_scanner":false},{"token":"Yeti","operator":null,"description":"Naver","purpose":"regional","identity_type":"search_index_crawler","identity_label":"search index crawler","verification_method":"user_agent_string_only","verification_source":null,"verifiable":false,"sent_by_scanner":false},{"token":"YoudaoBot","operator":null,"description":"Youdao","purpose":"regional","identity_type":"search_index_crawler","identity_label":"search index crawler","verification_method":"user_agent_string_only","verification_source":null,"verifiable":false,"sent_by_scanner":false},{"token":"Exabot","operator":null,"description":"Exalead","purpose":"regional","identity_type":"search_index_crawler","identity_label":"search index crawler","verification_method":"user_agent_string_only","verification_source":null,"verifiable":false,"sent_by_scanner":false},{"token":"Sogou web spider","operator":null,"description":"Sogou","purpose":"regional","identity_type":"search_index_crawler","identity_label":"search index crawler","verification_method":"user_agent_string_only","verification_source":null,"verifiable":false,"sent_by_scanner":false},{"token":"YisouSpider","operator":null,"description":"Yisou","purpose":"regional","identity_type":"search_index_crawler","identity_label":"search index crawler","verification_method":"user_agent_string_only","verification_source":null,"verifiable":false,"sent_by_scanner":false},{"token":"360Spider","operator":null,"description":"360 Search","purpose":"regional","identity_type":"search_index_crawler","identity_label":"search index crawler","verification_method":"user_agent_string_only","verification_source":null,"verifiable":false,"sent_by_scanner":false},{"token":"Sosospider","operator":null,"description":"Soso","purpose":"regional","identity_type":"search_index_crawler","identity_label":"search index crawler","verification_method":"user_agent_string_only","verification_source":null,"verifiable":false,"sent_by_scanner":false},{"token":"meta-webindexer","operator":null,"description":"Meta web index","purpose":"train","identity_type":"training_crawler","identity_label":"training crawler","verification_method":"user_agent_string_only","verification_source":null,"verifiable":false,"sent_by_scanner":false},{"token":"omgili","operator":null,"description":"Webz.io omgili","purpose":"train","identity_type":"training_crawler","identity_label":"training crawler","verification_method":"user_agent_string_only","verification_source":null,"verifiable":false,"sent_by_scanner":false},{"token":"cohere-training-data-crawler","operator":null,"description":"Cohere training corpus","purpose":"train","identity_type":"training_crawler","identity_label":"training crawler","verification_method":"user_agent_string_only","verification_source":null,"verifiable":false,"sent_by_scanner":false},{"token":"PanguBot","operator":null,"description":"Huawei PanGu","purpose":"train","identity_type":"training_crawler","identity_label":"training crawler","verification_method":"user_agent_string_only","verification_source":null,"verifiable":false,"sent_by_scanner":false},{"token":"Ai2Bot-Dolma","operator":null,"description":"Allen Institute Dolma","purpose":"train","identity_type":"training_crawler","identity_label":"training crawler","verification_method":"user_agent_string_only","verification_source":null,"verifiable":false,"sent_by_scanner":false},{"token":"FriendlyCrawler","operator":null,"description":"FriendlyCrawler ML","purpose":"train","identity_type":"training_crawler","identity_label":"training crawler","verification_method":"user_agent_string_only","verification_source":null,"verifiable":false,"sent_by_scanner":false},{"token":"VelenPublicWebCrawler","operator":null,"description":"Velen","purpose":"train","identity_type":"training_crawler","identity_label":"training crawler","verification_method":"user_agent_string_only","verification_source":null,"verifiable":false,"sent_by_scanner":false},{"token":"MyCentralAIScraperBot","operator":null,"description":"MyCentral AI","purpose":"train","identity_type":"training_crawler","identity_label":"training crawler","verification_method":"user_agent_string_only","verification_source":null,"verifiable":false,"sent_by_scanner":false},{"token":"DeepSeekBot","operator":null,"description":"DeepSeek","purpose":"train","identity_type":"training_crawler","identity_label":"training crawler","verification_method":"user_agent_string_only","verification_source":null,"verifiable":false,"sent_by_scanner":false},{"token":"ICC-Crawler","operator":null,"description":"NICT ICC","purpose":"train","identity_type":"training_crawler","identity_label":"training crawler","verification_method":"user_agent_string_only","verification_source":null,"verifiable":false,"sent_by_scanner":false},{"token":"GoogleOther","operator":null,"description":"Google non-search fetch","purpose":"train","identity_type":"training_crawler","identity_label":"training crawler","verification_method":"reverse_dns","verification_source":"Google common-crawler ranges (shared feed) + reverse DNS","verifiable":true,"sent_by_scanner":false},{"token":"CloudflareBrowserRenderingCrawler","operator":null,"description":"Cloudflare Browser Run /crawl","purpose":"tool","identity_type":"tool_or_developer_fetch","identity_label":"tool or developer fetch","verification_method":"user_agent_string_only","verification_source":null,"verifiable":false,"sent_by_scanner":false},{"token":"Cloudflare-AutoRAG","operator":null,"description":"Cloudflare AutoRAG","purpose":"tool","identity_type":"tool_or_developer_fetch","identity_label":"tool or developer fetch","verification_method":"user_agent_string_only","verification_source":null,"verifiable":false,"sent_by_scanner":false},{"token":"Peer39_Crawler","operator":null,"description":"Peer39 ad context","purpose":"tool","identity_type":"tool_or_developer_fetch","identity_label":"tool or developer fetch","verification_method":"user_agent_string_only","verification_source":null,"verifiable":false,"sent_by_scanner":false},{"token":"AdsBot-Google","operator":null,"description":"Google Ads quality","purpose":"tool","identity_type":"tool_or_developer_fetch","identity_label":"tool or developer fetch","verification_method":"user_agent_string_only","verification_source":null,"verifiable":false,"sent_by_scanner":false},{"token":"AmazonAdBot","operator":null,"description":"Amazon Ads","purpose":"tool","identity_type":"tool_or_developer_fetch","identity_label":"tool or developer fetch","verification_method":"user_agent_string_only","verification_source":null,"verifiable":false,"sent_by_scanner":false},{"token":"AdIdxBot","operator":null,"description":"Microsoft Ads","purpose":"tool","identity_type":"tool_or_developer_fetch","identity_label":"tool or developer fetch","verification_method":"user_agent_string_only","verification_source":null,"verifiable":false,"sent_by_scanner":false},{"token":"Twitterbot","operator":null,"description":"X link preview","purpose":"social","identity_type":"link_preview_fetcher","identity_label":"link-preview fetcher","verification_method":"user_agent_string_only","verification_source":null,"verifiable":false,"sent_by_scanner":false},{"token":"LinkedInBot","operator":null,"description":"LinkedIn link preview","purpose":"social","identity_type":"link_preview_fetcher","identity_label":"link-preview fetcher","verification_method":"user_agent_string_only","verification_source":null,"verifiable":false,"sent_by_scanner":false},{"token":"facebookexternalhit","operator":null,"description":"Facebook link preview","purpose":"social","identity_type":"link_preview_fetcher","identity_label":"link-preview fetcher","verification_method":"user_agent_string_only","verification_source":null,"verifiable":false,"sent_by_scanner":false},{"token":"Pinterestbot","operator":null,"description":"Pinterest","purpose":"social","identity_type":"link_preview_fetcher","identity_label":"link-preview fetcher","verification_method":"user_agent_string_only","verification_source":null,"verifiable":false,"sent_by_scanner":false},{"token":"Slackbot-LinkExpanding","operator":null,"description":"Slack unfurl","purpose":"social","identity_type":"link_preview_fetcher","identity_label":"link-preview fetcher","verification_method":"user_agent_string_only","verification_source":null,"verifiable":false,"sent_by_scanner":false},{"token":"Gemini-Deep-Research","operator":null,"description":"Gemini research agent","purpose":"answer","identity_type":"answer_engine_crawler","identity_label":"answer-engine crawler","verification_method":"user_agent_string_only","verification_source":null,"verifiable":false,"sent_by_scanner":false},{"token":"PhindBot","operator":null,"description":"Phind","purpose":"answer","identity_type":"answer_engine_crawler","identity_label":"answer-engine crawler","verification_method":"user_agent_string_only","verification_source":null,"verifiable":false,"sent_by_scanner":false},{"token":"Kagibot","operator":null,"description":"Kagi","purpose":"answer","identity_type":"answer_engine_crawler","identity_label":"answer-engine crawler","verification_method":"user_agent_string_only","verification_source":null,"verifiable":false,"sent_by_scanner":false},{"token":"Copilot-User","operator":null,"description":"Microsoft Copilot fetch","purpose":"answer","identity_type":"answer_engine_crawler","identity_label":"answer-engine crawler","verification_method":"user_agent_string_only","verification_source":null,"verifiable":false,"sent_by_scanner":false},{"token":"DuckDuckBot","operator":null,"description":"DuckDuckGo","purpose":"search","identity_type":"search_index_crawler","identity_label":"search index crawler","verification_method":"user_agent_string_only","verification_source":null,"verifiable":false,"sent_by_scanner":false},{"token":"YandexBot","operator":null,"description":"Yandex","purpose":"search","identity_type":"search_index_crawler","identity_label":"search index crawler","verification_method":"user_agent_string_only","verification_source":null,"verifiable":false,"sent_by_scanner":false},{"token":"Baiduspider","operator":null,"description":"Baidu","purpose":"search","identity_type":"search_index_crawler","identity_label":"search index crawler","verification_method":"user_agent_string_only","verification_source":null,"verifiable":false,"sent_by_scanner":false},{"token":"Seznambot","operator":null,"description":"Seznam","purpose":"search","identity_type":"search_index_crawler","identity_label":"search index crawler","verification_method":"user_agent_string_only","verification_source":null,"verifiable":false,"sent_by_scanner":false},{"token":"Neevabot","operator":null,"description":"Neeva","purpose":"search","identity_type":"search_index_crawler","identity_label":"search index crawler","verification_method":"user_agent_string_only","verification_source":null,"verifiable":false,"sent_by_scanner":false},{"token":"PetalBot","operator":null,"description":"Huawei Petal","purpose":"search","identity_type":"search_index_crawler","identity_label":"search index crawler","verification_method":"user_agent_string_only","verification_source":null,"verifiable":false,"sent_by_scanner":false},{"token":"anthropic-ai","operator":null,"description":"Anthropic legacy agent","purpose":"train","identity_type":"training_crawler","identity_label":"training crawler","verification_method":"user_agent_string_only","verification_source":null,"verifiable":false,"sent_by_scanner":false},{"token":"Claude-Web","operator":null,"description":"Anthropic legacy agent","purpose":"train","identity_type":"training_crawler","identity_label":"training crawler","verification_method":"user_agent_string_only","verification_source":null,"verifiable":false,"sent_by_scanner":false},{"token":"FacebookBot","operator":null,"description":"Meta legacy","purpose":"train","identity_type":"training_crawler","identity_label":"training crawler","verification_method":"user_agent_string_only","verification_source":null,"verifiable":false,"sent_by_scanner":false},{"token":"Omgilibot","operator":null,"description":"Webz.io","purpose":"train","identity_type":"training_crawler","identity_label":"training crawler","verification_method":"user_agent_string_only","verification_source":null,"verifiable":false,"sent_by_scanner":false},{"token":"ImagesiftBot","operator":null,"description":"Imagesift","purpose":"train","identity_type":"training_crawler","identity_label":"training crawler","verification_method":"user_agent_string_only","verification_source":null,"verifiable":false,"sent_by_scanner":false},{"token":"AI2Bot","operator":null,"description":"Allen Institute","purpose":"train","identity_type":"training_crawler","identity_label":"training crawler","verification_method":"user_agent_string_only","verification_source":null,"verifiable":false,"sent_by_scanner":false},{"token":"Scrapy","operator":null,"description":"Generic scraper framework","purpose":"train","identity_type":"training_crawler","identity_label":"training crawler","verification_method":"user_agent_string_only","verification_source":null,"verifiable":false,"sent_by_scanner":false},{"token":"SemrushBot-OCOB","operator":null,"description":"Semrush AI corpus","purpose":"train","identity_type":"training_crawler","identity_label":"training crawler","verification_method":"user_agent_string_only","verification_source":null,"verifiable":false,"sent_by_scanner":false},{"token":"Applebot-Extended-Ads","operator":null,"description":"Apple ads corpus","purpose":"train","identity_type":"training_crawler","identity_label":"training crawler","verification_method":"user_agent_string_only","verification_source":null,"verifiable":false,"sent_by_scanner":false},{"token":"TikTokSpider","operator":null,"description":"TikTok","purpose":"train","identity_type":"training_crawler","identity_label":"training crawler","verification_method":"user_agent_string_only","verification_source":null,"verifiable":false,"sent_by_scanner":false},{"token":"QuillBot","operator":null,"description":"QuillBot","purpose":"train","identity_type":"training_crawler","identity_label":"training crawler","verification_method":"user_agent_string_only","verification_source":null,"verifiable":false,"sent_by_scanner":false},{"token":"Webzio-Extended","operator":null,"description":"Webz.io extended","purpose":"train","identity_type":"training_crawler","identity_label":"training crawler","verification_method":"user_agent_string_only","verification_source":null,"verifiable":false,"sent_by_scanner":false},{"token":"ProRataInc","operator":null,"description":"ProRata","purpose":"train","identity_type":"training_crawler","identity_label":"training crawler","verification_method":"user_agent_string_only","verification_source":null,"verifiable":false,"sent_by_scanner":false},{"token":"AwarioBot","operator":null,"description":"Awario","purpose":"train","identity_type":"training_crawler","identity_label":"training crawler","verification_method":"user_agent_string_only","verification_source":null,"verifiable":false,"sent_by_scanner":false},{"token":"Amzn-SearchBot","operator":"Amazon","description":"Amazon search eligibility, separate from Amazonbot","purpose":"search","identity_type":"search_index_crawler","identity_label":"search index crawler","verification_method":"published_ip_feed","verification_source":"https://developer.amazon.com/amazonbot/searchbot-ip-addresses/","verifiable":true,"sent_by_scanner":false},{"token":"Amzn-User","operator":"Amazon","description":"Amazon live fetch for Alexa questions","purpose":"answer","identity_type":"answer_engine_crawler","identity_label":"answer-engine crawler","verification_method":"published_ip_feed","verification_source":"https://developer.amazon.com/amazonbot/live-ip-addresses/","verifiable":true,"sent_by_scanner":false},{"token":"MistralAI-Index","operator":"Mistral","description":"Mistral search index for Vibe","purpose":"search","identity_type":"search_index_crawler","identity_label":"search index crawler","verification_method":"published_ip_feed","verification_source":"https://mistral.ai/mistralai-index-ips.json","verifiable":true,"sent_by_scanner":false},{"token":"MistralAI-Training","operator":"Mistral","description":"Mistral training crawl","purpose":"train","identity_type":"training_crawler","identity_label":"training crawler","verification_method":"user_agent_string_only","verification_source":"no published ranges — a request wearing this name cannot be checked","verifiable":false,"sent_by_scanner":false},{"token":"OAI-AdsBot","operator":"OpenAI","description":"OpenAI ChatGPT ads page validation","purpose":"tool","identity_type":"tool_or_developer_fetch","identity_label":"tool or developer fetch","verification_method":"published_ip_feed","verification_source":"https://openai.com/adsbot.json","verifiable":true,"sent_by_scanner":false},{"token":"Google-Agent","operator":"Google","description":"Google user-triggered agent, acts on the web","purpose":"answer","identity_type":"user_triggered_fetch","identity_label":"user-triggered fetch","verification_method":"published_ip_feed","verification_source":"https://developers.google.com/static/crawling/ipranges/user-triggered-agents.json","verifiable":true,"sent_by_scanner":false},{"token":"Google-GeminiNotebook","operator":"Google","description":"Gemini Notebook user-supplied URL fetch","purpose":"answer","identity_type":"answer_engine_crawler","identity_label":"answer-engine crawler","verification_method":"published_ip_feed","verification_source":"https://developers.google.com/static/crawling/ipranges/user-triggered-fetchers-google.json","verifiable":true,"sent_by_scanner":false},{"token":"GoogleOther-Image","operator":"Google","description":"Google common crawler, public image URLs","purpose":"train","identity_type":"training_crawler","identity_label":"training crawler","verification_method":"published_ip_feed","verification_source":"https://developers.google.com/static/crawling/ipranges/common-crawlers.json","verifiable":true,"sent_by_scanner":false},{"token":"GoogleOther-Video","operator":"Google","description":"Google common crawler, public video URLs","purpose":"train","identity_type":"training_crawler","identity_label":"training crawler","verification_method":"published_ip_feed","verification_source":"https://developers.google.com/static/crawling/ipranges/common-crawlers.json","verifiable":true,"sent_by_scanner":false},{"token":"ExaSearchBot","operator":"Exa","description":"Exa AI search index, Web Bot Auth signed","purpose":"search","identity_type":"search_index_crawler","identity_label":"search index crawler","verification_method":"operator_documented","verification_source":"Web Bot Auth — every request is cryptographically signed; directory at crawler.exa.ai/.well-known/http-message-signatures-directory","verifiable":true,"sent_by_scanner":false}]}