{"domain":"lab2.crawlcheck.io","origin":"https://lab2.crawlcheck.io","checkedAt":"2026-11-04T05:57:10.226Z","score":89,"grade":"Good","robotsTxtStatus":200,"robotsTxtWafBlocked":false,"robotsFetchMethod":"primary","bots":{"AI2Bot":{"label":"AI2Bot","platform":"Allen Institute for AI","tier":2,"emoji":"🤖","cat":"ai-bots","safety":"safe","impact":"low","purpose":"training","robotsTxt":{"status":"ALLOWED","rule":"Full content access; 2 path rules (/assets/, /quote)","source":"wildcard"},"effectiveStatus":"ALLOWED","intel":{"botType":"training","respectsRobots":"unknown"},"robotsApplies":true},"ChatGPT-User":{"label":"ChatGPT Search","platform":"OpenAI","tier":1,"emoji":"🟢","cat":"ai-bots","safety":"safe","impact":"low","purpose":"user-request","robotsTxt":{"status":"ALLOWED","rule":"Full content access; 2 path rules (/assets/, /quote)","source":"wildcard"},"effectiveStatus":"ALLOWED","intel":{"botType":"user-triggered","respectsRobots":"inconsistent","docsUrl":"https://developers.openai.com/api/docs/bots","ipRangesUrl":"https://openai.com/chatgpt-user.json","caveat":"Fetches a page only when a ChatGPT user asks. OpenAI: \"Because these actions are initiated by a user, robots.txt rules may not apply.\" Not used for automatic crawling, training or Search inclusion (use OAI-SearchBot for Search)."},"robotsApplies":true},"ChatGPT-User/2.0":{"label":"ChatGPT Search v2","platform":"OpenAI","tier":1,"emoji":"🟢","cat":"ai-bots","safety":"safe","impact":"low","purpose":"user-request","robotsTxt":{"status":"ALLOWED","rule":"Full content access; 2 path rules (/assets/, /quote)","source":"wildcard"},"effectiveStatus":"ALLOWED","intel":{"botType":"user-triggered","respectsRobots":"yes","docsUrl":"https://developers.openai.com/api/docs/bots"},"robotsApplies":true},"Claude-SearchBot":{"label":"Claude Search","platform":"Anthropic","tier":1,"emoji":"🟠","cat":"ai-bots","safety":"safe","impact":"low","purpose":"user-request","robotsTxt":{"status":"ALLOWED","rule":"Full content access; 2 path rules (/assets/, /quote)","source":"wildcard"},"effectiveStatus":"ALLOWED","intel":{"botType":"retrieval","respectsRobots":"yes","docsUrl":"https://support.claude.com/en/articles/8896518-does-anthropic-crawl-data-from-the-web-and-how-can-site-owners-block-the-crawler","ipRangesUrl":"https://claude.com/crawling/bots.json","caveat":"Blocking this removes you from Claude search retrieval. Independently controllable from ClaudeBot (training)."},"robotsApplies":true},"Claude-User":{"label":"Claude User Agent","platform":"Anthropic","tier":1,"emoji":"🟠","cat":"ai-bots","safety":"safe","impact":"low","purpose":"user-request","robotsTxt":{"status":"ALLOWED","rule":"Full content access; 2 path rules (/assets/, /quote)","source":"wildcard"},"effectiveStatus":"ALLOWED","intel":{"botType":"user-triggered","respectsRobots":"yes","docsUrl":"https://support.claude.com/en/articles/8896518-does-anthropic-crawl-data-from-the-web-and-how-can-site-owners-block-the-crawler","ipRangesUrl":"https://claude.com/crawling/bots.json"},"robotsApplies":true},"ClaudeBot":{"label":"Claude AI Training","platform":"Anthropic","tier":1,"emoji":"🟠","cat":"ai-bots","safety":"safe","impact":"low","purpose":"training","robotsTxt":{"status":"ALLOWED","rule":"Disallow: (empty)","source":"specific"},"effectiveStatus":"ALLOWED","intel":{"botType":"training","respectsRobots":"yes","docsUrl":"https://support.claude.com/en/articles/8896518-does-anthropic-crawl-data-from-the-web-and-how-can-site-owners-block-the-crawler","ipRangesUrl":"https://claude.com/crawling/bots.json","caveat":"Also honors the non-standard Crawl-delay directive. Anthropic publishes its crawler IP ranges (claude.com/crawling/bots.json) for verification, but says opt-out should be done in robots.txt: blocking IPs can stop it reading your robots.txt."},"robotsApplies":true},"DuckAssistBot":{"label":"DuckDuckGo AI","platform":"DuckDuckGo","tier":2,"emoji":"🟡","cat":"ai-bots","safety":"safe","impact":"low","purpose":"user-request","robotsTxt":{"status":"ALLOWED","rule":"Full content access; 2 path rules (/assets/, /quote)","source":"wildcard"},"effectiveStatus":"ALLOWED","intel":{"botType":"retrieval","respectsRobots":"yes","docsUrl":"https://help.duckduckgo.com/duckduckgo-help-pages/results/duckassistbot/","caveat":"DuckDuckGo states content is not used to train AI models. Retrieval only."},"robotsApplies":true},"GPTBot":{"label":"ChatGPT Training","platform":"OpenAI","tier":1,"emoji":"🟢","cat":"ai-bots","safety":"safe","impact":"low","purpose":"training","robotsTxt":{"status":"ALLOWED","rule":"Full content access; 2 path rules (/assets/, /quote)","source":"wildcard"},"effectiveStatus":"ALLOWED","intel":{"botType":"training","respectsRobots":"yes","docsUrl":"https://developers.openai.com/api/docs/bots","ipRangesUrl":"https://openai.com/gptbot.json"},"robotsApplies":true},"Google-Extended":{"label":"Gemini AI","platform":"Google","tier":1,"emoji":"🔴","cat":"ai-bots","safety":"safe","impact":"low","purpose":"training","robotsTxt":{"status":"ALLOWED","rule":"Full content access; 2 path rules (/assets/, /quote)","source":"wildcard"},"effectiveStatus":"ALLOWED","intel":{"botType":"opt-out-token","respectsRobots":"na","docsUrl":"https://developers.google.com/crawling/docs/crawlers-fetchers/google-common-crawlers","caveat":"NOT a crawler. A robots.txt control token: disallowing it opts your content out of Gemini training and grounding. It never appears in server logs and has zero effect on Google Search rankings."},"robotsApplies":true,"isOptOutToken":true},"MistralAI-Index":{"label":"Mistral Search Index","platform":"Mistral AI","tier":2,"emoji":"🟠","cat":"ai-bots","safety":"safe","impact":"low","purpose":"indexing","robotsTxt":{"status":"ALLOWED","rule":"Full content access; 2 path rules (/assets/, /quote)","source":"wildcard"},"effectiveStatus":"ALLOWED","intel":{"botType":"retrieval","respectsRobots":"yes","docsUrl":"https://docs.mistral.ai/robots/"},"robotsApplies":true},"MistralAI-User":{"label":"Mistral AI (Vibe)","platform":"Mistral AI","tier":2,"emoji":"🟠","cat":"ai-bots","safety":"safe","impact":"low","purpose":"user-request","robotsTxt":{"status":"ALLOWED","rule":"Full content access; 2 path rules (/assets/, /quote)","source":"wildcard"},"effectiveStatus":"ALLOWED","intel":{"botType":"user-triggered","respectsRobots":"yes","docsUrl":"https://docs.mistral.ai/robots/"},"robotsApplies":true},"MistralAI-User-1.0":{"label":"Mistral AI v1","platform":"Mistral AI","tier":2,"emoji":"🟠","cat":"ai-bots","safety":"safe","impact":"low","purpose":"user-request","robotsTxt":{"status":"ALLOWED","rule":"Full content access; 2 path rules (/assets/, /quote)","source":"wildcard"},"effectiveStatus":"ALLOWED","intel":{"botType":"user-triggered","respectsRobots":"yes","docsUrl":"https://docs.mistral.ai/robots/"},"robotsApplies":true},"OAI-AdsBot":{"label":"OpenAI Ads Validator","platform":"OpenAI","tier":2,"emoji":"🟢","cat":"ai-bots","safety":"safe","impact":"low","purpose":"monitoring","robotsTxt":{"status":"ALLOWED","rule":"Full content access; 2 path rules (/assets/, /quote)","source":"wildcard"},"effectiveStatus":"ALLOWED","intel":{"botType":"other","respectsRobots":"yes","docsUrl":"https://developers.openai.com/api/docs/bots"},"robotsApplies":true},"OAI-SearchBot":{"label":"OpenAI Search","platform":"OpenAI","tier":1,"emoji":"🟢","cat":"ai-bots","safety":"safe","impact":"low","purpose":"user-request","robotsTxt":{"status":"ALLOWED","rule":"Full content access; 2 path rules (/assets/, /quote)","source":"wildcard"},"effectiveStatus":"ALLOWED","intel":{"botType":"retrieval","respectsRobots":"yes","docsUrl":"https://developers.openai.com/api/docs/bots","ipRangesUrl":"https://openai.com/searchbot.json","caveat":"Blocking this removes you from ChatGPT Search results. You can block GPTBot (training) while allowing OAI-SearchBot."},"robotsApplies":true},"Perplexity-User":{"label":"Perplexity User","platform":"Perplexity AI","tier":2,"emoji":"🔵","cat":"ai-bots","safety":"caution","impact":"low","purpose":"user-request","robotsTxt":{"status":"ALLOWED","rule":"Full content access; 2 path rules (/assets/, /quote)","source":"wildcard"},"effectiveStatus":"ALLOWED","intel":{"botType":"user-triggered","respectsRobots":"no","docsUrl":"https://docs.perplexity.ai/guides/bots","ipRangesUrl":"https://www.perplexity.com/perplexity-user.json","caveat":"Perplexity: \"Since a user requested the fetch, this fetcher generally ignores robots.txt rules.\" Not used for web crawling or to train AI foundation models."},"robotsApplies":false},"Perplexity-User-1.0":{"label":"Perplexity User v1","platform":"Perplexity AI","tier":2,"emoji":"🔵","cat":"ai-bots","safety":"caution","impact":"low","purpose":"user-request","robotsTxt":{"status":"ALLOWED","rule":"Full content access; 2 path rules (/assets/, /quote)","source":"wildcard"},"effectiveStatus":"ALLOWED","intel":{"botType":"user-triggered","respectsRobots":"inconsistent","docsUrl":"https://docs.perplexity.ai/guides/bots"},"robotsApplies":true},"PerplexityBot":{"label":"Perplexity AI","platform":"Perplexity AI","tier":1,"emoji":"🔵","cat":"ai-bots","safety":"safe","impact":"low","purpose":"both","robotsTxt":{"status":"ALLOWED","rule":"Full content access; 2 path rules (/assets/, /quote)","source":"wildcard"},"effectiveStatus":"ALLOWED","intel":{"botType":"retrieval","respectsRobots":"inconsistent","docsUrl":"https://docs.perplexity.ai/guides/bots","ipRangesUrl":"https://www.perplexity.com/perplexitybot.json","caveat":"Documentation says it respects robots.txt, but Cloudflare published evidence (Aug 2025) of Perplexity using undeclared stealth crawlers that bypass robots.txt, and de-listed Perplexity from its Verified Bots program. Do not rely on robots.txt alone for a hard block."},"robotsApplies":true},"img2dataset":{"label":"Image Dataset","platform":"Open Source Community","tier":2,"emoji":"🤖","cat":"ai-bots","safety":"safe","impact":"low","purpose":"training","robotsTxt":{"status":"ALLOWED","rule":"Full content access; 2 path rules (/assets/, /quote)","source":"wildcard"},"effectiveStatus":"ALLOWED","intel":{"botType":"training","respectsRobots":"unknown"},"robotsApplies":true},"Meta-ExternalAgent":{"label":"Meta AI Training","platform":"Meta","tier":1,"emoji":"🔵","cat":"ai-bots","safety":"caution","impact":"medium","purpose":"training","robotsTxt":{"status":"ALLOWED","rule":"Full content access; 2 path rules (/assets/, /quote)","source":"wildcard"},"effectiveStatus":"ALLOWED","intel":{"botType":"training","respectsRobots":"yes","docsUrl":"https://developers.facebook.com/docs/sharing/webmasters/web-crawlers/","caveat":"Trains Meta AI across Facebook, Instagram, WhatsApp, Messenger. Do NOT confuse with facebookexternalhit (link previews)."},"robotsApplies":true},"Meta-ExternalFetcher":{"label":"Meta AI Fetcher","platform":"Meta","tier":2,"emoji":"🔵","cat":"ai-bots","safety":"caution","impact":"low","purpose":"user-request","robotsTxt":{"status":"ALLOWED","rule":"Full content access; 2 path rules (/assets/, /quote)","source":"wildcard"},"effectiveStatus":"ALLOWED","intel":{"botType":"user-triggered","respectsRobots":"inconsistent","docsUrl":"https://developers.facebook.com/docs/sharing/webmasters/web-crawlers/","caveat":"Meta: it \"may bypass robots.txt because it performs fetches that were requested by the user\". Separate from Meta-ExternalAgent (training) and facebookexternalhit (link previews)."},"robotsApplies":true},"meta-webindexer":{"label":"Meta Web Indexer","platform":"Meta","tier":2,"emoji":"🔵","cat":"ai-bots","safety":"caution","impact":"low","purpose":"indexing","robotsTxt":{"status":"ALLOWED","rule":"Full content access; 2 path rules (/assets/, /quote)","source":"wildcard"},"effectiveStatus":"ALLOWED","intel":{"botType":"retrieval","respectsRobots":"yes","docsUrl":"https://developers.facebook.com/docs/sharing/webmasters/web-crawlers/"},"robotsApplies":true},"Amzn-User":{"label":"Amazon User Agent","platform":"Amazon","tier":2,"emoji":"🟠","cat":"ai-bots","safety":"safe","impact":"low","purpose":"user-request","robotsTxt":{"status":"ALLOWED","rule":"Full content access; 2 path rules (/assets/, /quote)","source":"wildcard"},"effectiveStatus":"ALLOWED","intel":{"docsUrl":"https://developer.amazon.com/amazonbot","botType":"user-triggered","respectsRobots":"yes"},"robotsApplies":true},"bedrockbot":{"label":"AWS Bedrock Bot","platform":"Amazon","tier":2,"emoji":"🟠","cat":"ai-bots","safety":"safe","impact":"low","purpose":"training","robotsTxt":{"status":"ALLOWED","rule":"Full content access; 2 path rules (/assets/, /quote)","source":"wildcard"},"effectiveStatus":"ALLOWED","intel":{"docsUrl":"https://docs.aws.amazon.com/bedrock/latest/userguide/webcrawl-data-source-connector.html","botType":"user-triggered","respectsRobots":"yes"},"robotsApplies":true},"Kimi-User":{"label":"Kimi (Moonshot AI)","platform":"Moonshot AI","tier":2,"emoji":"🟣","cat":"ai-bots","safety":"safe","impact":"low","purpose":"user-request","robotsTxt":{"status":"ALLOWED","rule":"Full content access; 2 path rules (/assets/, /quote)","source":"wildcard"},"effectiveStatus":"ALLOWED","intel":{"docsUrl":"https://www.kimi.ai/policies/kimi-crawlers","botType":"user-triggered","respectsRobots":"unknown"},"robotsApplies":true},"FirecrawlAgent":{"label":"Firecrawl Agent","platform":"Firecrawl","tier":2,"emoji":"🔥","cat":"ai-bots","safety":"caution","impact":"medium","purpose":"scraping","robotsTxt":{"status":"ALLOWED","rule":"Full content access; 2 path rules (/assets/, /quote)","source":"wildcard"},"effectiveStatus":"ALLOWED","intel":{"docsUrl":"https://docs.firecrawl.dev/advanced-scraping-guide","botType":"user-triggered","respectsRobots":"yes","caveat":"Multi-tenant scraper-as-a-service: many customers drive the same crawler. Blocking it blocks every downstream customer at once, including agentic tools you may want."},"robotsApplies":true},"Amzn-SearchBot":{"label":"Amzn-SearchBot","platform":"Amazon","tier":3,"emoji":"🔍","cat":"search-engines","safety":"safe","impact":"critical","purpose":"indexing","robotsTxt":{"status":"ALLOWED","rule":"Full content access; 2 path rules (/assets/, /quote)","source":"wildcard"},"effectiveStatus":"ALLOWED","intel":{"docsUrl":"https://developer.amazon.com/amazonbot","botType":"search","respectsRobots":"yes"},"robotsApplies":true},"Applebot":{"label":"Apple Search","platform":"Apple","tier":3,"emoji":"⚪","cat":"search-engines","safety":"safe","impact":"critical","purpose":"indexing","robotsTxt":{"status":"ALLOWED","rule":"Full content access; 2 path rules (/assets/, /quote)","source":"wildcard"},"effectiveStatus":"ALLOWED","intel":{"botType":"search","respectsRobots":"yes","docsUrl":"https://support.apple.com/en-us/119829","caveat":"Powers Siri, Spotlight and Safari Suggestions. Falls back to Googlebot directives when no Applebot-specific rules exist. AI training use is controlled separately via the Applebot-Extended token."},"robotsApplies":true},"Applebot-Extended":{"label":"Apple Intelligence","platform":"Apple","tier":2,"emoji":"⚪","cat":"search-engines","safety":"caution","impact":"critical","purpose":"training","robotsTxt":{"status":"ALLOWED","rule":"Full content access; 2 path rules (/assets/, /quote)","source":"wildcard"},"effectiveStatus":"ALLOWED","intel":{"botType":"opt-out-token","respectsRobots":"na","docsUrl":"https://support.apple.com/en-us/119829","caveat":"NOT a crawler. A robots.txt control token: disallowing it opts content crawled by Applebot out of Apple Intelligence model training. It never appears in server logs and does not affect Siri/Spotlight search inclusion."},"robotsApplies":true,"isOptOutToken":true},"Baiduspider":{"label":"Baidu Search","platform":"Baidu","tier":3,"emoji":"🔍","cat":"search-engines","safety":"safe","impact":"critical","purpose":"indexing","robotsTxt":{"status":"ALLOWED","rule":"Full content access; 2 path rules (/assets/, /quote)","source":"wildcard"},"effectiveStatus":"ALLOWED","intel":{"botType":"search","respectsRobots":"unknown"},"robotsApplies":true},"bingbot":{"label":"Bing Search","platform":"Microsoft","tier":3,"emoji":"🔵","cat":"search-engines","safety":"safe","impact":"critical","purpose":"indexing","robotsTxt":{"status":"ALLOWED","rule":"Full content access; 2 path rules (/assets/, /quote)","source":"wildcard"},"effectiveStatus":"ALLOWED","intel":{"botType":"search","respectsRobots":"yes","docsUrl":"https://www.bing.com/webmasters/help/which-crawlers-does-bing-use-8c184ec0","ipRangesUrl":"https://www.bing.com/toolbox/bingbot.json","caveat":"Feeds both Bing Search AND Microsoft Copilot retrieval. Blocking bingbot removes you from Copilot answers too. Copilot Actions (agentic) uses plain Edge browser UAs with no bot signal."},"robotsApplies":true},"DuckDuckBot":{"label":"DuckDuckGo Search","platform":"DuckDuckGo","tier":3,"emoji":"🟡","cat":"search-engines","safety":"safe","impact":"critical","purpose":"indexing","robotsTxt":{"status":"ALLOWED","rule":"Full content access; 2 path rules (/assets/, /quote)","source":"wildcard"},"effectiveStatus":"ALLOWED","intel":{"botType":"search","respectsRobots":"unknown"},"robotsApplies":true},"Googlebot":{"label":"Google Search","platform":"Google","tier":3,"emoji":"🔴","cat":"search-engines","safety":"safe","impact":"critical","purpose":"indexing","robotsTxt":{"status":"ALLOWED","rule":"Full content access; 2 path rules (/assets/, /quote)","source":"wildcard"},"effectiveStatus":"ALLOWED","intel":{"botType":"search","respectsRobots":"yes","docsUrl":"https://developers.google.com/crawling/docs/crawlers-fetchers/google-common-crawlers","ipRangesUrl":"https://developers.google.com/static/crawling/ipranges/common-crawlers.json"},"robotsApplies":true},"Mojeek":{"label":"Mojeek","platform":"Mojeek","tier":3,"emoji":"🔍","cat":"search-engines","safety":"safe","impact":"critical","purpose":"indexing","robotsTxt":{"status":"ALLOWED","rule":"Full content access; 2 path rules (/assets/, /quote)","source":"wildcard"},"effectiveStatus":"ALLOWED","intel":{"botType":"search","respectsRobots":"unknown"},"robotsApplies":true},"MojeekBot":{"label":"MojeekBot","platform":"Mojeek","tier":3,"emoji":"🔍","cat":"search-engines","safety":"safe","impact":"critical","purpose":"indexing","robotsTxt":{"status":"ALLOWED","rule":"Full content access; 2 path rules (/assets/, /quote)","source":"wildcard"},"effectiveStatus":"ALLOWED","intel":{"botType":"search","respectsRobots":"unknown"},"robotsApplies":true},"PetalBot":{"label":"Petal Search","platform":"Huawei","tier":3,"emoji":"🔴","cat":"search-engines","safety":"safe","impact":"critical","purpose":"indexing","robotsTxt":{"status":"ALLOWED","rule":"Full content access; 2 path rules (/assets/, /quote)","source":"wildcard"},"effectiveStatus":"ALLOWED","intel":{"botType":"search","respectsRobots":"unknown"},"robotsApplies":true},"SeznamBot":{"label":"Seznam Search","platform":"Seznam","tier":3,"emoji":"🔍","cat":"search-engines","safety":"safe","impact":"critical","purpose":"indexing","robotsTxt":{"status":"ALLOWED","rule":"Full content access; 2 path rules (/assets/, /quote)","source":"wildcard"},"effectiveStatus":"ALLOWED","intel":{"botType":"search","respectsRobots":"unknown"},"robotsApplies":true},"Yandex":{"label":"Yandex","platform":"Yandex","tier":3,"emoji":"🔍","cat":"search-engines","safety":"safe","impact":"critical","purpose":"indexing","robotsTxt":{"status":"ALLOWED","rule":"Full content access; 2 path rules (/assets/, /quote)","source":"wildcard"},"effectiveStatus":"ALLOWED","intel":{"botType":"search","respectsRobots":"unknown"},"robotsApplies":true},"YandexAdditional":{"label":"YandexAdditional","platform":"Yandex","tier":3,"emoji":"🔍","cat":"search-engines","safety":"safe","impact":"critical","purpose":"indexing","robotsTxt":{"status":"ALLOWED","rule":"Full content access; 2 path rules (/assets/, /quote)","source":"wildcard"},"effectiveStatus":"ALLOWED","intel":{"botType":"search","respectsRobots":"unknown"},"robotsApplies":true},"YandexAdditionalBot":{"label":"YandexAdditionalBot","platform":"Yandex","tier":3,"emoji":"🔍","cat":"search-engines","safety":"safe","impact":"critical","purpose":"indexing","robotsTxt":{"status":"ALLOWED","rule":"Full content access; 2 path rules (/assets/, /quote)","source":"wildcard"},"effectiveStatus":"ALLOWED","intel":{"botType":"search","respectsRobots":"unknown"},"robotsApplies":true},"YandexBot":{"label":"Yandex Search","platform":"Yandex","tier":3,"emoji":"🔍","cat":"search-engines","safety":"safe","impact":"critical","purpose":"indexing","robotsTxt":{"status":"ALLOWED","rule":"Full content access; 2 path rules (/assets/, /quote)","source":"wildcard"},"effectiveStatus":"ALLOWED","intel":{"botType":"search","respectsRobots":"unknown"},"robotsApplies":true},"baidu":{"label":"baidu","platform":"Baidu","tier":3,"emoji":"🔍","cat":"search-engines","safety":"safe","impact":"critical","purpose":"indexing","robotsTxt":{"status":"ALLOWED","rule":"Full content access; 2 path rules (/assets/, /quote)","source":"wildcard"},"effectiveStatus":"ALLOWED","intel":{"botType":"search","respectsRobots":"unknown"},"robotsApplies":true},"AhrefsBot":{"label":"Ahrefs SEO","platform":"Ahrefs","tier":4,"emoji":"🔧","cat":"seo-tools","safety":"safe","impact":"unknown","purpose":"monitoring","robotsTxt":{"status":"ALLOWED","rule":"Full content access; 2 path rules (/assets/, /quote)","source":"wildcard"},"effectiveStatus":"ALLOWED","intel":{"botType":"other","respectsRobots":"unknown"},"robotsApplies":true},"AwarioRssBot":{"label":"AwarioRssBot","platform":"Awario","tier":4,"emoji":"🔧","cat":"seo-tools","safety":"safe","impact":"unknown","purpose":"monitoring","robotsTxt":{"status":"ALLOWED","rule":"Full content access; 2 path rules (/assets/, /quote)","source":"wildcard"},"effectiveStatus":"ALLOWED","intel":{"botType":"other","respectsRobots":"unknown"},"robotsApplies":true},"AwarioSmartBot":{"label":"AwarioSmartBot","platform":"Awario","tier":4,"emoji":"🔧","cat":"seo-tools","safety":"safe","impact":"unknown","purpose":"monitoring","robotsTxt":{"status":"ALLOWED","rule":"Full content access; 2 path rules (/assets/, /quote)","source":"wildcard"},"effectiveStatus":"ALLOWED","intel":{"botType":"other","respectsRobots":"unknown"},"robotsApplies":true},"DataForSeoBot":{"label":"DataForSEO","platform":"DataForSEO","tier":4,"emoji":"🔧","cat":"seo-tools","safety":"safe","impact":"unknown","purpose":"monitoring","robotsTxt":{"status":"ALLOWED","rule":"Full content access; 2 path rules (/assets/, /quote)","source":"wildcard"},"effectiveStatus":"ALLOWED","intel":{"botType":"other","respectsRobots":"unknown"},"robotsApplies":true},"DotBot":{"label":"Moz SEO","platform":"Moz","tier":4,"emoji":"🔧","cat":"seo-tools","safety":"caution","impact":"low","purpose":"monitoring","robotsTxt":{"status":"ALLOWED","rule":"Full content access; 2 path rules (/assets/, /quote)","source":"wildcard"},"effectiveStatus":"ALLOWED","intel":{"botType":"other","respectsRobots":"unknown"},"robotsApplies":true},"MJ12bot":{"label":"Majestic SEO","platform":"Majestic","tier":4,"emoji":"🔧","cat":"seo-tools","safety":"caution","impact":"low","purpose":"monitoring","robotsTxt":{"status":"ALLOWED","rule":"Full content access; 2 path rules (/assets/, /quote)","source":"wildcard"},"effectiveStatus":"ALLOWED","intel":{"botType":"other","respectsRobots":"unknown"},"robotsApplies":true},"NeticleBot":{"label":"NeticleBot","platform":"Neticle","tier":4,"emoji":"🔧","cat":"seo-tools","safety":"safe","impact":"unknown","purpose":"monitoring","robotsTxt":{"status":"ALLOWED","rule":"Full content access; 2 path rules (/assets/, /quote)","source":"wildcard"},"effectiveStatus":"ALLOWED","intel":{"botType":"other","respectsRobots":"unknown"},"robotsApplies":true},"Screaming Frog SEO Spider":{"label":"Screaming Frog","platform":"Screaming Frog","tier":4,"emoji":"🔧","cat":"seo-tools","safety":"safe","impact":"unknown","purpose":"monitoring","robotsTxt":{"status":"ALLOWED","rule":"Full content access; 2 path rules (/assets/, /quote)","source":"wildcard"},"effectiveStatus":"ALLOWED","intel":{"botType":"other","respectsRobots":"unknown"},"robotsApplies":true},"SemrushBot":{"label":"Semrush SEO","platform":"Semrush","tier":4,"emoji":"🔧","cat":"seo-tools","safety":"safe","impact":"unknown","purpose":"monitoring","robotsTxt":{"status":"ALLOWED","rule":"Full content access; 2 path rules (/assets/, /quote)","source":"wildcard"},"effectiveStatus":"ALLOWED","intel":{"botType":"other","respectsRobots":"unknown"},"robotsApplies":true},"SemrushBot-OCOB":{"label":"SemrushBot-OCOB","platform":"Semrush","tier":4,"emoji":"🔧","cat":"seo-tools","safety":"caution","impact":"unknown","purpose":"monitoring","robotsTxt":{"status":"ALLOWED","rule":"Full content access; 2 path rules (/assets/, /quote)","source":"wildcard"},"effectiveStatus":"ALLOWED","intel":{"botType":"other","respectsRobots":"unknown"},"robotsApplies":true},"LinkedInBot":{"label":"LinkedIn Crawler","platform":"LinkedIn","tier":5,"emoji":"💬","cat":"social-bots","safety":"safe","impact":"unknown","purpose":"social","robotsTxt":{"status":"ALLOWED","rule":"Full content access; 2 path rules (/assets/, /quote)","source":"wildcard"},"effectiveStatus":"ALLOWED","intel":{"botType":"other","respectsRobots":"unknown"},"robotsApplies":true},"Pinterestbot":{"label":"Pinterest Crawler","platform":"Pinterest","tier":5,"emoji":"💬","cat":"social-bots","safety":"safe","impact":"unknown","purpose":"social","robotsTxt":{"status":"ALLOWED","rule":"Full content access; 2 path rules (/assets/, /quote)","source":"wildcard"},"effectiveStatus":"ALLOWED","intel":{"botType":"other","respectsRobots":"unknown"},"robotsApplies":true},"Slackbot":{"label":"Slack Preview","platform":"Slack","tier":5,"emoji":"💬","cat":"social-bots","safety":"safe","impact":"unknown","purpose":"social","robotsTxt":{"status":"ALLOWED","rule":"Full content access; 2 path rules (/assets/, /quote)","source":"wildcard"},"effectiveStatus":"ALLOWED","intel":{"botType":"other","respectsRobots":"unknown"},"robotsApplies":true},"Twitterbot":{"label":"Twitter/X Crawler","platform":"X (Twitter)","tier":5,"emoji":"💬","cat":"social-bots","safety":"safe","impact":"unknown","purpose":"social","robotsTxt":{"status":"ALLOWED","rule":"Full content access; 2 path rules (/assets/, /quote)","source":"wildcard"},"effectiveStatus":"ALLOWED","intel":{"botType":"other","respectsRobots":"unknown"},"robotsApplies":true},"facebookexternalhit":{"label":"Facebook Preview","platform":"Meta","tier":5,"emoji":"🔵","cat":"social-bots","safety":"safe","impact":"unknown","purpose":"social","robotsTxt":{"status":"ALLOWED","rule":"Full content access; 2 path rules (/assets/, /quote)","source":"wildcard"},"effectiveStatus":"ALLOWED","intel":{"botType":"other","respectsRobots":"yes","docsUrl":"https://developers.facebook.com/docs/sharing/webmasters/web-crawlers/","caveat":"NOT an AI bot. This is the link-preview fetcher for Facebook/WhatsApp shares. Blocking it breaks link previews across Meta products. Many AI blocklists conflate it with Meta-ExternalAgent by mistake."},"robotsApplies":true},"Google-InspectionTool":{"label":"Google-InspectionTool","platform":"Google","tier":3,"emoji":"🔴","cat":"google-bots","safety":"safe","impact":"critical","purpose":"monitoring","robotsTxt":{"status":"ALLOWED","rule":"Full content access; 2 path rules (/assets/, /quote)","source":"wildcard"},"effectiveStatus":"ALLOWED","intel":{"botType":"other","respectsRobots":"unknown"},"robotsApplies":true},"GoogleOther":{"label":"GoogleOther","platform":"Google","tier":3,"emoji":"🔴","cat":"google-bots","safety":"safe","impact":"critical","purpose":"indexing","robotsTxt":{"status":"ALLOWED","rule":"Full content access; 2 path rules (/assets/, /quote)","source":"wildcard"},"effectiveStatus":"ALLOWED","intel":{"botType":"search","respectsRobots":"unknown"},"robotsApplies":true},"GoogleOther-Image":{"label":"GoogleOther-Image","platform":"Google","tier":3,"emoji":"🔴","cat":"google-bots","safety":"safe","impact":"critical","purpose":"indexing","robotsTxt":{"status":"ALLOWED","rule":"Full content access; 2 path rules (/assets/, /quote)","source":"wildcard"},"effectiveStatus":"ALLOWED","intel":{"botType":"search","respectsRobots":"unknown"},"robotsApplies":true},"GoogleOther-Video":{"label":"GoogleOther-Video","platform":"Google","tier":3,"emoji":"🔴","cat":"google-bots","safety":"safe","impact":"critical","purpose":"indexing","robotsTxt":{"status":"ALLOWED","rule":"Full content access; 2 path rules (/assets/, /quote)","source":"wildcard"},"effectiveStatus":"ALLOWED","intel":{"botType":"search","respectsRobots":"unknown"},"robotsApplies":true},"Googlebot-Image":{"label":"Googlebot-Image","platform":"Google","tier":3,"emoji":"🔴","cat":"google-bots","safety":"safe","impact":"critical","purpose":"indexing","robotsTxt":{"status":"BLOCKED","rule":"Disallow: /","source":"specific"},"effectiveStatus":"BLOCKED","intel":{"botType":"search","respectsRobots":"unknown"},"robotsApplies":true},"Googlebot-News":{"label":"Googlebot-News","platform":"Google","tier":3,"emoji":"🔴","cat":"google-bots","safety":"safe","impact":"critical","purpose":"indexing","robotsTxt":{"status":"ALLOWED","rule":"Full content access; 2 path rules (/assets/, /quote)","source":"wildcard"},"effectiveStatus":"ALLOWED","intel":{"botType":"search","respectsRobots":"unknown"},"robotsApplies":true},"Googlebot-Video":{"label":"Googlebot-Video","platform":"Google","tier":3,"emoji":"🔴","cat":"google-bots","safety":"safe","impact":"critical","purpose":"indexing","robotsTxt":{"status":"ALLOWED","rule":"Full content access; 2 path rules (/assets/, /quote)","source":"wildcard"},"effectiveStatus":"ALLOWED","intel":{"botType":"search","respectsRobots":"unknown"},"robotsApplies":true},"Google-Agent":{"label":"Google Agent (Project Mariner)","platform":"Google","tier":2,"emoji":"🔴","cat":"google-bots","safety":"safe","impact":"low","purpose":"user-request","robotsTxt":{"status":"ALLOWED","rule":"Full content access; 2 path rules (/assets/, /quote)","source":"wildcard"},"effectiveStatus":"ALLOWED","intel":{"botType":"user-triggered","respectsRobots":"no","docsUrl":"https://developers.google.com/crawling/docs/crawlers-fetchers/google-user-triggered-fetchers","ipRangesUrl":"https://developers.google.com/static/crawling/ipranges/user-triggered-agents.json","webBotAuth":"experimental","caveat":"Google policy: user-requested fetches generally IGNORE robots.txt. Blocking it in robots.txt does not work; server-side controls are required."},"robotsApplies":false},"Google-Read-Aloud":{"label":"Google Read Aloud","platform":"Google","tier":3,"emoji":"🔴","cat":"google-bots","safety":"safe","impact":"low","purpose":"user-request","robotsTxt":{"status":"ALLOWED","rule":"Full content access; 2 path rules (/assets/, /quote)","source":"wildcard"},"effectiveStatus":"ALLOWED","intel":{"botType":"user-triggered","respectsRobots":"no","docsUrl":"https://developers.google.com/crawling/docs/crawlers-fetchers/google-user-triggered-fetchers","caveat":"User-triggered fetcher: ignores robots.txt by Google policy."},"robotsApplies":false},"Google-Site-Verification":{"label":"Google Site Verification","platform":"Google","tier":3,"emoji":"🔴","cat":"google-bots","safety":"safe","impact":"critical","purpose":"monitoring","robotsTxt":{"status":"ALLOWED","rule":"Full content access; 2 path rules (/assets/, /quote)","source":"wildcard"},"effectiveStatus":"ALLOWED","intel":{"botType":"user-triggered","respectsRobots":"no","docsUrl":"https://developers.google.com/crawling/docs/crawlers-fetchers/google-user-triggered-fetchers","caveat":"User-triggered fetcher: ignores robots.txt by Google policy."},"robotsApplies":false},"FeedFetcher-Google":{"label":"Google Feed Fetcher","platform":"Google","tier":3,"emoji":"🔴","cat":"google-bots","safety":"safe","impact":"critical","purpose":"indexing","robotsTxt":{"status":"ALLOWED","rule":"Full content access; 2 path rules (/assets/, /quote)","source":"wildcard"},"effectiveStatus":"ALLOWED","intel":{"botType":"user-triggered","respectsRobots":"no","docsUrl":"https://developers.google.com/crawling/docs/crawlers-fetchers/google-user-triggered-fetchers","caveat":"User-triggered fetcher: ignores robots.txt by Google policy."},"robotsApplies":false},"GoogleProducer":{"label":"Google News Producer","platform":"Google","tier":3,"emoji":"🔴","cat":"google-bots","safety":"safe","impact":"critical","purpose":"indexing","robotsTxt":{"status":"ALLOWED","rule":"Full content access; 2 path rules (/assets/, /quote)","source":"wildcard"},"effectiveStatus":"ALLOWED","intel":{"botType":"user-triggered","respectsRobots":"no","docsUrl":"https://developers.google.com/crawling/docs/crawlers-fetchers/google-user-triggered-fetchers","caveat":"User-triggered fetcher: ignores robots.txt by Google policy."},"robotsApplies":false},"Storebot-Google":{"label":"Storebot-Google","platform":"Google","tier":3,"emoji":"🔴","cat":"google-bots","safety":"safe","impact":"critical","purpose":"indexing","robotsTxt":{"status":"BLOCKED","rule":"Disallow: /","source":"specific"},"effectiveStatus":"BLOCKED","intel":{"botType":"search","respectsRobots":"unknown"},"robotsApplies":true},"Amazon Kendra":{"label":"Amazon Kendra","platform":"Amazon","tier":6,"emoji":"🕷","cat":"scrapers","safety":"safe","impact":"low","purpose":"scraping","robotsTxt":{"status":"ALLOWED","rule":"Full content access; 2 path rules (/assets/, /quote)","source":"wildcard"},"effectiveStatus":"ALLOWED","intel":{"botType":"other","respectsRobots":"unknown"},"robotsApplies":true},"Barkrowler":{"label":"Barkrowler","platform":"Babbar","tier":6,"emoji":"🕷","cat":"scrapers","safety":"safe","impact":"low","purpose":"scraping","robotsTxt":{"status":"ALLOWED","rule":"Full content access; 2 path rules (/assets/, /quote)","source":"wildcard"},"effectiveStatus":"ALLOWED","intel":{"botType":"other","respectsRobots":"unknown"},"robotsApplies":true},"CCBot":{"label":"Common Crawl","platform":"Common Crawl","tier":6,"emoji":"🕷","cat":"scrapers","safety":"caution","impact":"low","purpose":"training","robotsTxt":{"status":"ALLOWED","rule":"Full content access; 2 path rules (/assets/, /quote)","source":"wildcard"},"effectiveStatus":"ALLOWED","intel":{"botType":"training","respectsRobots":"yes","docsUrl":"https://commoncrawl.org/ccbot","ipRangesUrl":"https://index.commoncrawl.org/ccbot.json","caveat":"Common Crawl feeds nearly every open LLM training dataset. Blocking CCBot opts you out of most open-source model training corpora, not just one vendor. A larger opt-out than most operators realize."},"robotsApplies":true},"Crawl4AI":{"label":"Crawl4AI","platform":"Unknown","tier":6,"emoji":"🕷","cat":"scrapers","safety":"safe","impact":"low","purpose":"scraping","robotsTxt":{"status":"ALLOWED","rule":"Full content access; 2 path rules (/assets/, /quote)","source":"wildcard"},"effectiveStatus":"ALLOWED","intel":{"botType":"other","respectsRobots":"unknown"},"robotsApplies":true},"Diffbot":{"label":"Diffbot Scraper","platform":"Diffbot","tier":6,"emoji":"🕷","cat":"scrapers","safety":"safe","impact":"low","purpose":"scraping","robotsTxt":{"status":"ALLOWED","rule":"Full content access; 2 path rules (/assets/, /quote)","source":"wildcard"},"effectiveStatus":"ALLOWED","intel":{"docsUrl":"https://docs.diffbot.com/docs/does-crawl-respect-robotstxt","botType":"training","respectsRobots":"inconsistent","caveat":"Commercial extraction service. Respects robots.txt by default but per-customer configuration can override it."},"robotsApplies":true},"ICC-Crawler":{"label":"ICC-Crawler","platform":"ICC","tier":6,"emoji":"🕷","cat":"scrapers","safety":"caution","impact":"low","purpose":"scraping","robotsTxt":{"status":"ALLOWED","rule":"Full content access; 2 path rules (/assets/, /quote)","source":"wildcard"},"effectiveStatus":"ALLOWED","intel":{"botType":"other","respectsRobots":"unknown"},"robotsApplies":true},"ImagesiftBot":{"label":"ImageSift (Hive)","platform":"Hive","tier":6,"emoji":"🕷","cat":"scrapers","safety":"safe","impact":"low","purpose":"scraping","robotsTxt":{"status":"ALLOWED","rule":"Full content access; 2 path rules (/assets/, /quote)","source":"wildcard"},"effectiveStatus":"ALLOWED","intel":{"botType":"other","respectsRobots":"yes","docsUrl":"https://imagesift.com/about","caveat":"Image crawler for ImageSift reverse-image search (Hive). Its relationship to Hive visual AI training is not documented by the vendor."},"robotsApplies":true},"Scrapy":{"label":"Scrapy","platform":"Unknown","tier":6,"emoji":"🕷","cat":"scrapers","safety":"safe","impact":"low","purpose":"scraping","robotsTxt":{"status":"ALLOWED","rule":"Full content access; 2 path rules (/assets/, /quote)","source":"wildcard"},"effectiveStatus":"ALLOWED","intel":{"botType":"other","respectsRobots":"unknown"},"robotsApplies":true},"SeekportBot":{"label":"SeekportBot","platform":"Seekport","tier":6,"emoji":"🕷","cat":"scrapers","safety":"safe","impact":"low","purpose":"scraping","robotsTxt":{"status":"ALLOWED","rule":"Full content access; 2 path rules (/assets/, /quote)","source":"wildcard"},"effectiveStatus":"ALLOWED","intel":{"botType":"other","respectsRobots":"unknown"},"robotsApplies":true},"VelenPublicWebCrawler":{"label":"VelenPublicWebCrawler","platform":"Velen","tier":6,"emoji":"🕷","cat":"scrapers","safety":"safe","impact":"low","purpose":"scraping","robotsTxt":{"status":"ALLOWED","rule":"Full content access; 2 path rules (/assets/, /quote)","source":"wildcard"},"effectiveStatus":"ALLOWED","intel":{"botType":"other","respectsRobots":"unknown"},"robotsApplies":true},"coccocbot-web":{"label":"coccocbot-web","platform":"Unknown","tier":6,"emoji":"🕷","cat":"scrapers","safety":"safe","impact":"low","purpose":"scraping","robotsTxt":{"status":"ALLOWED","rule":"Full content access; 2 path rules (/assets/, /quote)","source":"wildcard"},"effectiveStatus":"ALLOWED","intel":{"botType":"other","respectsRobots":"unknown"},"robotsApplies":true},"crawler4j":{"label":"crawler4j","platform":"Unknown","tier":6,"emoji":"🕷","cat":"scrapers","safety":"safe","impact":"low","purpose":"scraping","robotsTxt":{"status":"ALLOWED","rule":"Full content access; 2 path rules (/assets/, /quote)","source":"wildcard"},"effectiveStatus":"ALLOWED","intel":{"botType":"other","respectsRobots":"unknown"},"robotsApplies":true},"imgproxy":{"label":"imgproxy","platform":"Open Source","tier":6,"emoji":"🕷","cat":"scrapers","safety":"safe","impact":"low","purpose":"scraping","robotsTxt":{"status":"ALLOWED","rule":"Full content access; 2 path rules (/assets/, /quote)","source":"wildcard"},"effectiveStatus":"ALLOWED","intel":{"botType":"other","respectsRobots":"unknown"},"robotsApplies":true},"magpie-crawler":{"label":"magpie-crawler","platform":"Brandwatch","tier":6,"emoji":"🕷","cat":"scrapers","safety":"caution","impact":"low","purpose":"scraping","robotsTxt":{"status":"ALLOWED","rule":"Full content access; 2 path rules (/assets/, /quote)","source":"wildcard"},"effectiveStatus":"ALLOWED","intel":{"botType":"other","respectsRobots":"unknown"},"robotsApplies":true},"news-please":{"label":"news-please","platform":"Open Source","tier":6,"emoji":"🕷","cat":"scrapers","safety":"aggressive","impact":"low","purpose":"scraping","robotsTxt":{"status":"ALLOWED","rule":"Full content access; 2 path rules (/assets/, /quote)","source":"wildcard"},"effectiveStatus":"ALLOWED","intel":{"botType":"other","respectsRobots":"unknown"},"robotsApplies":true},"omgili":{"label":"omgili","platform":"Unknown","tier":6,"emoji":"🕷","cat":"scrapers","safety":"safe","impact":"low","purpose":"scraping","robotsTxt":{"status":"ALLOWED","rule":"Full content access; 2 path rules (/assets/, /quote)","source":"wildcard"},"effectiveStatus":"ALLOWED","intel":{"botType":"other","respectsRobots":"unknown"},"robotsApplies":true},"omgilibot":{"label":"omgilibot","platform":"Unknown","tier":6,"emoji":"🕷","cat":"scrapers","safety":"safe","impact":"low","purpose":"scraping","robotsTxt":{"status":"ALLOWED","rule":"Full content access; 2 path rules (/assets/, /quote)","source":"wildcard"},"effectiveStatus":"ALLOWED","intel":{"botType":"other","respectsRobots":"unknown"},"robotsApplies":true},"yacy":{"label":"yacy","platform":"YaCy Community","tier":6,"emoji":"🕷","cat":"scrapers","safety":"safe","impact":"low","purpose":"indexing","robotsTxt":{"status":"ALLOWED","rule":"Full content access; 2 path rules (/assets/, /quote)","source":"wildcard"},"effectiveStatus":"ALLOWED","intel":{"botType":"retrieval","respectsRobots":"unknown"},"robotsApplies":true},"yacybot":{"label":"yacybot","platform":"YaCy Community","tier":6,"emoji":"🕷","cat":"scrapers","safety":"safe","impact":"low","purpose":"indexing","robotsTxt":{"status":"ALLOWED","rule":"Full content access; 2 path rules (/assets/, /quote)","source":"wildcard"},"effectiveStatus":"ALLOWED","intel":{"botType":"retrieval","respectsRobots":"unknown"},"robotsApplies":true},"AliyunSecBot":{"label":"AliyunSecBot","platform":"Alibaba Cloud","tier":6,"emoji":"☁","cat":"cloud-services","safety":"safe","impact":"unknown","purpose":"monitoring","robotsTxt":{"status":"ALLOWED","rule":"Full content access; 2 path rules (/assets/, /quote)","source":"wildcard"},"effectiveStatus":"ALLOWED","intel":{"botType":"other","respectsRobots":"unknown"},"robotsApplies":true},"Amazonbot":{"label":"Amazon Nova AI","platform":"Amazon","tier":2,"emoji":"☁","cat":"cloud-services","safety":"safe","impact":"low","purpose":"training","robotsTxt":{"status":"ALLOWED","rule":"Full content access; 2 path rules (/assets/, /quote)","source":"wildcard"},"effectiveStatus":"ALLOWED","intel":{"botType":"training","respectsRobots":"yes","docsUrl":"https://developer.amazon.com/amazonbot","caveat":"Indexes for Alexa and trains Amazon Nova. Unique opt-out: a page-level noarchive meta tag excludes a page from Nova training while still allowing Amazonbot to crawl. Verify via reverse DNS (no IP JSON published)."},"robotsApplies":true},"Google-CloudVertexBot":{"label":"Google Vertex AI","platform":"Google","tier":3,"emoji":"🔴","cat":"cloud-services","safety":"safe","impact":"low","purpose":"user-request","robotsTxt":{"status":"ALLOWED","rule":"Full content access; 2 path rules (/assets/, /quote)","source":"wildcard"},"effectiveStatus":"ALLOWED","intel":{"botType":"retrieval","respectsRobots":"yes","docsUrl":"https://developers.google.com/crawling/docs/crawlers-fetchers/google-common-crawlers","ipRangesUrl":"https://developers.google.com/static/crawling/ipranges/common-crawlers.json","caveat":"Enterprise crawler for Vertex AI Agents. Blocking it does not affect Google Search inclusion."},"robotsApplies":true},"TurnitinBot":{"label":"TurnitinBot","platform":"Turnitin","tier":6,"emoji":"📡","cat":"other-agents","safety":"safe","impact":"unknown","purpose":"scraping","robotsTxt":{"status":"ALLOWED","rule":"Full content access; 2 path rules (/assets/, /quote)","source":"wildcard"},"effectiveStatus":"ALLOWED","intel":{"botType":"other","respectsRobots":"unknown"},"robotsApplies":true},"archive.org_bot":{"label":"archive.org_bot","platform":"Internet Archive","tier":6,"emoji":"📡","cat":"other-agents","safety":"caution","impact":"unknown","purpose":"scraping","robotsTxt":{"status":"ALLOWED","rule":"Full content access; 2 path rules (/assets/, /quote)","source":"wildcard"},"effectiveStatus":"ALLOWED","intel":{"botType":"other","respectsRobots":"unknown"},"robotsApplies":true},"Google-GeminiNotebook":{"label":"Gemini Notebook","platform":"Google","tier":2,"emoji":"🔴","cat":"google-bots","safety":"safe","impact":"low","purpose":"user-request","robotsTxt":{"status":"ALLOWED","rule":"Full content access; 2 path rules (/assets/, /quote)","source":"wildcard"},"effectiveStatus":"ALLOWED","intel":{"botType":"user-triggered","respectsRobots":"no","docsUrl":"https://developers.google.com/crawling/docs/crawlers-fetchers/google-user-triggered-fetchers","caveat":"Replaces Google-NotebookLM (retired August 2026). Fetches URLs a Gemini Notebook user added as a source. Ignores robots.txt (user-triggered fetcher)."},"robotsApplies":false},"Google-CWS":{"label":"Chrome Web Store Fetcher","platform":"Google","tier":4,"emoji":"🔴","cat":"google-bots","safety":"safe","impact":"unknown","purpose":"monitoring","robotsTxt":{"status":"ALLOWED","rule":"Full content access; 2 path rules (/assets/, /quote)","source":"wildcard"},"effectiveStatus":"ALLOWED","intel":{"botType":"other","respectsRobots":"no","docsUrl":"https://developers.google.com/crawling/docs/crawlers-fetchers/google-user-triggered-fetchers","caveat":"Chrome Web Store fetcher for URLs listed in extension metadata. Only reaches sites named by an extension developer."},"robotsApplies":false},"GoogleMessages":{"label":"Google Messages Preview","platform":"Google","tier":5,"emoji":"🔴","cat":"google-bots","safety":"safe","impact":"unknown","purpose":"social","robotsTxt":{"status":"ALLOWED","rule":"Full content access; 2 path rules (/assets/, /quote)","source":"wildcard"},"effectiveStatus":"ALLOWED","intel":{"botType":"other","respectsRobots":"no","docsUrl":"https://developers.google.com/crawling/docs/crawlers-fetchers/google-user-triggered-fetchers","caveat":"Link-preview fetcher for URLs sent in Google Messages chats. Same role as facebookexternalhit."},"robotsApplies":false},"Google-Pinpoint":{"label":"Google Pinpoint","platform":"Google","tier":3,"emoji":"🔴","cat":"google-bots","safety":"safe","impact":"unknown","purpose":"user-request","robotsTxt":{"status":"ALLOWED","rule":"Full content access; 2 path rules (/assets/, /quote)","source":"wildcard"},"effectiveStatus":"ALLOWED","intel":{"botType":"user-triggered","respectsRobots":"no","docsUrl":"https://developers.google.com/crawling/docs/crawlers-fetchers/google-user-triggered-fetchers","caveat":"Fetches URLs a Pinpoint (journalism research tool) user added to a collection."},"robotsApplies":false},"Meta-ExternalAds":{"label":"Meta Ads Crawler","platform":"Meta","tier":4,"emoji":"🔵","cat":"social-bots","safety":"safe","impact":"unknown","purpose":"monitoring","robotsTxt":{"status":"ALLOWED","rule":"Full content access; 2 path rules (/assets/, /quote)","source":"wildcard"},"effectiveStatus":"ALLOWED","intel":{"botType":"other","respectsRobots":"yes","docsUrl":"https://developers.facebook.com/docs/sharing/webmasters/web-crawlers/","caveat":"Crawls for advertising and business products. Documented by Meta alongside meta-externalagent and meta-externalfetcher."},"robotsApplies":true},"Kimi-SearchBot":{"label":"Kimi Search","platform":"Moonshot AI","tier":2,"emoji":"🟣","cat":"ai-bots","safety":"safe","impact":"low","purpose":"indexing","robotsTxt":{"status":"ALLOWED","rule":"Full content access; 2 path rules (/assets/, /quote)","source":"wildcard"},"effectiveStatus":"ALLOWED","intel":{"docsUrl":"https://www.kimi.ai/policies/kimi-crawlers","botType":"retrieval","respectsRobots":"unknown","caveat":"Moonshot AI search index crawler. Sibling of Kimi-User."},"robotsApplies":true},"KimiBot":{"label":"Kimi Crawler","platform":"Moonshot AI","tier":2,"emoji":"🟣","cat":"ai-bots","safety":"caution","impact":"low","purpose":"training","robotsTxt":{"status":"ALLOWED","rule":"Full content access; 2 path rules (/assets/, /quote)","source":"wildcard"},"effectiveStatus":"ALLOWED","intel":{"docsUrl":"https://www.kimi.ai/policies/kimi-crawlers","botType":"training","respectsRobots":"unknown","caveat":"Moonshot AI data crawler."},"robotsApplies":true},"MistralAI-Training":{"label":"Mistral Training","platform":"Mistral AI","tier":2,"emoji":"🟠","cat":"ai-bots","safety":"safe","impact":"low","purpose":"training","robotsTxt":{"status":"ALLOWED","rule":"Full content access; 2 path rules (/assets/, /quote)","source":"wildcard"},"effectiveStatus":"ALLOWED","intel":{"botType":"training","respectsRobots":"yes","docsUrl":"https://docs.mistral.ai/robots/","caveat":"Mistral's training crawler, separately controllable from MistralAI-User and MistralAI-Index."},"robotsApplies":true},"Diffbot-User":{"label":"Diffbot Fetcher","platform":"Diffbot","tier":3,"emoji":"🤖","cat":"ai-bots","safety":"safe","impact":"low","purpose":"user-request","robotsTxt":{"status":"ALLOWED","rule":"Full content access; 2 path rules (/assets/, /quote)","source":"wildcard"},"effectiveStatus":"ALLOWED","intel":{"docsUrl":"https://docs.diffbot.com/docs/does-crawl-respect-robotstxt","botType":"user-triggered","respectsRobots":"unknown","caveat":"On-demand fetcher for Diffbot API calls. Sibling of the Diffbot crawler."},"robotsApplies":true},"ShapBot":{"label":"Parallel Web Crawler","platform":"Parallel","tier":3,"emoji":"🤖","cat":"ai-bots","safety":"caution","impact":"low","purpose":"indexing","robotsTxt":{"status":"ALLOWED","rule":"Full content access; 2 path rules (/assets/, /quote)","source":"wildcard"},"effectiveStatus":"ALLOWED","intel":{"docsUrl":"https://docs.parallel.ai/resources/crawler","botType":"retrieval","respectsRobots":"unknown","caveat":"Parallel Web Systems index crawler serving AI agents."},"robotsApplies":true},"Brightbot":{"label":"Bright Data Crawler","platform":"Bright Data","tier":6,"emoji":"🕷","cat":"scrapers","safety":"aggressive","impact":"low","purpose":"training","robotsTxt":{"status":"ALLOWED","rule":"Full content access; 2 path rules (/assets/, /quote)","source":"wildcard"},"effectiveStatus":"ALLOWED","intel":{"docsUrl":"https://brightdata.com/brightbot","botType":"training","respectsRobots":"inconsistent","caveat":"Bright Data commercial scraping network. Also appears as \"Brightbot 1.0\"."},"robotsApplies":true},"ExaSearchBot":{"label":"Exa Search","platform":"Exa","tier":2,"emoji":"🔎","cat":"ai-bots","safety":"safe","impact":"low","purpose":"indexing","robotsTxt":{"status":"ALLOWED","rule":"Full content access; 2 path rules (/assets/, /quote)","source":"wildcard"},"effectiveStatus":"ALLOWED","intel":{"botType":"retrieval","respectsRobots":"unknown"},"robotsApplies":true},"KlaviyoAIBot":{"label":"Klaviyo AI","platform":"Klaviyo","tier":2,"emoji":"🟢","cat":"ai-bots","safety":"safe","impact":"low","purpose":"training","robotsTxt":{"status":"ALLOWED","rule":"Full content access; 2 path rules (/assets/, /quote)","source":"wildcard"},"effectiveStatus":"ALLOWED","intel":{"botType":"training","respectsRobots":"unknown"},"robotsApplies":true},"SBIntuitionsBot":{"label":"SB Intuitions Crawler","platform":"SB Intuitions","tier":2,"emoji":"🤖","cat":"ai-bots","safety":"safe","impact":"low","purpose":"training","robotsTxt":{"status":"ALLOWED","rule":"Full content access; 2 path rules (/assets/, /quote)","source":"wildcard"},"effectiveStatus":"ALLOWED","intel":{"botType":"training","respectsRobots":"unknown"},"robotsApplies":true},"SBIntuitions-SearchBot":{"label":"Sarashina Search","platform":"SB Intuitions","tier":2,"emoji":"🤖","cat":"ai-bots","safety":"safe","impact":"low","purpose":"user-request","robotsTxt":{"status":"ALLOWED","rule":"Full content access; 2 path rules (/assets/, /quote)","source":"wildcard"},"effectiveStatus":"ALLOWED","intel":{"botType":"user-triggered","respectsRobots":"unknown"},"robotsApplies":true},"Slurp":{"label":"Yahoo Slurp","platform":"Yahoo","tier":3,"emoji":"🔍","cat":"search-engines","safety":"safe","impact":"critical","purpose":"indexing","robotsTxt":{"status":"ALLOWED","rule":"Full content access; 2 path rules (/assets/, /quote)","source":"wildcard"},"effectiveStatus":"ALLOWED","intel":{"botType":"search","respectsRobots":"unknown"},"robotsApplies":true},"Bytespider":{"label":"ByteDance / Toutiao Search","platform":"ByteDance","tier":2,"emoji":"⚫","cat":"scrapers","safety":"aggressive","impact":"low","purpose":"both","robotsTxt":{"status":"ALLOWED","rule":"Full content access; 2 path rules (/assets/, /quote)","source":"wildcard"},"effectiveStatus":"ALLOWED","intel":{"botType":"training","respectsRobots":"unknown"},"robotsApplies":true},"Arquivo-web-crawler":{"label":"Arquivo.pt Web Archive","platform":"Arquivo.pt","tier":6,"emoji":"📡","cat":"other-agents","safety":"safe","impact":"low","purpose":"scraping","robotsTxt":{"status":"ALLOWED","rule":"Full content access; 2 path rules (/assets/, /quote)","source":"wildcard"},"effectiveStatus":"ALLOWED","intel":{"botType":"other","respectsRobots":"unknown"},"robotsApplies":true},"Cotoyogi":{"label":"Cotoyogi (ROIS-DS)","platform":"ROIS-DS","tier":6,"emoji":"🕷","cat":"scrapers","safety":"safe","impact":"low","purpose":"training","robotsTxt":{"status":"ALLOWED","rule":"Full content access; 2 path rules (/assets/, /quote)","source":"wildcard"},"effectiveStatus":"ALLOWED","intel":{"botType":"training","respectsRobots":"unknown"},"robotsApplies":true},"Webzio-Extended":{"label":"Webz.io Extended","platform":"Webz.io","tier":6,"emoji":"🕷","cat":"scrapers","safety":"caution","impact":"low","purpose":"training","robotsTxt":{"status":"ALLOWED","rule":"Full content access; 2 path rules (/assets/, /quote)","source":"wildcard"},"effectiveStatus":"ALLOWED","intel":{"botType":"training","respectsRobots":"unknown"},"robotsApplies":true},"Kagibot":{"label":"Kagi Search","platform":"Kagi","tier":3,"emoji":"🔍","cat":"search-engines","safety":"safe","impact":"critical","purpose":"indexing","robotsTxt":{"status":"ALLOWED","rule":"Full content access; 2 path rules (/assets/, /quote)","source":"wildcard"},"effectiveStatus":"ALLOWED","intel":{"botType":"search","respectsRobots":"unknown"},"robotsApplies":true},"LinerBot":{"label":"LINER AI","platform":"LINER","tier":2,"emoji":"🤖","cat":"ai-bots","safety":"safe","impact":"low","purpose":"both","robotsTxt":{"status":"ALLOWED","rule":"Full content access; 2 path rules (/assets/, /quote)","source":"wildcard"},"effectiveStatus":"ALLOWED","intel":{"botType":"training","respectsRobots":"unknown"},"robotsApplies":true},"APIs-Google":{"label":"APIs-Google","platform":"Google","tier":3,"emoji":"🔴","cat":"google-bots","safety":"safe","impact":"unknown","purpose":"user-request","robotsTxt":{"status":"ALLOWED","rule":"Full content access; 2 path rules (/assets/, /quote)","source":"wildcard"},"effectiveStatus":"ALLOWED","intel":{"botType":"user-triggered","respectsRobots":"unknown"},"robotsApplies":true},"AdsBot-Google":{"label":"Google Ads Quality (Desktop)","platform":"Google","tier":3,"emoji":"🔴","cat":"google-bots","safety":"safe","impact":"unknown","purpose":"monitoring","robotsTxt":{"status":"ALLOWED","rule":"Full content access; 2 path rules (/assets/, /quote)","source":"wildcard"},"effectiveStatus":"ALLOWED","intel":{"botType":"other","respectsRobots":"unknown"},"robotsApplies":true},"AdsBot-Google-Mobile":{"label":"Google Ads Quality (Mobile)","platform":"Google","tier":3,"emoji":"🔴","cat":"google-bots","safety":"safe","impact":"unknown","purpose":"monitoring","robotsTxt":{"status":"ALLOWED","rule":"Full content access; 2 path rules (/assets/, /quote)","source":"wildcard"},"effectiveStatus":"ALLOWED","intel":{"botType":"other","respectsRobots":"unknown"},"robotsApplies":true},"Mediapartners-Google":{"label":"Google AdSense","platform":"Google","tier":3,"emoji":"🔴","cat":"google-bots","safety":"safe","impact":"unknown","purpose":"monitoring","robotsTxt":{"status":"ALLOWED","rule":"Full content access; 2 path rules (/assets/, /quote)","source":"wildcard"},"effectiveStatus":"ALLOWED","intel":{"botType":"other","respectsRobots":"yes","docsUrl":"https://developers.google.com/crawling/docs/crawlers-fetchers/google-special-case-crawlers","caveat":"Google updated this entry on 2026-09-17: rules for Mediapartners-Google affect several ad products, including Google AdSense and Google Ad Manager. It ignores the * group, so it needs its own User-agent line."},"robotsApplies":true},"Google-Safety":{"label":"Google Safety","platform":"Google","tier":3,"emoji":"🔴","cat":"google-bots","safety":"safe","impact":"unknown","purpose":"monitoring","robotsTxt":{"status":"ALLOWED","rule":"Full content access; 2 path rules (/assets/, /quote)","source":"wildcard"},"effectiveStatus":"ALLOWED","intel":{"botType":"other","respectsRobots":"unknown"},"robotsApplies":true},"AdIdxBot":{"label":"Bing Ads Crawler","platform":"Microsoft","tier":3,"emoji":"🔵","cat":"search-engines","safety":"safe","impact":"unknown","purpose":"monitoring","robotsTxt":{"status":"ALLOWED","rule":"Full content access; 2 path rules (/assets/, /quote)","source":"wildcard"},"effectiveStatus":"ALLOWED","intel":{"botType":"other","respectsRobots":"unknown"},"robotsApplies":true},"BingPreview":{"label":"Bing Page Snapshot","platform":"Microsoft","tier":3,"emoji":"🔵","cat":"search-engines","safety":"safe","impact":"critical","purpose":"indexing","robotsTxt":{"status":"ALLOWED","rule":"Full content access; 2 path rules (/assets/, /quote)","source":"wildcard"},"effectiveStatus":"ALLOWED","intel":{"botType":"search","respectsRobots":"unknown"},"robotsApplies":true},"BingVideoPreview":{"label":"Bing Video Preview","platform":"Microsoft","tier":3,"emoji":"🔵","cat":"search-engines","safety":"safe","impact":"unknown","purpose":"indexing","robotsTxt":{"status":"ALLOWED","rule":"Full content access; 2 path rules (/assets/, /quote)","source":"wildcard"},"effectiveStatus":"ALLOWED","intel":{"botType":"search","respectsRobots":"unknown"},"robotsApplies":true},"MicrosoftPreview":{"label":"Microsoft Preview","platform":"Microsoft","tier":3,"emoji":"🔵","cat":"search-engines","safety":"safe","impact":"unknown","purpose":"social","robotsTxt":{"status":"ALLOWED","rule":"Full content access; 2 path rules (/assets/, /quote)","source":"wildcard"},"effectiveStatus":"ALLOWED","intel":{"botType":"other","respectsRobots":"unknown"},"robotsApplies":true},"Baiduspider-image":{"label":"Baidu Image Search","platform":"Baidu","tier":3,"emoji":"🔍","cat":"search-engines","safety":"safe","impact":"critical","purpose":"indexing","robotsTxt":{"status":"ALLOWED","rule":"Full content access; 2 path rules (/assets/, /quote)","source":"wildcard"},"effectiveStatus":"ALLOWED","intel":{"botType":"search","respectsRobots":"unknown"},"robotsApplies":true},"Baiduspider-video":{"label":"Baidu Video Search","platform":"Baidu","tier":3,"emoji":"🔍","cat":"search-engines","safety":"safe","impact":"critical","purpose":"indexing","robotsTxt":{"status":"ALLOWED","rule":"Full content access; 2 path rules (/assets/, /quote)","source":"wildcard"},"effectiveStatus":"ALLOWED","intel":{"botType":"search","respectsRobots":"unknown"},"robotsApplies":true},"Baiduspider-news":{"label":"Baidu News Search","platform":"Baidu","tier":3,"emoji":"🔍","cat":"search-engines","safety":"safe","impact":"critical","purpose":"indexing","robotsTxt":{"status":"ALLOWED","rule":"Full content access; 2 path rules (/assets/, /quote)","source":"wildcard"},"effectiveStatus":"ALLOWED","intel":{"botType":"search","respectsRobots":"unknown"},"robotsApplies":true},"Baiduspider-favo":{"label":"Baidu Bookmark","platform":"Baidu","tier":3,"emoji":"🔍","cat":"search-engines","safety":"safe","impact":"unknown","purpose":"indexing","robotsTxt":{"status":"ALLOWED","rule":"Full content access; 2 path rules (/assets/, /quote)","source":"wildcard"},"effectiveStatus":"ALLOWED","intel":{"botType":"search","respectsRobots":"unknown"},"robotsApplies":true},"Baiduspider-cpro":{"label":"Baidu Union","platform":"Baidu","tier":3,"emoji":"🔍","cat":"search-engines","safety":"safe","impact":"unknown","purpose":"monitoring","robotsTxt":{"status":"ALLOWED","rule":"Full content access; 2 path rules (/assets/, /quote)","source":"wildcard"},"effectiveStatus":"ALLOWED","intel":{"botType":"other","respectsRobots":"unknown"},"robotsApplies":true},"Baiduspider-ads":{"label":"Baidu Business Search","platform":"Baidu","tier":3,"emoji":"🔍","cat":"search-engines","safety":"safe","impact":"unknown","purpose":"monitoring","robotsTxt":{"status":"ALLOWED","rule":"Full content access; 2 path rules (/assets/, /quote)","source":"wildcard"},"effectiveStatus":"ALLOWED","intel":{"botType":"other","respectsRobots":"unknown"},"robotsApplies":true},"YandexImages":{"label":"Yandex Images","platform":"Yandex","tier":3,"emoji":"🔍","cat":"search-engines","safety":"safe","impact":"critical","purpose":"indexing","robotsTxt":{"status":"ALLOWED","rule":"Full content access; 2 path rules (/assets/, /quote)","source":"wildcard"},"effectiveStatus":"ALLOWED","intel":{"botType":"search","respectsRobots":"unknown"},"robotsApplies":true},"YandexVideo":{"label":"Yandex Video","platform":"Yandex","tier":3,"emoji":"🔍","cat":"search-engines","safety":"safe","impact":"critical","purpose":"indexing","robotsTxt":{"status":"ALLOWED","rule":"Full content access; 2 path rules (/assets/, /quote)","source":"wildcard"},"effectiveStatus":"ALLOWED","intel":{"botType":"search","respectsRobots":"unknown"},"robotsApplies":true},"YandexVideoParser":{"label":"Yandex Video Parser","platform":"Yandex","tier":3,"emoji":"🔍","cat":"search-engines","safety":"safe","impact":"unknown","purpose":"indexing","robotsTxt":{"status":"ALLOWED","rule":"Full content access; 2 path rules (/assets/, /quote)","source":"wildcard"},"effectiveStatus":"ALLOWED","intel":{"botType":"search","respectsRobots":"unknown"},"robotsApplies":true},"YandexMedia":{"label":"Yandex Media","platform":"Yandex","tier":3,"emoji":"🔍","cat":"search-engines","safety":"safe","impact":"unknown","purpose":"indexing","robotsTxt":{"status":"ALLOWED","rule":"Full content access; 2 path rules (/assets/, /quote)","source":"wildcard"},"effectiveStatus":"ALLOWED","intel":{"botType":"search","respectsRobots":"unknown"},"robotsApplies":true},"YandexBlogs":{"label":"Yandex Blogs","platform":"Yandex","tier":3,"emoji":"🔍","cat":"search-engines","safety":"safe","impact":"unknown","purpose":"indexing","robotsTxt":{"status":"ALLOWED","rule":"Full content access; 2 path rules (/assets/, /quote)","source":"wildcard"},"effectiveStatus":"ALLOWED","intel":{"botType":"search","respectsRobots":"unknown"},"robotsApplies":true},"YandexMobileBot":{"label":"Yandex Mobile","platform":"Yandex","tier":3,"emoji":"🔍","cat":"search-engines","safety":"safe","impact":"critical","purpose":"indexing","robotsTxt":{"status":"ALLOWED","rule":"Full content access; 2 path rules (/assets/, /quote)","source":"wildcard"},"effectiveStatus":"ALLOWED","intel":{"botType":"search","respectsRobots":"unknown"},"robotsApplies":true},"YandexAccessibilityBot":{"label":"Yandex Accessibility","platform":"Yandex","tier":3,"emoji":"🔍","cat":"search-engines","safety":"safe","impact":"unknown","purpose":"indexing","robotsTxt":{"status":"ALLOWED","rule":"Full content access; 2 path rules (/assets/, /quote)","source":"wildcard"},"effectiveStatus":"ALLOWED","intel":{"botType":"search","respectsRobots":"unknown"},"robotsApplies":true},"YandexRenderResourcesBot":{"label":"Yandex Render Resources","platform":"Yandex","tier":3,"emoji":"🔍","cat":"search-engines","safety":"safe","impact":"critical","purpose":"indexing","robotsTxt":{"status":"ALLOWED","rule":"Full content access; 2 path rules (/assets/, /quote)","source":"wildcard"},"effectiveStatus":"ALLOWED","intel":{"botType":"search","respectsRobots":"unknown"},"robotsApplies":true},"YandexPagechecker":{"label":"Yandex Page Checker","platform":"Yandex","tier":3,"emoji":"🔍","cat":"search-engines","safety":"safe","impact":"unknown","purpose":"monitoring","robotsTxt":{"status":"ALLOWED","rule":"Full content access; 2 path rules (/assets/, /quote)","source":"wildcard"},"effectiveStatus":"ALLOWED","intel":{"botType":"other","respectsRobots":"unknown"},"robotsApplies":true},"YandexCheckBot":{"label":"Yandex Check","platform":"Yandex","tier":3,"emoji":"🔍","cat":"search-engines","safety":"safe","impact":"unknown","purpose":"monitoring","robotsTxt":{"status":"ALLOWED","rule":"Full content access; 2 path rules (/assets/, /quote)","source":"wildcard"},"effectiveStatus":"ALLOWED","intel":{"botType":"other","respectsRobots":"unknown"},"robotsApplies":true},"YandexComBot":{"label":"Yandex.com","platform":"Yandex","tier":3,"emoji":"🔍","cat":"search-engines","safety":"safe","impact":"critical","purpose":"indexing","robotsTxt":{"status":"ALLOWED","rule":"Full content access; 2 path rules (/assets/, /quote)","source":"wildcard"},"effectiveStatus":"ALLOWED","intel":{"botType":"search","respectsRobots":"unknown"},"robotsApplies":true},"YandexFavicons":{"label":"Yandex Favicons","platform":"Yandex","tier":3,"emoji":"🔍","cat":"search-engines","safety":"safe","impact":"unknown","purpose":"indexing","robotsTxt":{"status":"ALLOWED","rule":"Full content access; 2 path rules (/assets/, /quote)","source":"wildcard"},"effectiveStatus":"ALLOWED","intel":{"botType":"search","respectsRobots":"unknown"},"robotsApplies":true},"YandexFeed":{"label":"Yandex Feed","platform":"Yandex","tier":3,"emoji":"🔍","cat":"search-engines","safety":"safe","impact":"unknown","purpose":"indexing","robotsTxt":{"status":"ALLOWED","rule":"Full content access; 2 path rules (/assets/, /quote)","source":"wildcard"},"effectiveStatus":"ALLOWED","intel":{"botType":"search","respectsRobots":"unknown"},"robotsApplies":true},"YandexImageResizer":{"label":"Yandex Image Resizer","platform":"Yandex","tier":3,"emoji":"🔍","cat":"search-engines","safety":"safe","impact":"unknown","purpose":"user-request","robotsTxt":{"status":"ALLOWED","rule":"Full content access; 2 path rules (/assets/, /quote)","source":"wildcard"},"effectiveStatus":"ALLOWED","intel":{"botType":"user-triggered","respectsRobots":"unknown"},"robotsApplies":true},"YandexMobileScreenShotBot":{"label":"Yandex Mobile Screenshot","platform":"Yandex","tier":3,"emoji":"🔍","cat":"search-engines","safety":"safe","impact":"unknown","purpose":"monitoring","robotsTxt":{"status":"ALLOWED","rule":"Full content access; 2 path rules (/assets/, /quote)","source":"wildcard"},"effectiveStatus":"ALLOWED","intel":{"botType":"other","respectsRobots":"unknown"},"robotsApplies":true},"YandexScreenshotBot":{"label":"Yandex Screenshot","platform":"Yandex","tier":3,"emoji":"🔍","cat":"search-engines","safety":"safe","impact":"unknown","purpose":"monitoring","robotsTxt":{"status":"ALLOWED","rule":"Full content access; 2 path rules (/assets/, /quote)","source":"wildcard"},"effectiveStatus":"ALLOWED","intel":{"botType":"other","respectsRobots":"unknown"},"robotsApplies":true},"YandexSitelinks":{"label":"Yandex Sitelinks","platform":"Yandex","tier":3,"emoji":"🔍","cat":"search-engines","safety":"safe","impact":"unknown","purpose":"monitoring","robotsTxt":{"status":"ALLOWED","rule":"Full content access; 2 path rules (/assets/, /quote)","source":"wildcard"},"effectiveStatus":"ALLOWED","intel":{"botType":"other","respectsRobots":"unknown"},"robotsApplies":true},"YandexUserproxy":{"label":"Yandex User Proxy","platform":"Yandex","tier":3,"emoji":"🔍","cat":"search-engines","safety":"safe","impact":"unknown","purpose":"user-request","robotsTxt":{"status":"ALLOWED","rule":"Full content access; 2 path rules (/assets/, /quote)","source":"wildcard"},"effectiveStatus":"ALLOWED","intel":{"botType":"user-triggered","respectsRobots":"unknown"},"robotsApplies":true},"YandexWebmaster":{"label":"Yandex Webmaster","platform":"Yandex","tier":3,"emoji":"🔍","cat":"search-engines","safety":"safe","impact":"unknown","purpose":"monitoring","robotsTxt":{"status":"ALLOWED","rule":"Full content access; 2 path rules (/assets/, /quote)","source":"wildcard"},"effectiveStatus":"ALLOWED","intel":{"botType":"other","respectsRobots":"unknown"},"robotsApplies":true},"YandexMetrika":{"label":"Yandex Metrica","platform":"Yandex","tier":4,"emoji":"🔍","cat":"seo-tools","safety":"safe","impact":"unknown","purpose":"monitoring","robotsTxt":{"status":"ALLOWED","rule":"Full content access; 2 path rules (/assets/, /quote)","source":"wildcard"},"effectiveStatus":"ALLOWED","intel":{"botType":"other","respectsRobots":"unknown"},"robotsApplies":true},"YandexTracker":{"label":"Yandex Tracker","platform":"Yandex","tier":4,"emoji":"🔍","cat":"seo-tools","safety":"safe","impact":"unknown","purpose":"user-request","robotsTxt":{"status":"ALLOWED","rule":"Full content access; 2 path rules (/assets/, /quote)","source":"wildcard"},"effectiveStatus":"ALLOWED","intel":{"botType":"user-triggered","respectsRobots":"unknown"},"robotsApplies":true},"YandexMarket":{"label":"Yandex Market","platform":"Yandex","tier":3,"emoji":"🔍","cat":"search-engines","safety":"safe","impact":"unknown","purpose":"indexing","robotsTxt":{"status":"ALLOWED","rule":"Full content access; 2 path rules (/assets/, /quote)","source":"wildcard"},"effectiveStatus":"ALLOWED","intel":{"botType":"search","respectsRobots":"unknown"},"robotsApplies":true},"YandexSearchShop":{"label":"Yandex Search Shop","platform":"Yandex","tier":3,"emoji":"🔍","cat":"search-engines","safety":"safe","impact":"unknown","purpose":"indexing","robotsTxt":{"status":"ALLOWED","rule":"Full content access; 2 path rules (/assets/, /quote)","source":"wildcard"},"effectiveStatus":"ALLOWED","intel":{"botType":"search","respectsRobots":"unknown"},"robotsApplies":true},"YandexVerticals":{"label":"Yandex Verticals","platform":"Yandex","tier":3,"emoji":"🔍","cat":"search-engines","safety":"safe","impact":"unknown","purpose":"indexing","robotsTxt":{"status":"ALLOWED","rule":"Full content access; 2 path rules (/assets/, /quote)","source":"wildcard"},"effectiveStatus":"ALLOWED","intel":{"botType":"search","respectsRobots":"unknown"},"robotsApplies":true},"YandexVertis":{"label":"Yandex Vertis","platform":"Yandex","tier":3,"emoji":"🔍","cat":"search-engines","safety":"safe","impact":"unknown","purpose":"indexing","robotsTxt":{"status":"ALLOWED","rule":"Full content access; 2 path rules (/assets/, /quote)","source":"wildcard"},"effectiveStatus":"ALLOWED","intel":{"botType":"search","respectsRobots":"unknown"},"robotsApplies":true},"YandexSpravBot":{"label":"Yandex Business Directory","platform":"Yandex","tier":3,"emoji":"🔍","cat":"search-engines","safety":"safe","impact":"unknown","purpose":"indexing","robotsTxt":{"status":"ALLOWED","rule":"Full content access; 2 path rules (/assets/, /quote)","source":"wildcard"},"effectiveStatus":"ALLOWED","intel":{"botType":"search","respectsRobots":"unknown"},"robotsApplies":true},"YandexOntoDB":{"label":"Yandex Object Answer","platform":"Yandex","tier":3,"emoji":"🔍","cat":"search-engines","safety":"safe","impact":"unknown","purpose":"indexing","robotsTxt":{"status":"ALLOWED","rule":"Full content access; 2 path rules (/assets/, /quote)","source":"wildcard"},"effectiveStatus":"ALLOWED","intel":{"botType":"search","respectsRobots":"unknown"},"robotsApplies":true},"YandexOntoDBAPI":{"label":"Yandex Object Answer API","platform":"Yandex","tier":3,"emoji":"🔍","cat":"search-engines","safety":"safe","impact":"unknown","purpose":"user-request","robotsTxt":{"status":"ALLOWED","rule":"Full content access; 2 path rules (/assets/, /quote)","source":"wildcard"},"effectiveStatus":"ALLOWED","intel":{"botType":"user-triggered","respectsRobots":"unknown"},"robotsApplies":true},"YandexCalendar":{"label":"Yandex Calendar","platform":"Yandex","tier":3,"emoji":"🔍","cat":"search-engines","safety":"safe","impact":"unknown","purpose":"user-request","robotsTxt":{"status":"ALLOWED","rule":"Full content access; 2 path rules (/assets/, /quote)","source":"wildcard"},"effectiveStatus":"ALLOWED","intel":{"botType":"user-triggered","respectsRobots":"unknown"},"robotsApplies":true},"YandexPartner":{"label":"Yandex Partner","platform":"Yandex","tier":3,"emoji":"🔍","cat":"search-engines","safety":"safe","impact":"unknown","purpose":"monitoring","robotsTxt":{"status":"ALLOWED","rule":"Full content access; 2 path rules (/assets/, /quote)","source":"wildcard"},"effectiveStatus":"ALLOWED","intel":{"botType":"other","respectsRobots":"unknown"},"robotsApplies":true},"YandexRCA":{"label":"Yandex RCA","platform":"Yandex","tier":3,"emoji":"🔍","cat":"search-engines","safety":"safe","impact":"unknown","purpose":"indexing","robotsTxt":{"status":"ALLOWED","rule":"Full content access; 2 path rules (/assets/, /quote)","source":"wildcard"},"effectiveStatus":"ALLOWED","intel":{"botType":"search","respectsRobots":"unknown"},"robotsApplies":true},"YandexAdNet":{"label":"Yandex Advertising Network","platform":"Yandex","tier":3,"emoji":"🔍","cat":"search-engines","safety":"safe","impact":"unknown","purpose":"monitoring","robotsTxt":{"status":"ALLOWED","rule":"Full content access; 2 path rules (/assets/, /quote)","source":"wildcard"},"effectiveStatus":"ALLOWED","intel":{"botType":"other","respectsRobots":"unknown"},"robotsApplies":true},"YandexDirect":{"label":"Yandex Direct","platform":"Yandex","tier":3,"emoji":"🔍","cat":"search-engines","safety":"safe","impact":"unknown","purpose":"monitoring","robotsTxt":{"status":"ALLOWED","rule":"Full content access; 2 path rules (/assets/, /quote)","source":"wildcard"},"effectiveStatus":"ALLOWED","intel":{"botType":"other","respectsRobots":"unknown"},"robotsApplies":true},"YandexDirectDyn":{"label":"Yandex Direct Dynamic","platform":"Yandex","tier":3,"emoji":"🔍","cat":"search-engines","safety":"safe","impact":"unknown","purpose":"monitoring","robotsTxt":{"status":"ALLOWED","rule":"Full content access; 2 path rules (/assets/, /quote)","source":"wildcard"},"effectiveStatus":"ALLOWED","intel":{"botType":"other","respectsRobots":"unknown"},"robotsApplies":true},"YaDirectFetcher":{"label":"Yandex Direct Fetcher","platform":"Yandex","tier":3,"emoji":"🔍","cat":"search-engines","safety":"safe","impact":"unknown","purpose":"monitoring","robotsTxt":{"status":"ALLOWED","rule":"Full content access; 2 path rules (/assets/, /quote)","source":"wildcard"},"effectiveStatus":"ALLOWED","intel":{"botType":"other","respectsRobots":"unknown"},"robotsApplies":true},"AhrefsSiteAudit":{"label":"Ahrefs Site Audit","platform":"Ahrefs","tier":4,"emoji":"🔧","cat":"seo-tools","safety":"safe","impact":"unknown","purpose":"monitoring","robotsTxt":{"status":"ALLOWED","rule":"Full content access; 2 path rules (/assets/, /quote)","source":"wildcard"},"effectiveStatus":"ALLOWED","intel":{"botType":"other","respectsRobots":"unknown"},"robotsApplies":true},"Rogerbot":{"label":"Moz Campaign Crawler","platform":"Moz","tier":4,"emoji":"🔧","cat":"seo-tools","safety":"safe","impact":"unknown","purpose":"monitoring","robotsTxt":{"status":"ALLOWED","rule":"Full content access; 2 path rules (/assets/, /quote)","source":"wildcard"},"effectiveStatus":"ALLOWED","intel":{"botType":"other","respectsRobots":"unknown"},"robotsApplies":true},"AwarioBot":{"label":"AwarioBot","platform":"Awario","tier":4,"emoji":"🔧","cat":"seo-tools","safety":"safe","impact":"unknown","purpose":"monitoring","robotsTxt":{"status":"ALLOWED","rule":"Full content access; 2 path rules (/assets/, /quote)","source":"wildcard"},"effectiveStatus":"ALLOWED","intel":{"botType":"other","respectsRobots":"unknown"},"robotsApplies":true},"Slackbot-LinkExpanding":{"label":"Slack Link Expander","platform":"Slack","tier":5,"emoji":"💬","cat":"social-bots","safety":"safe","impact":"unknown","purpose":"social","robotsTxt":{"status":"ALLOWED","rule":"Full content access; 2 path rules (/assets/, /quote)","source":"wildcard"},"effectiveStatus":"ALLOWED","intel":{"botType":"other","respectsRobots":"unknown"},"robotsApplies":true},"AddSearchBot":{"label":"AddSearch AI","platform":"AddSearch","tier":2,"emoji":"🟡","cat":"ai-bots","safety":"safe","impact":"low","purpose":"indexing","robotsTxt":{"status":"ALLOWED","rule":"Full content access; 2 path rules (/assets/, /quote)","source":"wildcard"},"effectiveStatus":"ALLOWED","intel":{"botType":"retrieval","respectsRobots":"unknown"},"robotsApplies":true},"AI2Bot-DeepResearchEval":{"label":"Ai2 Deep Research","platform":"Allen Institute for AI","tier":2,"emoji":"🤖","cat":"ai-bots","safety":"safe","impact":"low","purpose":"user-request","robotsTxt":{"status":"ALLOWED","rule":"Full content access; 2 path rules (/assets/, /quote)","source":"wildcard"},"effectiveStatus":"ALLOWED","intel":{"botType":"user-triggered","respectsRobots":"unknown"},"robotsApplies":true},"aiHitBot":{"label":"aiHit Data","platform":"aiHit","tier":2,"emoji":"🤖","cat":"ai-bots","safety":"caution","impact":"low","purpose":"training","robotsTxt":{"status":"ALLOWED","rule":"Full content access; 2 path rules (/assets/, /quote)","source":"wildcard"},"effectiveStatus":"ALLOWED","intel":{"botType":"training","respectsRobots":"unknown"},"robotsApplies":true},"amazon-QBusiness":{"label":"Amazon Q Business","platform":"Amazon","tier":3,"emoji":"🟠","cat":"ai-bots","safety":"safe","impact":"low","purpose":"indexing","robotsTxt":{"status":"ALLOWED","rule":"Full content access; 2 path rules (/assets/, /quote)","source":"wildcard"},"effectiveStatus":"ALLOWED","intel":{"botType":"retrieval","respectsRobots":"unknown"},"robotsApplies":true},"AmazonBuyForMe":{"label":"Amazon Buy For Me","platform":"Amazon","tier":2,"emoji":"🟠","cat":"ai-bots","safety":"safe","impact":"low","purpose":"user-request","robotsTxt":{"status":"ALLOWED","rule":"Full content access; 2 path rules (/assets/, /quote)","source":"wildcard"},"effectiveStatus":"ALLOWED","intel":{"botType":"user-triggered","respectsRobots":"unknown"},"robotsApplies":true},"Andibot":{"label":"Andi Search","platform":"Andi","tier":2,"emoji":"🟡","cat":"ai-bots","safety":"safe","impact":"low","purpose":"both","robotsTxt":{"status":"ALLOWED","rule":"Full content access; 2 path rules (/assets/, /quote)","source":"wildcard"},"effectiveStatus":"ALLOWED","intel":{"botType":"training","respectsRobots":"unknown"},"robotsApplies":true},"anthropic-ai":{"label":"Anthropic AI (Deprecated)","platform":"Anthropic","tier":2,"emoji":"🟠","cat":"ai-bots","safety":"safe","impact":"low","purpose":"training","robotsTxt":{"status":"ALLOWED","rule":"Full content access; 2 path rules (/assets/, /quote)","source":"wildcard"},"effectiveStatus":"ALLOWED","intel":{"botType":"training","respectsRobots":"unknown"},"robotsApplies":true},"ApifyBot":{"label":"Apify Crawler","platform":"Apify","tier":6,"emoji":"🕷","cat":"scrapers","safety":"caution","impact":"low","purpose":"scraping","robotsTxt":{"status":"ALLOWED","rule":"Full content access; 2 path rules (/assets/, /quote)","source":"wildcard"},"effectiveStatus":"ALLOWED","intel":{"botType":"other","respectsRobots":"unknown"},"robotsApplies":true},"atlassian-bot":{"label":"Atlassian Rovo","platform":"Atlassian","tier":2,"emoji":"🔵","cat":"ai-bots","safety":"safe","impact":"low","purpose":"indexing","robotsTxt":{"status":"ALLOWED","rule":"Full content access; 2 path rules (/assets/, /quote)","source":"wildcard"},"effectiveStatus":"ALLOWED","intel":{"botType":"retrieval","respectsRobots":"unknown"},"robotsApplies":true},"AzureAI-SearchBot":{"label":"Azure AI Search","platform":"Microsoft","tier":2,"emoji":"🔵","cat":"ai-bots","safety":"safe","impact":"low","purpose":"indexing","robotsTxt":{"status":"ALLOWED","rule":"Full content access; 2 path rules (/assets/, /quote)","source":"wildcard"},"effectiveStatus":"ALLOWED","intel":{"botType":"retrieval","respectsRobots":"unknown"},"robotsApplies":true},"BLEXBot":{"label":"BLEXBot Scraper","platform":"WebMeUp","tier":6,"emoji":"🕷","cat":"scrapers","safety":"safe","impact":"low","purpose":"scraping","robotsTxt":{"status":"ALLOWED","rule":"Full content access; 2 path rules (/assets/, /quote)","source":"wildcard"},"effectiveStatus":"ALLOWED","intel":{"botType":"other","respectsRobots":"unknown"},"robotsApplies":true},"Bravebot":{"label":"Brave Leo AI","platform":"Brave","tier":2,"emoji":"🦁","cat":"ai-bots","safety":"safe","impact":"low","purpose":"user-request","robotsTxt":{"status":"ALLOWED","rule":"Full content access; 2 path rules (/assets/, /quote)","source":"wildcard"},"effectiveStatus":"ALLOWED","intel":{"botType":"user-triggered","respectsRobots":"unknown"},"robotsApplies":true},"ChatGLM-Spider":{"label":"ChatGLM Spider","platform":"Zhipu AI","tier":2,"emoji":"🟣","cat":"ai-bots","safety":"caution","impact":"low","purpose":"training","robotsTxt":{"status":"ALLOWED","rule":"Full content access; 2 path rules (/assets/, /quote)","source":"wildcard"},"effectiveStatus":"ALLOWED","intel":{"botType":"training","respectsRobots":"unknown"},"robotsApplies":true},"Chrome-Lighthouse":{"label":"Lighthouse Audit","platform":"Google","tier":4,"emoji":"🔴","cat":"seo-tools","safety":"safe","impact":"unknown","purpose":"monitoring","robotsTxt":{"status":"ALLOWED","rule":"Full content access; 2 path rules (/assets/, /quote)","source":"wildcard"},"effectiveStatus":"ALLOWED","intel":{"botType":"other","respectsRobots":"unknown"},"robotsApplies":true},"Claude-Code":{"label":"Claude Code Agent","platform":"Anthropic","tier":2,"emoji":"🟠","cat":"ai-bots","safety":"safe","impact":"low","purpose":"user-request","robotsTxt":{"status":"ALLOWED","rule":"Full content access; 2 path rules (/assets/, /quote)","source":"wildcard"},"effectiveStatus":"ALLOWED","intel":{"botType":"user-triggered","respectsRobots":"unknown"},"robotsApplies":true},"Claude-Web":{"label":"Claude Web (Deprecated)","platform":"Anthropic","tier":2,"emoji":"🟠","cat":"ai-bots","safety":"safe","impact":"low","purpose":"user-request","robotsTxt":{"status":"ALLOWED","rule":"Full content access; 2 path rules (/assets/, /quote)","source":"wildcard"},"effectiveStatus":"ALLOWED","intel":{"botType":"user-triggered","respectsRobots":"unknown"},"robotsApplies":true},"Cloudflare-AutoRAG":{"label":"Cloudflare AutoRAG","platform":"Cloudflare","tier":2,"emoji":"🟠","cat":"ai-bots","safety":"safe","impact":"low","purpose":"indexing","robotsTxt":{"status":"ALLOWED","rule":"Full content access; 2 path rules (/assets/, /quote)","source":"wildcard"},"effectiveStatus":"ALLOWED","intel":{"botType":"retrieval","respectsRobots":"yes","docsUrl":"https://developers.cloudflare.com/ai-search/platform/release-note/","deprecated":true,"replacedBy":"Cloudflare-AI-Search","caveat":"Old name of the Cloudflare AI Search crawler (renamed 2026-02-09). Cloudflare still honours it in robots.txt, but new requests say Cloudflare-AI-Search."},"robotsApplies":true,"deprecated":true},"cohere-ai":{"label":"Cohere AI","platform":"Cohere","tier":2,"emoji":"🟣","cat":"ai-bots","safety":"safe","impact":"low","purpose":"both","robotsTxt":{"status":"ALLOWED","rule":"Full content access; 2 path rules (/assets/, /quote)","source":"wildcard"},"effectiveStatus":"ALLOWED","intel":{"botType":"training","respectsRobots":"unknown"},"robotsApplies":true},"cohere-training-data-crawler":{"label":"Cohere Training","platform":"Cohere","tier":2,"emoji":"🟣","cat":"ai-bots","safety":"caution","impact":"low","purpose":"training","robotsTxt":{"status":"ALLOWED","rule":"Full content access; 2 path rules (/assets/, /quote)","source":"wildcard"},"effectiveStatus":"ALLOWED","intel":{"botType":"training","respectsRobots":"unknown"},"robotsApplies":true},"Crawlspace":{"label":"Crawlspace","platform":"Crawlspace","tier":6,"emoji":"🕷","cat":"scrapers","safety":"safe","impact":"low","purpose":"scraping","robotsTxt":{"status":"ALLOWED","rule":"Full content access; 2 path rules (/assets/, /quote)","source":"wildcard"},"effectiveStatus":"ALLOWED","intel":{"botType":"other","respectsRobots":"unknown"},"robotsApplies":true},"Cursor":{"label":"Cursor Agent","platform":"Anysphere","tier":3,"emoji":"💻","cat":"ai-bots","safety":"safe","impact":"low","purpose":"user-request","robotsTxt":{"status":"ALLOWED","rule":"Full content access; 2 path rules (/assets/, /quote)","source":"wildcard"},"effectiveStatus":"ALLOWED","intel":{"botType":"user-triggered","respectsRobots":"unknown"},"robotsApplies":true},"DeepSeekBot":{"label":"DeepSeek AI","platform":"DeepSeek","tier":2,"emoji":"🟣","cat":"ai-bots","safety":"aggressive","impact":"low","purpose":"both","robotsTxt":{"status":"ALLOWED","rule":"Full content access; 2 path rules (/assets/, /quote)","source":"wildcard"},"effectiveStatus":"ALLOWED","intel":{"botType":"training","respectsRobots":"unknown"},"robotsApplies":true},"Devin":{"label":"Devin (Cognition)","platform":"Cognition AI","tier":2,"emoji":"🟤","cat":"ai-bots","safety":"safe","impact":"low","purpose":"user-request","robotsTxt":{"status":"ALLOWED","rule":"Full content access; 2 path rules (/assets/, /quote)","source":"wildcard"},"effectiveStatus":"ALLOWED","intel":{"botType":"user-triggered","respectsRobots":"unknown"},"robotsApplies":true},"DoubaoBot":{"label":"Doubao (ByteDance)","platform":"ByteDance","tier":2,"emoji":"⚫","cat":"ai-bots","safety":"caution","impact":"low","purpose":"both","robotsTxt":{"status":"ALLOWED","rule":"Full content access; 2 path rules (/assets/, /quote)","source":"wildcard"},"effectiveStatus":"ALLOWED","intel":{"botType":"training","respectsRobots":"unknown"},"robotsApplies":true},"Echobot Bot":{"label":"Echobot Bot","platform":"Dealfront","tier":6,"emoji":"🕷","cat":"scrapers","safety":"safe","impact":"low","purpose":"scraping","robotsTxt":{"status":"ALLOWED","rule":"Full content access; 2 path rules (/assets/, /quote)","source":"wildcard"},"effectiveStatus":"ALLOWED","intel":{"botType":"other","respectsRobots":"unknown"},"robotsApplies":true},"EchoboxBot":{"label":"EchoboxBot","platform":"Echobox","tier":6,"emoji":"🕷","cat":"scrapers","safety":"safe","impact":"low","purpose":"scraping","robotsTxt":{"status":"ALLOWED","rule":"Full content access; 2 path rules (/assets/, /quote)","source":"wildcard"},"effectiveStatus":"ALLOWED","intel":{"botType":"other","respectsRobots":"unknown"},"robotsApplies":true},"ERNIEBot":{"label":"ERNIE (Baidu)","platform":"Baidu","tier":2,"emoji":"🔵","cat":"ai-bots","safety":"caution","impact":"low","purpose":"training","robotsTxt":{"status":"ALLOWED","rule":"Full content access; 2 path rules (/assets/, /quote)","source":"wildcard"},"effectiveStatus":"ALLOWED","intel":{"botType":"training","respectsRobots":"unknown"},"robotsApplies":true},"ExaBot":{"label":"Exa AI Search","platform":"Exa","tier":2,"emoji":"🟡","cat":"ai-bots","safety":"safe","impact":"low","purpose":"indexing","robotsTxt":{"status":"ALLOWED","rule":"Full content access; 2 path rules (/assets/, /quote)","source":"wildcard"},"effectiveStatus":"ALLOWED","intel":{"botType":"retrieval","respectsRobots":"unknown"},"robotsApplies":true},"FacebookBot":{"label":"Facebook Crawler","platform":"Meta","tier":5,"emoji":"🔵","cat":"social-bots","safety":"safe","impact":"unknown","purpose":"social","robotsTxt":{"status":"ALLOWED","rule":"Full content access; 2 path rules (/assets/, /quote)","source":"wildcard"},"effectiveStatus":"ALLOWED","intel":{"botType":"other","respectsRobots":"unknown"},"robotsApplies":true},"Factset_spyderbot":{"label":"Factset_spyderbot","platform":"FactSet","tier":6,"emoji":"🕷","cat":"scrapers","safety":"caution","impact":"low","purpose":"scraping","robotsTxt":{"status":"ALLOWED","rule":"Full content access; 2 path rules (/assets/, /quote)","source":"wildcard"},"effectiveStatus":"ALLOWED","intel":{"botType":"other","respectsRobots":"unknown"},"robotsApplies":true},"FriendlyCrawler":{"label":"FriendlyCrawler","platform":"FriendlyCrawler","tier":6,"emoji":"🕷","cat":"scrapers","safety":"safe","impact":"low","purpose":"scraping","robotsTxt":{"status":"ALLOWED","rule":"Full content access; 2 path rules (/assets/, /quote)","source":"wildcard"},"effectiveStatus":"ALLOWED","intel":{"botType":"other","respectsRobots":"unknown"},"robotsApplies":true},"Gemini-Deep-Research":{"label":"Gemini-Deep-Research","platform":"Google","tier":3,"emoji":"🔴","cat":"google-bots","safety":"safe","impact":"critical","purpose":"user-request","robotsTxt":{"status":"ALLOWED","rule":"Full content access; 2 path rules (/assets/, /quote)","source":"wildcard"},"effectiveStatus":"ALLOWED","intel":{"botType":"user-triggered","respectsRobots":"unknown"},"robotsApplies":true},"Google-Firebase":{"label":"Google Firebase AI","platform":"Google","tier":2,"emoji":"🔴","cat":"ai-bots","safety":"safe","impact":"low","purpose":"user-request","robotsTxt":{"status":"ALLOWED","rule":"Full content access; 2 path rules (/assets/, /quote)","source":"wildcard"},"effectiveStatus":"ALLOWED","intel":{"botType":"user-triggered","respectsRobots":"unknown"},"robotsApplies":true},"Google-Gemini-CLI":{"label":"Gemini CLI","platform":"Google","tier":2,"emoji":"🔴","cat":"ai-bots","safety":"safe","impact":"low","purpose":"user-request","robotsTxt":{"status":"ALLOWED","rule":"Full content access; 2 path rules (/assets/, /quote)","source":"wildcard"},"effectiveStatus":"ALLOWED","intel":{"botType":"user-triggered","respectsRobots":"unknown"},"robotsApplies":true},"Google-NotebookLM":{"label":"Google-NotebookLM (superseded)","platform":"Google","tier":3,"emoji":"🔴","cat":"google-bots","safety":"safe","impact":"low","purpose":"user-request","robotsTxt":{"status":"ALLOWED","rule":"Full content access; 2 path rules (/assets/, /quote)","source":"wildcard"},"effectiveStatus":"ALLOWED","intel":{"botType":"user-triggered","respectsRobots":"unknown"},"robotsApplies":true},"GoogleAgent-Mariner":{"label":"Project Mariner","platform":"Google","tier":2,"emoji":"🔴","cat":"ai-bots","safety":"safe","impact":"low","purpose":"user-request","robotsTxt":{"status":"ALLOWED","rule":"Full content access; 2 path rules (/assets/, /quote)","source":"wildcard"},"effectiveStatus":"ALLOWED","intel":{"botType":"user-triggered","respectsRobots":"unknown"},"robotsApplies":true},"GoogleAgent-URLContext":{"label":"Gemini URL Context","platform":"Google","tier":2,"emoji":"🔴","cat":"ai-bots","safety":"safe","impact":"low","purpose":"user-request","robotsTxt":{"status":"ALLOWED","rule":"Full content access; 2 path rules (/assets/, /quote)","source":"wildcard"},"effectiveStatus":"ALLOWED","intel":{"botType":"user-triggered","respectsRobots":"unknown"},"robotsApplies":true},"GrokBot":{"label":"GrokBot (xAI)","platform":"xAI","tier":2,"emoji":"🤖","cat":"ai-bots","safety":"safe","impact":"low","purpose":"both","robotsTxt":{"status":"ALLOWED","rule":"Full content access; 2 path rules (/assets/, /quote)","source":"wildcard"},"effectiveStatus":"ALLOWED","intel":{"botType":"training","respectsRobots":"unknown"},"robotsApplies":true},"ia_archiver":{"label":"ia_archiver","platform":"Internet Archive","tier":6,"emoji":"📡","cat":"other-agents","safety":"caution","impact":"unknown","purpose":"scraping","robotsTxt":{"status":"ALLOWED","rule":"Full content access; 2 path rules (/assets/, /quote)","source":"wildcard"},"effectiveStatus":"ALLOWED","intel":{"botType":"other","respectsRobots":"unknown"},"robotsApplies":true},"iAskBot":{"label":"iAsk AI","platform":"iAsk","tier":2,"emoji":"🟡","cat":"ai-bots","safety":"safe","impact":"low","purpose":"both","robotsTxt":{"status":"ALLOWED","rule":"Full content access; 2 path rules (/assets/, /quote)","source":"wildcard"},"effectiveStatus":"ALLOWED","intel":{"botType":"training","respectsRobots":"unknown"},"robotsApplies":true},"iaskspider":{"label":"iAsk Spider","platform":"iAsk","tier":2,"emoji":"🔎","cat":"ai-bots","safety":"safe","impact":"low","purpose":"scraping","robotsTxt":{"status":"ALLOWED","rule":"Full content access; 2 path rules (/assets/, /quote)","source":"wildcard"},"effectiveStatus":"ALLOWED","intel":{"botType":"other","respectsRobots":"unknown"},"robotsApplies":true},"ISSCyberRiskCrawler":{"label":"ISSCyberRiskCrawler","platform":"ISS","tier":6,"emoji":"🕷","cat":"scrapers","safety":"safe","impact":"low","purpose":"scraping","robotsTxt":{"status":"ALLOWED","rule":"Full content access; 2 path rules (/assets/, /quote)","source":"wildcard"},"effectiveStatus":"ALLOWED","intel":{"botType":"other","respectsRobots":"unknown"},"robotsApplies":true},"kagi-fetcher":{"label":"Kagi Assistant","platform":"Kagi","tier":2,"emoji":"🟡","cat":"ai-bots","safety":"safe","impact":"low","purpose":"user-request","robotsTxt":{"status":"ALLOWED","rule":"Full content access; 2 path rules (/assets/, /quote)","source":"wildcard"},"effectiveStatus":"ALLOWED","intel":{"botType":"user-triggered","respectsRobots":"unknown"},"robotsApplies":true},"Kangaroo Bot":{"label":"Kangaroo Bot","platform":"Kangaroo","tier":6,"emoji":"🕷","cat":"scrapers","safety":"safe","impact":"low","purpose":"scraping","robotsTxt":{"status":"ALLOWED","rule":"Full content access; 2 path rules (/assets/, /quote)","source":"wildcard"},"effectiveStatus":"ALLOWED","intel":{"botType":"other","respectsRobots":"unknown"},"robotsApplies":true},"Manus-User":{"label":"Manus AI Agent","platform":"Manus","tier":2,"emoji":"🟤","cat":"ai-bots","safety":"safe","impact":"low","purpose":"user-request","robotsTxt":{"status":"ALLOWED","rule":"Full content access; 2 path rules (/assets/, /quote)","source":"wildcard"},"effectiveStatus":"ALLOWED","intel":{"botType":"user-triggered","respectsRobots":"unknown"},"robotsApplies":true},"MauiBot":{"label":"MauiBot","platform":"MauiBot","tier":6,"emoji":"🕷","cat":"scrapers","safety":"safe","impact":"low","purpose":"scraping","robotsTxt":{"status":"ALLOWED","rule":"Full content access; 2 path rules (/assets/, /quote)","source":"wildcard"},"effectiveStatus":"ALLOWED","intel":{"botType":"other","respectsRobots":"unknown"},"robotsApplies":true},"MyCentralAIScraperBot":{"label":"MyCentralAIScraperBot","platform":"MyCentral AI","tier":2,"emoji":"🤖","cat":"ai-bots","safety":"safe","impact":"low","purpose":"training","robotsTxt":{"status":"ALLOWED","rule":"Full content access; 2 path rules (/assets/, /quote)","source":"wildcard"},"effectiveStatus":"ALLOWED","intel":{"botType":"training","respectsRobots":"unknown"},"robotsApplies":true},"netEstate Imprint Crawler":{"label":"netEstate Imprint Crawler","platform":"netEstate","tier":6,"emoji":"🕷","cat":"scrapers","safety":"safe","impact":"low","purpose":"scraping","robotsTxt":{"status":"ALLOWED","rule":"Full content access; 2 path rules (/assets/, /quote)","source":"wildcard"},"effectiveStatus":"ALLOWED","intel":{"botType":"other","respectsRobots":"unknown"},"robotsApplies":true},"Netvibes":{"label":"Netvibes","platform":"Dassault Systemes (Netvibes)","tier":4,"emoji":"🔧","cat":"seo-tools","safety":"safe","impact":"unknown","purpose":"monitoring","robotsTxt":{"status":"ALLOWED","rule":"Full content access; 2 path rules (/assets/, /quote)","source":"wildcard"},"effectiveStatus":"ALLOWED","intel":{"botType":"other","respectsRobots":"unknown"},"robotsApplies":true},"NovaAct":{"label":"Amazon Nova Act","platform":"Amazon","tier":2,"emoji":"🤖","cat":"ai-bots","safety":"safe","impact":"low","purpose":"scraping","robotsTxt":{"status":"ALLOWED","rule":"Full content access; 2 path rules (/assets/, /quote)","source":"wildcard"},"effectiveStatus":"ALLOWED","intel":{"botType":"other","respectsRobots":"unknown"},"robotsApplies":true},"opencode":{"label":"OpenCode Agent","platform":"OpenCode","tier":3,"emoji":"💻","cat":"ai-bots","safety":"safe","impact":"low","purpose":"user-request","robotsTxt":{"status":"ALLOWED","rule":"Full content access; 2 path rules (/assets/, /quote)","source":"wildcard"},"effectiveStatus":"ALLOWED","intel":{"botType":"user-triggered","respectsRobots":"unknown"},"robotsApplies":true},"PanguBot":{"label":"Huawei PanguBot","platform":"Huawei","tier":2,"emoji":"🔴","cat":"ai-bots","safety":"safe","impact":"low","purpose":"training","robotsTxt":{"status":"ALLOWED","rule":"Full content access; 2 path rules (/assets/, /quote)","source":"wildcard"},"effectiveStatus":"ALLOWED","intel":{"botType":"training","respectsRobots":"unknown"},"robotsApplies":true},"PhindBot":{"label":"Phind AI","platform":"Phind","tier":2,"emoji":"🟣","cat":"ai-bots","safety":"safe","impact":"low","purpose":"user-request","robotsTxt":{"status":"ALLOWED","rule":"Full content access; 2 path rules (/assets/, /quote)","source":"wildcard"},"effectiveStatus":"ALLOWED","intel":{"botType":"user-triggered","respectsRobots":"unknown"},"robotsApplies":true},"Poseidon Research Crawler":{"label":"Poseidon Research Crawler","platform":"Poseidon Research","tier":6,"emoji":"🕷","cat":"scrapers","safety":"safe","impact":"low","purpose":"scraping","robotsTxt":{"status":"ALLOWED","rule":"Full content access; 2 path rules (/assets/, /quote)","source":"wildcard"},"effectiveStatus":"ALLOWED","intel":{"botType":"other","respectsRobots":"unknown"},"robotsApplies":true},"QualifiedBot":{"label":"QualifiedBot","platform":"Qualified","tier":6,"emoji":"🕷","cat":"scrapers","safety":"safe","impact":"low","purpose":"scraping","robotsTxt":{"status":"ALLOWED","rule":"Full content access; 2 path rules (/assets/, /quote)","source":"wildcard"},"effectiveStatus":"ALLOWED","intel":{"botType":"other","respectsRobots":"unknown"},"robotsApplies":true},"QuillBot":{"label":"QuillBot AI","platform":"QuillBot","tier":2,"emoji":"🟡","cat":"ai-bots","safety":"caution","impact":"low","purpose":"user-request","robotsTxt":{"status":"ALLOWED","rule":"Full content access; 2 path rules (/assets/, /quote)","source":"wildcard"},"effectiveStatus":"ALLOWED","intel":{"botType":"user-triggered","respectsRobots":"unknown"},"robotsApplies":true},"Quora-Bot":{"label":"Quora-Bot","platform":"Quora","tier":5,"emoji":"💬","cat":"social-bots","safety":"safe","impact":"unknown","purpose":"social","robotsTxt":{"status":"ALLOWED","rule":"Full content access; 2 path rules (/assets/, /quote)","source":"wildcard"},"effectiveStatus":"ALLOWED","intel":{"botType":"other","respectsRobots":"unknown"},"robotsApplies":true},"QwenBot":{"label":"Qwen (Alibaba)","platform":"Alibaba","tier":2,"emoji":"🟠","cat":"ai-bots","safety":"caution","impact":"low","purpose":"training","robotsTxt":{"status":"ALLOWED","rule":"Full content access; 2 path rules (/assets/, /quote)","source":"wildcard"},"effectiveStatus":"ALLOWED","intel":{"botType":"training","respectsRobots":"unknown"},"robotsApplies":true},"Shap-User":{"label":"Parallel Fetcher","platform":"Parallel","tier":3,"emoji":"🤖","cat":"ai-bots","safety":"safe","impact":"low","purpose":"user-request","robotsTxt":{"status":"ALLOWED","rule":"Full content access; 2 path rules (/assets/, /quote)","source":"wildcard"},"effectiveStatus":"ALLOWED","intel":{"botType":"user-triggered","respectsRobots":"unknown"},"robotsApplies":true},"Sidetrade Indexer Bot":{"label":"Sidetrade Indexer Bot","platform":"Sidetrade","tier":4,"emoji":"🔧","cat":"seo-tools","safety":"safe","impact":"unknown","purpose":"monitoring","robotsTxt":{"status":"ALLOWED","rule":"Full content access; 2 path rules (/assets/, /quote)","source":"wildcard"},"effectiveStatus":"ALLOWED","intel":{"botType":"other","respectsRobots":"unknown"},"robotsApplies":true},"TavilyBot":{"label":"Tavily Search","platform":"Tavily","tier":2,"emoji":"🟡","cat":"ai-bots","safety":"safe","impact":"low","purpose":"user-request","robotsTxt":{"status":"ALLOWED","rule":"Full content access; 2 path rules (/assets/, /quote)","source":"wildcard"},"effectiveStatus":"ALLOWED","intel":{"botType":"user-triggered","respectsRobots":"unknown"},"robotsApplies":true},"TikTokSpider":{"label":"TikTok Spider","platform":"ByteDance","tier":2,"emoji":"⚫","cat":"scrapers","safety":"aggressive","impact":"low","purpose":"training","robotsTxt":{"status":"ALLOWED","rule":"Full content access; 2 path rules (/assets/, /quote)","source":"wildcard"},"effectiveStatus":"ALLOWED","intel":{"botType":"training","respectsRobots":"unknown"},"robotsApplies":true},"Timpibot":{"label":"Timpibot","platform":"Timpi","tier":6,"emoji":"🕷","cat":"scrapers","safety":"safe","impact":"low","purpose":"scraping","robotsTxt":{"status":"ALLOWED","rule":"Full content access; 2 path rules (/assets/, /quote)","source":"wildcard"},"effectiveStatus":"ALLOWED","intel":{"botType":"other","respectsRobots":"unknown"},"robotsApplies":true},"TongyiBot":{"label":"Tongyi Qianwen Fetcher","platform":"Alibaba","tier":2,"emoji":"🟠","cat":"ai-bots","safety":"caution","impact":"low","purpose":"user-request","robotsTxt":{"status":"ALLOWED","rule":"Full content access; 2 path rules (/assets/, /quote)","source":"wildcard"},"effectiveStatus":"ALLOWED","intel":{"botType":"user-triggered","respectsRobots":"unknown"},"robotsApplies":true},"Trae":{"label":"Trae Agent (ByteDance)","platform":"ByteDance","tier":3,"emoji":"💻","cat":"ai-bots","safety":"safe","impact":"low","purpose":"user-request","robotsTxt":{"status":"ALLOWED","rule":"Full content access; 2 path rules (/assets/, /quote)","source":"wildcard"},"effectiveStatus":"ALLOWED","intel":{"botType":"user-triggered","respectsRobots":"unknown"},"robotsApplies":true},"WRTNBot":{"label":"WRTN AI","platform":"WRTN","tier":2,"emoji":"🟡","cat":"ai-bots","safety":"safe","impact":"low","purpose":"both","robotsTxt":{"status":"ALLOWED","rule":"Full content access; 2 path rules (/assets/, /quote)","source":"wildcard"},"effectiveStatus":"ALLOWED","intel":{"botType":"training","respectsRobots":"unknown"},"robotsApplies":true},"YiyanBot":{"label":"Yiyan Fetcher (Baidu)","platform":"Baidu","tier":2,"emoji":"🔵","cat":"ai-bots","safety":"caution","impact":"low","purpose":"user-request","robotsTxt":{"status":"ALLOWED","rule":"Full content access; 2 path rules (/assets/, /quote)","source":"wildcard"},"effectiveStatus":"ALLOWED","intel":{"botType":"user-triggered","respectsRobots":"unknown"},"robotsApplies":true},"YouBot":{"label":"You.com AI","platform":"You","tier":2,"emoji":"🟡","cat":"ai-bots","safety":"safe","impact":"low","purpose":"both","robotsTxt":{"status":"ALLOWED","rule":"Full content access; 2 path rules (/assets/, /quote)","source":"wildcard"},"effectiveStatus":"ALLOWED","intel":{"botType":"training","respectsRobots":"unknown"},"robotsApplies":true},"IbouBot":{"label":"Ibou Search","platform":"Ibou","tier":3,"emoji":"🔍","cat":"search-engines","safety":"safe","impact":"unknown","purpose":"indexing","robotsTxt":{"status":"ALLOWED","rule":"Full content access; 2 path rules (/assets/, /quote)","source":"wildcard"},"effectiveStatus":"ALLOWED","intel":{"botType":"search","respectsRobots":"yes","docsUrl":"https://www.ibou.com/iboubot.html","caveat":"Ibou documents IbouBot as the crawler for its own search engine, says it follows robots.txt and Crawl-delay, and says it does not train AI models with the data."},"robotsApplies":true},"LAIONDownloader":{"label":"LAION Downloader","platform":"LAION","tier":2,"emoji":"🤖","cat":"ai-bots","safety":"caution","impact":"low","purpose":"training","robotsTxt":{"status":"ALLOWED","rule":"Full content access; 2 path rules (/assets/, /quote)","source":"wildcard"},"effectiveStatus":"ALLOWED","intel":{"botType":"training","respectsRobots":"unknown"},"robotsApplies":true},"xAI-SearchBot":{"label":"Grok Search","platform":"xAI","tier":2,"emoji":"⚫","cat":"ai-bots","safety":"caution","impact":"low","purpose":"indexing","robotsTxt":{"status":"ALLOWED","rule":"Full content access; 2 path rules (/assets/, /quote)","source":"wildcard"},"effectiveStatus":"ALLOWED","intel":{"botType":"retrieval","respectsRobots":"unknown"},"robotsApplies":true},"ChatGPT Agent":{"label":"ChatGPT Agent (Cloud browser)","platform":"OpenAI","tier":1,"emoji":"🟢","cat":"ai-bots","safety":"safe","impact":"low","purpose":"user-request","robotsTxt":{"status":"ALLOWED","rule":"Full content access; 2 path rules (/assets/, /quote)","source":"wildcard"},"effectiveStatus":"ALLOWED","intel":{"botType":"user-triggered","respectsRobots":"unknown"},"robotsApplies":true},"ChatGPT Operator":{"label":"ChatGPT Operator (Cloudflare name)","platform":"OpenAI","tier":1,"emoji":"🟢","cat":"ai-bots","safety":"safe","impact":"low","purpose":"user-request","robotsTxt":{"status":"ALLOWED","rule":"Full content access; 2 path rules (/assets/, /quote)","source":"wildcard"},"effectiveStatus":"ALLOWED","intel":{"botType":"user-triggered","respectsRobots":"unknown"},"robotsApplies":true},"AIWebIndex":{"label":"Lyrenth AI Web Index","platform":"Lyrenth","tier":2,"emoji":"🤖","cat":"ai-bots","safety":"safe","impact":"low","purpose":"indexing","robotsTxt":{"status":"ALLOWED","rule":"Full content access; 2 path rules (/assets/, /quote)","source":"wildcard"},"effectiveStatus":"ALLOWED","intel":{"botType":"retrieval","respectsRobots":"unknown"},"robotsApplies":true},"LinkupBot":{"label":"Linkup Search","platform":"Linkup","tier":2,"emoji":"🔎","cat":"ai-bots","safety":"safe","impact":"low","purpose":"indexing","robotsTxt":{"status":"ALLOWED","rule":"Full content access; 2 path rules (/assets/, /quote)","source":"wildcard"},"effectiveStatus":"ALLOWED","intel":{"botType":"retrieval","respectsRobots":"unknown"},"robotsApplies":true},"Mozilla-Tabstack":{"label":"Mozilla Tabstack","platform":"Mozilla","tier":2,"emoji":"🤖","cat":"ai-bots","safety":"safe","impact":"low","purpose":"user-request","robotsTxt":{"status":"ALLOWED","rule":"Full content access; 2 path rules (/assets/, /quote)","source":"wildcard"},"effectiveStatus":"ALLOWED","intel":{"botType":"user-triggered","respectsRobots":"unknown"},"robotsApplies":true},"TerraCotta":{"label":"Ceramic AI TerraCotta","platform":"Ceramic AI","tier":2,"emoji":"🤖","cat":"ai-bots","safety":"caution","impact":"low","purpose":"training","robotsTxt":{"status":"ALLOWED","rule":"Full content access; 2 path rules (/assets/, /quote)","source":"wildcard"},"effectiveStatus":"ALLOWED","intel":{"botType":"training","respectsRobots":"unknown"},"robotsApplies":true},"Panscient":{"label":"Panscient","platform":"Panscient","tier":6,"emoji":"🕷","cat":"scrapers","safety":"safe","impact":"low","purpose":"scraping","robotsTxt":{"status":"ALLOWED","rule":"Full content access; 2 path rules (/assets/, /quote)","source":"wildcard"},"effectiveStatus":"ALLOWED","intel":{"botType":"other","respectsRobots":"unknown"},"robotsApplies":true},"Reflectionbot":{"label":"Reflectionbot (Reflection AI)","platform":"Reflection AI","tier":2,"emoji":"🤖","cat":"ai-bots","safety":"safe","impact":"low","purpose":"training","robotsTxt":{"status":"ALLOWED","rule":"Full content access; 2 path rules (/assets/, /quote)","source":"wildcard"},"effectiveStatus":"ALLOWED","intel":{"botType":"training","respectsRobots":"unknown"},"robotsApplies":true},"KernelSearchBot":{"label":"Kernel Search","platform":"Kernel","tier":2,"emoji":"🔎","cat":"ai-bots","safety":"safe","impact":"low","purpose":"indexing","robotsTxt":{"status":"ALLOWED","rule":"Full content access; 2 path rules (/assets/, /quote)","source":"wildcard"},"effectiveStatus":"ALLOWED","intel":{"botType":"retrieval","respectsRobots":"unknown"},"robotsApplies":true},"alphalens-bot":{"label":"Alphalens Bot","platform":"Alphalens","tier":3,"emoji":"🔎","cat":"ai-bots","safety":"safe","impact":"low","purpose":"indexing","robotsTxt":{"status":"ALLOWED","rule":"Full content access; 2 path rules (/assets/, /quote)","source":"wildcard"},"effectiveStatus":"ALLOWED","intel":{"botType":"retrieval","respectsRobots":"unknown"},"robotsApplies":true},"Anomura":{"label":"Direqt Anomura","platform":"Direqt","tier":3,"emoji":"🔎","cat":"ai-bots","safety":"safe","impact":"low","purpose":"indexing","robotsTxt":{"status":"ALLOWED","rule":"Full content access; 2 path rules (/assets/, /quote)","source":"wildcard"},"effectiveStatus":"ALLOWED","intel":{"botType":"retrieval","respectsRobots":"unknown"},"robotsApplies":true},"HarkBot":{"label":"Hark Agent","platform":"Hark","tier":2,"emoji":"🤖","cat":"ai-bots","safety":"safe","impact":"low","purpose":"user-request","robotsTxt":{"status":"ALLOWED","rule":"Full content access; 2 path rules (/assets/, /quote)","source":"wildcard"},"effectiveStatus":"ALLOWED","intel":{"botType":"user-triggered","respectsRobots":"unknown"},"robotsApplies":true},"ZipchatBot":{"label":"Zipchat Shopping Assistant","platform":"Zipchat","tier":3,"emoji":"🛍","cat":"ai-bots","safety":"safe","impact":"low","purpose":"user-request","robotsTxt":{"status":"ALLOWED","rule":"Full content access; 2 path rules (/assets/, /quote)","source":"wildcard"},"effectiveStatus":"ALLOWED","intel":{"botType":"user-triggered","respectsRobots":"unknown"},"robotsApplies":true},"GrokAgent":{"label":"Grok Bot (xAI agent)","platform":"xAI","tier":1,"emoji":"⚫","cat":"ai-bots","safety":"caution","impact":"low","purpose":"user-request","robotsTxt":{"status":"ALLOWED","rule":"Full content access; 2 path rules (/assets/, /quote)","source":"wildcard"},"effectiveStatus":"ALLOWED","intel":{"botType":"user-triggered","respectsRobots":"unknown"},"robotsApplies":true},"muse-":{"label":"Meta Muse (personal agent)","platform":"Meta","tier":1,"emoji":"🔵","cat":"ai-bots","safety":"caution","impact":"low","purpose":"user-request","robotsTxt":{"status":"ALLOWED","rule":"Full content access; 2 path rules (/assets/, /quote)","source":"wildcard"},"effectiveStatus":"ALLOWED","intel":{"botType":"user-triggered","respectsRobots":"unknown"},"robotsApplies":true},"meta-externaltest":{"label":"Meta-ExternalTest","platform":"Meta","tier":4,"emoji":"🔵","cat":"ai-bots","safety":"safe","impact":"low","purpose":"monitoring","robotsTxt":{"status":"ALLOWED","rule":"Full content access; 2 path rules (/assets/, /quote)","source":"wildcard"},"effectiveStatus":"ALLOWED","intel":{"botType":"other","respectsRobots":"unknown"},"robotsApplies":true},"SemrushBot-SWA":{"label":"Semrush Writing Assistant","platform":"Semrush","tier":4,"emoji":"🔧","cat":"seo-tools","safety":"safe","impact":"unknown","purpose":"monitoring","robotsTxt":{"status":"ALLOWED","rule":"Full content access; 2 path rules (/assets/, /quote)","source":"wildcard"},"effectiveStatus":"ALLOWED","intel":{"botType":"other","respectsRobots":"unknown"},"robotsApplies":true},"Cloudflare-AI-Search":{"label":"Cloudflare AI Search","platform":"Cloudflare","tier":2,"emoji":"🟠","cat":"ai-bots","safety":"safe","impact":"low","purpose":"indexing","robotsTxt":{"status":"ALLOWED","rule":"Full content access; 2 path rules (/assets/, /quote)","source":"wildcard"},"effectiveStatus":"ALLOWED","intel":{"botType":"retrieval","respectsRobots":"yes","docsUrl":"https://developers.cloudflare.com/ai-search/configuration/data-source/website/parse-types/","caveat":"Crawls a site for a Cloudflare customer who built an AI Search index of it. Renamed from Cloudflare-AutoRAG on 2026-02-09; Cloudflare still honours the old token in robots.txt."},"robotsApplies":true},"Cloudflare-AI-Search-External":{"label":"Cloudflare AI Search (external sites)","platform":"Cloudflare","tier":3,"emoji":"🟠","cat":"ai-bots","safety":"safe","impact":"low","purpose":"indexing","robotsTxt":{"status":"ALLOWED","rule":"Full content access; 2 path rules (/assets/, /quote)","source":"wildcard"},"effectiveStatus":"ALLOWED","intel":{"botType":"retrieval","respectsRobots":"yes","docsUrl":"https://developers.cloudflare.com/ai-search/configuration/data-source/website/parse-types/","caveat":"The same AI Search crawler when a customer index follows links onto a domain outside that customer's Cloudflare account. Pages robots.txt disallows are skipped."},"robotsApplies":true},"CloudflareBrowserRenderingCrawler":{"label":"Cloudflare Crawler (Browser Run)","platform":"Cloudflare","tier":3,"emoji":"🟠","cat":"ai-bots","safety":"safe","impact":"low","purpose":"scraping","robotsTxt":{"status":"ALLOWED","rule":"Full content access; 2 path rules (/assets/, /quote)","source":"wildcard"},"effectiveStatus":"ALLOWED","intel":{"botType":"other","respectsRobots":"yes","docsUrl":"https://developers.cloudflare.com/browser-run/reference/robots-txt/","caveat":"The /crawl endpoint of Cloudflare Browser Run, used by Cloudflare customers to crawl sites. Other Browser Run endpoints use a different user agent, so this token only blocks /crawl."},"robotsApplies":true},"AIWebIndex-Agent":{"label":"Lyrenth on-demand fetch","platform":"Lyrenth","tier":3,"emoji":"🤖","cat":"ai-bots","safety":"safe","impact":"low","purpose":"user-request","robotsTxt":{"status":"ALLOWED","rule":"Full content access; 2 path rules (/assets/, /quote)","source":"wildcard"},"effectiveStatus":"ALLOWED","intel":{"botType":"user-triggered","respectsRobots":"yes","docsUrl":"https://lyrenth.com/bot","webBotAuth":"implemented","caveat":"On-demand fetch for the Lyrenth index when a customer asks for a specific URL. Lyrenth says it honours robots.txt the same way as AIWebIndex and does not train models."},"robotsApplies":true},"KeenableBot":{"label":"Keenable Search","platform":"Keenable","tier":2,"emoji":"🔎","cat":"ai-bots","safety":"safe","impact":"low","purpose":"indexing","robotsTxt":{"status":"ALLOWED","rule":"Full content access; 2 path rules (/assets/, /quote)","source":"wildcard"},"effectiveStatus":"ALLOWED","intel":{"botType":"retrieval","respectsRobots":"yes","docsUrl":"https://app.keenable.ai/bot","ipRangesUrl":"https://app.keenable.ai/bot/keenablebot.json","caveat":"Builds the Keenable search index that answers AI agents and cites the source page. Blocking it removes the site from those answers."},"robotsApplies":true},"Keenable-User":{"label":"Keenable user fetch","platform":"Keenable","tier":3,"emoji":"🤖","cat":"ai-bots","safety":"safe","impact":"low","purpose":"user-request","robotsTxt":{"status":"ALLOWED","rule":"Full content access; 2 path rules (/assets/, /quote)","source":"wildcard"},"effectiveStatus":"ALLOWED","intel":{"botType":"user-triggered","respectsRobots":"yes","docsUrl":"https://app.keenable.ai/bot","ipRangesUrl":"https://app.keenable.ai/bot/keenable-user.json","caveat":"Fetches one page when a person, or an agent acting for one, asks for it. Keenable says it follows robots.txt (RFC 9309) for this token too."},"robotsApplies":true},"Channel3Bot":{"label":"Channel3 product index","platform":"Channel3","tier":3,"emoji":"🛍","cat":"ai-bots","safety":"safe","impact":"low","purpose":"indexing","robotsTxt":{"status":"ALLOWED","rule":"Full content access; 2 path rules (/assets/, /quote)","source":"wildcard"},"effectiveStatus":"ALLOWED","intel":{"botType":"retrieval","respectsRobots":"yes","docsUrl":"https://www.trychannel3.com/channel3bot","caveat":"Indexes product detail pages into a catalogue AI shopping agents search. Verify with reverse DNS: hostnames end in bot.trychannel3.com."},"robotsApplies":true},"Yeti":{"label":"Naver Search","platform":"Naver","tier":3,"emoji":"🔍","cat":"search-engines","safety":"safe","impact":"critical","purpose":"indexing","robotsTxt":{"status":"ALLOWED","rule":"Full content access; 2 path rules (/assets/, /quote)","source":"wildcard"},"effectiveStatus":"ALLOWED","intel":{"botType":"search","respectsRobots":"yes","docsUrl":"https://searchadvisor.naver.com/guide/seo-basic-firewall","caveat":"Naver, South Korea's largest search engine. Blocking Yeti removes the site from Naver Search. Verify with reverse DNS: hostnames end in .naver.com."},"robotsApplies":true}},"aiFiles":{"llms.txt":{"exists":true,"wordCount":31,"statusCode":200},"llms-full.txt":{"exists":false,"wordCount":null,"statusCode":404}},"scoreBreakdown":{"total":89,"botAccess":70,"aiInfra":10,"techReady":9,"catSummary":{"ai-bots":{"total":104,"allowed":104,"blocked":0,"partial":0},"search-engines":{"total":63,"allowed":63,"blocked":0,"partial":0},"seo-tools":{"total":19,"allowed":19,"blocked":0,"partial":0},"social-bots":{"total":9,"allowed":9,"blocked":0,"partial":0},"google-bots":{"total":24,"allowed":22,"blocked":2,"partial":0},"scrapers":{"total":39,"allowed":39,"blocked":0,"partial":0},"cloud-services":{"total":3,"allowed":3,"blocked":0,"partial":0},"other-agents":{"total":4,"allowed":4,"blocked":0,"partial":0}}},"recommendations":[{"priority":"high","title":"2 search engine(s) blocked","description":"Googlebot-Image, Storebot-Google are blocked. This directly impacts your SEO rankings.","impact":20,"fix":"# Unblock critical search engines:\nUser-agent: Googlebot-Image\nAllow: /\n\nUser-agent: Storebot-Google\nAllow: /\n"},{"priority":"high","title":"Add structured data (JSON-LD)","description":"No Schema.org markup detected on your homepage. Structured data helps AI engines understand and cite your content correctly.","impact":6,"fix":"<script type=\"application/ld+json\">\n{\n  \"@context\": \"https://schema.org\",\n  \"@type\": \"Organization\",\n  \"name\": \"[Your Company]\",\n  \"url\": \"https://yoursite.com\"\n}\n</script>"},{"priority":"low","title":"Add llms-full.txt (optional)","description":"A consolidated Markdown dump of your key content for direct LLM ingestion. Pairs with llms.txt. Same honest caveat: publish-side adoption is real, read-side adoption is not yet.","impact":5,"fix":"# Create /llms-full.txt - consolidated Markdown of key content"}],"tier1BlockedCount":0,"tier1Total":14,"totalBots":265,"totalAllowed":263,"totalBlocked":2,"totalPartial":0,"siteReachable":true,"homepageBlocked":false,"homepageFetchMethod":"browser-ua","isHttps":true,"processingTime":8000,"version":"6.2","intelVersion":"2026-10-19","staleTokensFound":[],"webBotAuthDirectory":false,"undetectableAgents":[{"name":"ChatGPT Atlas","operator":"OpenAI","note":"Agentic browser. Browses with a standard browser user agent; no bot signal at the HTTP layer."},{"name":"Perplexity Comet","operator":"Perplexity","note":"Agentic browser launched July 2025. Traffic looks like a normal Chrome user."},{"name":"Microsoft Copilot Actions","operator":"Microsoft","note":"Agentic automation in Microsoft 365 Copilot. Uses standard Edge/Chromium UAs with no bot signal."},{"name":"Claude for Chrome","operator":"Anthropic","note":"Drives a real browser session; HTTP layer is silent about the AI agent."},{"name":"Opera Neon","operator":"Opera","note":"Agentic browser; traffic matches the underlying browser UA."},{"name":"xAI Grok retrieval","operator":"xAI","note":"Reported to use residential IP rotation and spoofed browser UAs. No documentation, no IP ranges, no robots.txt contract."}],"tech":{"sitemapExists":true,"sitemapUrl":"https://lab2.crawlcheck.io/sitemap.xml","hasSchema":false,"isHttps":true},"crawlability":{"url":"https://lab2.crawlcheck.io/","path":"/","crawlable":true,"indexable":true,"verdict":"Crawlable and indexable: Googlebot can fetch this page and may index it.","gates":[{"id":"robots","label":"robots.txt allows Googlebot","status":"pass","detail":"No rule in the User-agent: * group matches /"},{"id":"status","label":"Page answers Googlebot","status":"pass","detail":"HTTP 200 to the Googlebot smartphone user agent"},{"id":"challenge","label":"No challenge or block page","status":"pass","detail":"The response is the real page, not a challenge"},{"id":"index","label":"Page is indexable (no noindex)","status":"warn","detail":"<meta name=\"robots\"> says \"nosnippet\": indexable, but Google will not show a snippet"},{"id":"canonical","label":"Canonical points to this URL","status":"pass","detail":"Self-referencing: https://lab2.crawlcheck.io/"},{"id":"render","label":"Content is in the HTML before JavaScript","status":"pass","detail":"1,229 characters of text are present without JavaScript."}],"robots":[{"token":"Googlebot","name":"Googlebot","allowed":true,"rule":"No rule in the User-agent: * group matches /"},{"token":"Googlebot-Image","name":"Googlebot-Image","allowed":false,"rule":"Disallow: / (line 11, its own Googlebot-Image group)"},{"token":"bingbot","name":"Bingbot","allowed":true,"rule":"No rule in the User-agent: * group matches /"},{"token":"Applebot","name":"Applebot","allowed":true,"rule":"No rule in the User-agent: * group matches /"}],"mobile":{"status":200,"bytes":4130,"ttfbMs":57,"finalUrl":"https://lab2.crawlcheck.io/","redirected":false},"fakeBotProtection":false,"note":"Fetched from our server with the Googlebot smartphone user agent, not from Google IP ranges. A site that verifies Googlebot by reverse DNS may answer the real Googlebot differently, so confirm important pages with URL Inspection in Google Search Console."},"geoAudit":{"engine":"2.0","overallScore":67,"grade":{"letter":"C","label":"Fair"},"gate":null,"categories":[{"key":"access","name":"AI Crawler Access","weight":30,"score":88,"passed":7,"total":9},{"key":"technical","name":"Technical SEO","weight":20,"score":98,"passed":8,"total":9},{"key":"content","name":"Content Structure","weight":20,"score":52,"passed":5,"total":10},{"key":"structured","name":"Structured Data","weight":15,"score":9,"passed":0,"total":5},{"key":"trust","name":"Trust & Authority","weight":15,"score":60,"passed":2,"total":3}],"summary":{"passed":22,"warnings":6,"failed":8,"notApplicable":6,"total":42},"recommendations":[{"id":"schema_present","title":"Valid JSON-LD structured data","fix":"Add JSON-LD for the page type (Organization, Article, Product, FAQPage) in the <head>.","why":"Structured data states facts about the page in a form engines do not have to guess. Pages cited by AI Mode and ChatGPT carry it far more often than not.","category":"Structured Data","priority":"high","gain":4.6,"evidence":"1 JSON-LD block, none of them valid JSON.","tool":{"href":"/tools/schema-generator","label":"schema markup generator"}},{"id":"content_depth","title":"Enough main content to cite","fix":"Expand the main content with the details, examples and data a reader would ask for next.","why":"Engines cite pages that answer a question fully. A few sentences rarely win a citation over a thorough source.","category":"Content Structure","priority":"high","gain":3.3,"evidence":"145 words of main content."},{"id":"outbound_sources","title":"Links to supporting sources","fix":"Link to the primary sources behind your data and claims.","why":"Citing sources shows the claims are checkable, which is part of how engines judge trustworthiness.","category":"Trust & Authority","priority":"medium","gain":6,"evidence":"No links to external sources."},{"id":"schema_entity","title":"Organization or Person entity","fix":"Add an Organization node with name, url, logo and sameAs links to your official profiles.","why":"An Organization or Person node tells engines who publishes the content, which links the page to a known entity.","category":"Structured Data","priority":"medium","gain":3.5,"evidence":"No Organization, LocalBusiness or Person node.","tool":{"href":"/tools/schema-generator","label":"schema markup generator"}},{"id":"nosnippet","title":"Snippets allowed (no nosnippet)","fix":"Remove nosnippet, or use data-nosnippet on the specific passages you want hidden.","why":"nosnippet or max-snippet:0 lets engines index the page but stops them quoting it, which also removes it from AI Overviews text.","category":"AI Crawler Access","priority":"medium","gain":3,"evidence":"nosnippet or max-snippet:0 is set."},{"id":"question_headings","title":"Questions answered in headings","fix":"Phrase two or more headings as the questions your audience asks, and answer each in the first sentence below it.","why":"People ask AI assistants questions. Headings phrased as those questions, followed by a direct answer, map cleanly onto prompts.","category":"Content Structure","priority":"medium","gain":2.5,"evidence":"No headings phrased as questions.","tool":{"href":"/blog/answer-first-content-structure","label":"answer-first structure guide"}},{"id":"schema_content","title":"Content-type schema (Article, Product, FAQ...)","fix":"Add the schema type that matches the page: Article for posts, Product for products, FAQPage for visible Q&A.","why":"A type that describes the page itself (Article, Product, FAQPage, HowTo, Service) gives engines the headline, dates and answers directly.","category":"Structured Data","priority":"medium","gain":2.4,"evidence":"Only site-level types (WebSite, WebPage, BreadcrumbList) or none.","tool":{"href":"/tools/schema-generator","label":"schema markup generator"}},{"id":"open_graph","title":"Open Graph title, description and image","fix":"Add og:title, og:description and og:image.","why":"Open Graph tags control how the page appears when shared in chat apps and assistants that render link previews.","category":"Structured Data","priority":"medium","gain":2.3,"evidence":"No Open Graph tags."},{"id":"subheadings","title":"Sections under descriptive H2s","fix":"Break the content into sections of 150 to 300 words, each under an H2 that names its topic.","why":"AI engines extract passages section by section. Clear H2s let them find the part that answers a question.","category":"Content Structure","priority":"medium","gain":1.3,"evidence":"Only 1 H2."},{"id":"sitemap","title":"XML sitemap found","fix":"Publish an XML sitemap and reference it with a Sitemap: line in robots.txt.","why":"Sitemaps help crawlers discover pages and notice updates. Bing states that it uses lastmod to prioritise recrawls for Copilot.","category":"Technical SEO","priority":"medium","gain":0.5,"evidence":"Sitemap index with 2 child sitemaps at https://lab2.crawlcheck.io/sitemap.xml. Not declared in robots.txt.","tool":{"href":"/blog/sitemap-xml-for-ai-discoverability","label":"sitemap guide"}}],"criteria":[{"id":"ai_search_robots","cat":"access","name":"AI search bots allowed in robots.txt","weight":5,"status":"pass","score":100,"evidence":"OAI-SearchBot, Claude-SearchBot and PerplexityBot may fetch this URL.","fix":"Give each AI search bot its own group with Allow: /, or remove the Disallow that catches them.","tool":{"href":"/tools/robots-txt-validator","label":"robots.txt validator"},"gain":0},{"id":"ai_user_robots","cat":"access","name":"AI assistants allowed in robots.txt","weight":3,"status":"pass","score":100,"evidence":"ChatGPT-User, Claude-User and Perplexity-User are allowed.","fix":"Allow the user-triggered agents; they fetch single pages on request and add little load.","tool":{"href":"/tools/robots-txt-generator","label":"robots.txt generator"},"gain":0},{"id":"search_engine_robots","cat":"access","name":"Search engines allowed in robots.txt","weight":5,"status":"pass","score":100,"evidence":"Googlebot and Bingbot may fetch this URL.","fix":"Remove the Disallow that applies to Googlebot or Bingbot for this path.","tool":{"href":"/tools/robots-txt-validator","label":"robots.txt validator"},"gain":0},{"id":"training_policy","cat":"access","name":"AI training policy is explicit","weight":1,"status":"warn","score":50,"evidence":"Only ClaudeBot named; the rest follow the wildcard.","fix":"Add named groups for GPTBot, ClaudeBot, Google-Extended and the other training tokens, set to the policy you want.","tool":{"href":"/blog/training-bots-vs-search-bots","label":"training vs search bots guide"},"gain":0.5},{"id":"firewall","cat":"access","name":"No firewall block for AI crawlers","weight":5,"status":"na","score":0,"evidence":"Not tested in the quick scan. Run the full GEO audit.","fix":"Allow verified AI search bots in your CDN or WAF bot settings, then re-run the check.","tool":{"href":"/tools/ai-crawler-diagnostic","label":"AI crawler diagnostic"},"gain":0},{"id":"noindex","cat":"access","name":"Page is indexable (no noindex)","weight":5,"status":"pass","score":100,"evidence":"Directives present but indexable: \"nosnippet\".","fix":"Remove noindex from pages you want cited.","gain":0},{"id":"nosnippet","cat":"access","name":"Snippets allowed (no nosnippet)","weight":3,"status":"fail","score":0,"evidence":"nosnippet or max-snippet:0 is set.","fix":"Remove nosnippet, or use data-nosnippet on the specific passages you want hidden.","gain":3},{"id":"server_rendered","cat":"access","name":"Content is in the HTML without JavaScript","weight":5,"status":"pass","score":100,"evidence":"1,138 characters of text are present before any script runs.","fix":"Server-render or pre-render the main content (SSR, SSG, or a prerender service).","tool":{"href":"/blog/javascript-rendering-and-ai-crawlers","label":"JavaScript and AI crawlers guide"},"gain":0},{"id":"robots_file","cat":"access","name":"robots.txt is valid and reachable","weight":2,"status":"pass","score":100,"evidence":"Found at https://lab2.crawlcheck.io/robots.txt.","fix":"Serve /robots.txt as plain text with a 200 status.","tool":{"href":"/tools/robots-txt-generator","label":"robots.txt generator"},"gain":0},{"id":"llms_txt","cat":"access","name":"llms.txt published","weight":1,"status":"pass","score":100,"evidence":"Found (Found).","fix":"Publish /llms.txt with a one-paragraph summary and links to your key pages.","tool":{"href":"/tools/llms-txt-generator","label":"llms.txt generator"},"gain":0},{"id":"https","cat":"technical","name":"Served over HTTPS","weight":4,"status":"pass","score":100,"evidence":"The page is served over HTTPS.","fix":"Install a TLS certificate and serve every page over HTTPS.","gain":0},{"id":"http_redirect","cat":"technical","name":"HTTP redirects to HTTPS","weight":2,"status":"na","score":0,"evidence":"Not tested in the quick scan.","fix":"Add a permanent 301 redirect from http:// to https:// for every URL.","gain":0},{"id":"title","cat":"technical","name":"Title tag present and 15 to 65 characters","weight":4,"status":"pass","score":100,"evidence":"\"Northfield Gutter Co. | Seamless gutters in Northfield\" (54 characters).","fix":"Write a unique title of about 50 to 60 characters that states the page topic first.","gain":0},{"id":"meta_description","cat":"technical","name":"Meta description 70 to 170 characters","weight":3,"status":"pass","score":100,"evidence":"125 characters.","fix":"Write a 140 to 160 character description that answers the main question the page covers.","tool":{"href":"/blog/ai-overviews-optimization-guide","label":"AI Overviews guide"},"gain":0},{"id":"canonical","cat":"technical","name":"Canonical tag points to this URL","weight":3,"status":"pass","score":100,"evidence":"Self-referencing: https://lab2.crawlcheck.io/.","fix":"Add <link rel=\"canonical\"> with this page's own absolute URL.","gain":0},{"id":"sitemap","cat":"technical","name":"XML sitemap found","weight":3,"status":"warn","score":80,"evidence":"Sitemap index with 2 child sitemaps at https://lab2.crawlcheck.io/sitemap.xml. Not declared in robots.txt.","fix":"Publish an XML sitemap and reference it with a Sitemap: line in robots.txt.","tool":{"href":"/blog/sitemap-xml-for-ai-discoverability","label":"sitemap guide"},"gain":0.5},{"id":"lang","cat":"technical","name":"Language declared","weight":2,"status":"pass","score":100,"evidence":"lang=\"en\".","fix":"Add lang to the <html> element, for example lang=\"en\".","gain":0},{"id":"hreflang","cat":"technical","name":"hreflang for alternate languages","weight":1,"status":"na","score":0,"evidence":"No hreflang tags. Not needed for single-language sites.","fix":"Add hreflang links for each language version, including x-default.","gain":0},{"id":"viewport","cat":"technical","name":"Mobile viewport set","weight":2,"status":"pass","score":100,"evidence":"Viewport meta tag present.","fix":"Add <meta name=\"viewport\" content=\"width=device-width, initial-scale=1\">.","gain":0},{"id":"response_time","cat":"technical","name":"Server responds quickly","weight":2,"status":"pass","score":100,"evidence":"46 ms to first byte (measured from our server).","fix":"Cache HTML at the edge or reduce server work before the first byte.","gain":0},{"id":"page_weight","cat":"technical","name":"HTML size under 1 MB","weight":1,"status":"pass","score":100,"evidence":"4 KB of HTML.","fix":"Move inline data and scripts out of the HTML document.","gain":0},{"id":"h1","cat":"content","name":"Exactly one H1","weight":3,"status":"pass","score":100,"evidence":"\"Seamless gutters, fitted right the first time\".","fix":"Use one H1 that matches the page topic; turn other H1s into H2s.","gain":0},{"id":"subheadings","cat":"content","name":"Sections under descriptive H2s","weight":3,"status":"warn","score":50,"evidence":"Only 1 H2.","fix":"Break the content into sections of 150 to 300 words, each under an H2 that names its topic.","gain":1.3},{"id":"heading_order","cat":"content","name":"Heading levels in order","weight":1,"status":"pass","score":100,"evidence":"No skipped heading levels.","fix":"Do not skip heading levels.","gain":0},{"id":"content_depth","cat":"content","name":"Enough main content to cite","weight":4,"status":"fail","score":0,"evidence":"145 words of main content.","fix":"Expand the main content with the details, examples and data a reader would ask for next.","gain":3.3},{"id":"question_headings","cat":"content","name":"Questions answered in headings","weight":3,"status":"fail","score":0,"evidence":"No headings phrased as questions.","fix":"Phrase two or more headings as the questions your audience asks, and answer each in the first sentence below it.","tool":{"href":"/blog/answer-first-content-structure","label":"answer-first structure guide"},"gain":2.5},{"id":"lists_tables","cat":"content","name":"Lists or tables for scannable facts","weight":2,"status":"warn","score":50,"evidence":"1 list, 0 tables.","fix":"Put steps in ordered lists and comparisons in tables with header cells.","gain":0.8},{"id":"readability","cat":"content","name":"Sentences and paragraphs are short","weight":2,"status":"pass","score":100,"evidence":"Average sentence 16 words; no paragraph over 150 words.","fix":"Aim for sentences under 22 words and paragraphs under 120 words.","gain":0},{"id":"semantic_html","cat":"content","name":"Main content marked with <main> or <article>","weight":2,"status":"pass","score":100,"evidence":"Landmarks: header, nav, main, footer.","fix":"Wrap the primary content in <main>, and posts in <article>.","gain":0},{"id":"image_alt","cat":"content","name":"Images have alt attributes","weight":2,"status":"fail","score":0,"evidence":"0 of 3 images have alt.","fix":"Add descriptive alt text to content images and alt=\"\" to decorative ones.","gain":1.7},{"id":"internal_links","cat":"content","name":"Internal links to related pages","weight":2,"status":"pass","score":100,"evidence":"23 internal links.","fix":"Link to the related guides, products and hub pages with descriptive anchor text.","gain":0},{"id":"schema_present","cat":"structured","name":"Valid JSON-LD structured data","weight":4,"status":"fail","score":0,"evidence":"1 JSON-LD block, none of them valid JSON.","fix":"Add JSON-LD for the page type (Organization, Article, Product, FAQPage) in the <head>.","tool":{"href":"/tools/schema-generator","label":"schema markup generator"},"gain":4.6},{"id":"schema_entity","cat":"structured","name":"Organization or Person entity","weight":3,"status":"fail","score":0,"evidence":"No Organization, LocalBusiness or Person node.","fix":"Add an Organization node with name, url, logo and sameAs links to your official profiles.","tool":{"href":"/tools/schema-generator","label":"schema markup generator"},"gain":3.5},{"id":"schema_content","cat":"structured","name":"Content-type schema (Article, Product, FAQ...)","weight":3,"status":"warn","score":30,"evidence":"Only site-level types (WebSite, WebPage, BreadcrumbList) or none.","fix":"Add the schema type that matches the page: Article for posts, Product for products, FAQPage for visible Q&A.","tool":{"href":"/tools/schema-generator","label":"schema markup generator"},"gain":2.4},{"id":"schema_dates","cat":"structured","name":"Dates in schema (published and modified)","weight":2,"status":"na","score":0,"evidence":"Not an article page, so publish dates are optional.","fix":"Add datePublished and dateModified to the Article or WebPage node, and update dateModified when content changes.","gain":0},{"id":"breadcrumbs","cat":"structured","name":"BreadcrumbList schema","weight":1,"status":"warn","score":30,"evidence":"No BreadcrumbList.","fix":"Add BreadcrumbList JSON-LD that mirrors the visible breadcrumb trail.","gain":0.8},{"id":"open_graph","cat":"structured","name":"Open Graph title, description and image","weight":2,"status":"fail","score":0,"evidence":"No Open Graph tags.","fix":"Add og:title, og:description and og:image.","gain":2.3},{"id":"author","cat":"trust","name":"Author shown or declared","weight":3,"status":"na","score":0,"evidence":"Not an article page, so a byline is not expected. Articles on the site should carry one.","fix":"Show a byline linked to an author page, and add author to the Article schema.","tool":{"href":"/blog/author-bylines-credentials-ai-trust","label":"author bylines guide"},"gain":0},{"id":"visible_date","cat":"trust","name":"Visible publish or update date","weight":2,"status":"na","score":0,"evidence":"Not an article page, so a visible date is optional.","fix":"Show the publish and last-updated dates in a <time datetime> element.","gain":0},{"id":"outbound_sources","cat":"trust","name":"Links to supporting sources","weight":2,"status":"fail","score":0,"evidence":"No links to external sources.","fix":"Link to the primary sources behind your data and claims.","gain":6},{"id":"about_contact","cat":"trust","name":"About or contact page linked","weight":2,"status":"pass","score":100,"evidence":"Link to an about or contact page found.","fix":"Link to your About and Contact pages from the header or footer.","gain":0},{"id":"mixed_content","cat":"trust","name":"No insecure (http) resources","weight":1,"status":"pass","score":100,"evidence":"All resources load over HTTPS.","fix":"Load every image, script and stylesheet over HTTPS.","gain":0}],"page":{"title":"Northfield Gutter Co. | Seamless gutters in Northfield","metaDescription":"Seamless gutters, gutter guards and cleaning in Northfield and seven nearby towns. Free fixed quotes, ten-year leak warranty.","lang":"en","canonical":"https://lab2.crawlcheck.io/","wordCount":185,"mainWordCount":145,"h1":"Seamless gutters, fitted right the first time","h2":1,"h3":0,"images":3,"imagesWithAlt":0,"internalLinks":23,"externalLinks":0,"schemaTypes":[]},"fullAudit":"https://aicrawlercheck.com/geo-audit?url=https%3A%2F%2Flab2.crawlcheck.io"}}