{"kind":"crawlcheck-resolve","v":1,"schema":"https://crawlcheck.io/schemas/resolve.json","domain":"scrapingbee.com","answered_at":"2026-10-19T02:26:13.101Z","reading":"Each section is declared (what the site says about itself), observed (what CrawlCheck was served) or not_measured, with when it was observed. There is no overall score: decide on the section your action depends on.","status":"measured","subject":{"domain":"scrapingbee.com","host":"www.scrapingbee.com","path":"/","report_id":"d0s3d4e8hi","scanned_at":"2026-10-17T21:09:03.098Z","score_version":32},"freshness":{"observed_at":"2026-10-17T21:09:03.098Z","age_hours":29.3,"latest_report_id":"d0s3d4e8hi","is_latest":true,"newer_scan_exists":false,"last_second_reading_at":null,"note":"observed_at is when CrawlCheck fetched the site; nothing here says the site is unchanged since","last_verified_at":"2026-10-17T21:09:03.098Z","fresh_until":"2026-10-24T21:09:03.098Z","stale":false,"fresh_policy":"an answer is fresh for 168 hours after observed_at; each action below has its own maximum age. last_verified_at is the latest reading of this domain by any enrolled observer."},"observers":{"primary":{"version_tag":"236ea88378fe6658","vantage":{"kind":"cloudflare-worker","colo":null}},"independent_readings":0,"by_field":"https://crawlcheck.io/api/corroboration?domain=scrapingbee.com","directory":"https://crawlcheck.io/.well-known/crawlcheck-observers.json","samples_by_network":{"networks":0,"samples":0,"groups":[],"window":"the latest 12 independent readings kept for this domain; the primary scan (Cloudflare) is not counted here"}},"crawl_policy":{"state":"declared","observed_at":"2026-10-17T21:09:03.098Z","source":"https://www.scrapingbee.com/robots.txt","http_status":200,"path_tested":"/","agents":[{"agent":"Googlebot-Image","label":"Google image crawler","role":"search","allowed_homepage":true,"allowed_path":true,"has_own_group":false,"wildcard_rules_shadowed":false},{"agent":"Googlebot-News","label":"Google News","role":"search","allowed_homepage":true,"allowed_path":true,"has_own_group":false,"wildcard_rules_shadowed":false},{"agent":"Googlebot-Video","label":"Google video crawler","role":"search","allowed_homepage":true,"allowed_path":true,"has_own_group":false,"wildcard_rules_shadowed":false},{"agent":"Storebot-Google","label":"Google Shopping","role":"search","allowed_homepage":true,"allowed_path":true,"has_own_group":false,"wildcard_rules_shadowed":false},{"agent":"Mediapartners-Google","label":"Google AdSense","role":"search","allowed_homepage":true,"allowed_path":true,"has_own_group":false,"wildcard_rules_shadowed":false},{"agent":"APIs-Google","label":"Google push delivery","role":"search","allowed_homepage":true,"allowed_path":true,"has_own_group":false,"wildcard_rules_shadowed":false},{"agent":"BingPreview","label":"Bing page preview","role":"search","allowed_homepage":true,"allowed_path":true,"has_own_group":false,"wildcard_rules_shadowed":false},{"agent":"msnbot","label":"Microsoft legacy crawler","role":"search","allowed_homepage":true,"allowed_path":true,"has_own_group":false,"wildcard_rules_shadowed":false},{"agent":"Slurp","label":"Yahoo Slurp","role":"search","allowed_homepage":true,"allowed_path":true,"has_own_group":false,"wildcard_rules_shadowed":false},{"agent":"Yandex","label":"Yandex (umbrella token)","role":"search","allowed_homepage":true,"allowed_path":true,"has_own_group":false,"wildcard_rules_shadowed":false},{"agent":"coccocbot-web","label":"Coc Coc","role":"search","allowed_homepage":true,"allowed_path":true,"has_own_group":false,"wildcard_rules_shadowed":false},{"agent":"Facebot","label":"Meta link crawler","role":"social","allowed_homepage":true,"allowed_path":true,"has_own_group":false,"wildcard_rules_shadowed":false},{"agent":"Magpie-crawler","label":"Magpie AI","role":"train","allowed_homepage":true,"allowed_path":true,"has_own_group":false,"wildcard_rules_shadowed":false},{"agent":"img2dataset","label":"img2dataset image corpus","role":"train","allowed_homepage":true,"allowed_path":true,"has_own_group":false,"wildcard_rules_shadowed":false},{"agent":"AwarioRssBot","label":"Awario RSS","role":"train","allowed_homepage":true,"allowed_path":true,"has_own_group":false,"wildcard_rules_shadowed":false},{"agent":"AwarioSmartBot","label":"Awario smart","role":"train","allowed_homepage":true,"allowed_path":true,"has_own_group":false,"wildcard_rules_shadowed":false},{"agent":"TurnitinBot","label":"Turnitin","role":"train","allowed_homepage":true,"allowed_path":true,"has_own_group":false,"wildcard_rules_shadowed":false},{"agent":"archive.org_bot","label":"Internet Archive","role":"train","allowed_homepage":true,"allowed_path":true,"has_own_group":false,"wildcard_rules_shadowed":false},{"agent":"ia_archiver","label":"Internet Archive (legacy)","role":"train","allowed_homepage":true,"allowed_path":true,"has_own_group":false,"wildcard_rules_shadowed":false},{"agent":"AhrefsBot","label":"Ahrefs link index","role":"tool","allowed_homepage":true,"allowed_path":true,"has_own_group":false,"wildcard_rules_shadowed":false},{"agent":"SemrushBot","label":"Semrush crawler","role":"tool","allowed_homepage":true,"allowed_path":true,"has_own_group":false,"wildcard_rules_shadowed":false},{"agent":"MJ12bot","label":"Majestic link index","role":"tool","allowed_homepage":true,"allowed_path":true,"has_own_group":false,"wildcard_rules_shadowed":false},{"agent":"DotBot","label":"Moz link index","role":"tool","allowed_homepage":true,"allowed_path":true,"has_own_group":false,"wildcard_rules_shadowed":false},{"agent":"rogerbot","label":"Moz site crawler","role":"tool","allowed_homepage":true,"allowed_path":true,"has_own_group":false,"wildcard_rules_shadowed":false},{"agent":"DataForSeoBot","label":"DataForSEO","role":"tool","allowed_homepage":true,"allowed_path":true,"has_own_group":false,"wildcard_rules_shadowed":false},{"agent":"BLEXBot","label":"WebMeUp link index","role":"tool","allowed_homepage":true,"allowed_path":true,"has_own_group":false,"wildcard_rules_shadowed":false},{"agent":"Yeti","label":"Naver","role":"regional","allowed_homepage":true,"allowed_path":true,"has_own_group":false,"wildcard_rules_shadowed":false},{"agent":"YoudaoBot","label":"Youdao","role":"regional","allowed_homepage":true,"allowed_path":true,"has_own_group":false,"wildcard_rules_shadowed":false},{"agent":"Exabot","label":"Exalead","role":"regional","allowed_homepage":true,"allowed_path":true,"has_own_group":false,"wildcard_rules_shadowed":false},{"agent":"Sogou web spider","label":"Sogou","role":"regional","allowed_homepage":true,"allowed_path":true,"has_own_group":false,"wildcard_rules_shadowed":false},{"agent":"YisouSpider","label":"Yisou","role":"regional","allowed_homepage":true,"allowed_path":true,"has_own_group":false,"wildcard_rules_shadowed":false},{"agent":"360Spider","label":"360 Search","role":"regional","allowed_homepage":true,"allowed_path":true,"has_own_group":false,"wildcard_rules_shadowed":false},{"agent":"Sosospider","label":"Soso","role":"regional","allowed_homepage":true,"allowed_path":true,"has_own_group":false,"wildcard_rules_shadowed":false},{"agent":"meta-webindexer","label":"Meta web index","role":"train","allowed_homepage":true,"allowed_path":true,"has_own_group":false,"wildcard_rules_shadowed":false},{"agent":"omgili","label":"Webz.io omgili","role":"train","allowed_homepage":true,"allowed_path":true,"has_own_group":false,"wildcard_rules_shadowed":false},{"agent":"cohere-training-data-crawler","label":"Cohere training corpus","role":"train","allowed_homepage":true,"allowed_path":true,"has_own_group":false,"wildcard_rules_shadowed":false},{"agent":"PanguBot","label":"Huawei PanGu","role":"train","allowed_homepage":true,"allowed_path":true,"has_own_group":false,"wildcard_rules_shadowed":false},{"agent":"Ai2Bot-Dolma","label":"Allen Institute Dolma","role":"train","allowed_homepage":true,"allowed_path":true,"has_own_group":false,"wildcard_rules_shadowed":false},{"agent":"FriendlyCrawler","label":"FriendlyCrawler ML","role":"train","allowed_homepage":true,"allowed_path":true,"has_own_group":false,"wildcard_rules_shadowed":false},{"agent":"VelenPublicWebCrawler","label":"Velen","role":"train","allowed_homepage":true,"allowed_path":true,"has_own_group":false,"wildcard_rules_shadowed":false},{"agent":"MyCentralAIScraperBot","label":"MyCentral AI","role":"train","allowed_homepage":true,"allowed_path":true,"has_own_group":false,"wildcard_rules_shadowed":false},{"agent":"DeepSeekBot","label":"DeepSeek","role":"train","allowed_homepage":true,"allowed_path":true,"has_own_group":false,"wildcard_rules_shadowed":false},{"agent":"ICC-Crawler","label":"NICT ICC","role":"train","allowed_homepage":true,"allowed_path":true,"has_own_group":false,"wildcard_rules_shadowed":false},{"agent":"GoogleOther","label":"Google non-search fetch","role":"train","allowed_homepage":true,"allowed_path":true,"has_own_group":false,"wildcard_rules_shadowed":false},{"agent":"Google-CloudVertexBot","label":"Vertex AI agent build","role":"train","allowed_homepage":true,"allowed_path":true,"has_own_group":false,"wildcard_rules_shadowed":false},{"agent":"CloudflareBrowserRenderingCrawler","label":"Cloudflare Browser Run /crawl","role":"tool","allowed_homepage":true,"allowed_path":true,"has_own_group":false,"wildcard_rules_shadowed":false},{"agent":"Cloudflare-AutoRAG","label":"Cloudflare AutoRAG","role":"tool","allowed_homepage":true,"allowed_path":true,"has_own_group":false,"wildcard_rules_shadowed":false},{"agent":"Peer39_Crawler","label":"Peer39 ad context","role":"tool","allowed_homepage":true,"allowed_path":true,"has_own_group":false,"wildcard_rules_shadowed":false},{"agent":"AdsBot-Google","label":"Google Ads quality","role":"tool","allowed_homepage":true,"allowed_path":true,"has_own_group":false,"wildcard_rules_shadowed":false},{"agent":"AmazonAdBot","label":"Amazon Ads","role":"tool","allowed_homepage":true,"allowed_path":true,"has_own_group":false,"wildcard_rules_shadowed":false},{"agent":"AdIdxBot","label":"Microsoft Ads","role":"tool","allowed_homepage":true,"allowed_path":true,"has_own_group":false,"wildcard_rules_shadowed":false},{"agent":"Twitterbot","label":"X link preview","role":"social","allowed_homepage":true,"allowed_path":true,"has_own_group":false,"wildcard_rules_shadowed":false},{"agent":"LinkedInBot","label":"LinkedIn link preview","role":"social","allowed_homepage":true,"allowed_path":true,"has_own_group":false,"wildcard_rules_shadowed":false},{"agent":"facebookexternalhit","label":"Facebook link preview","role":"social","allowed_homepage":true,"allowed_path":true,"has_own_group":false,"wildcard_rules_shadowed":false},{"agent":"Pinterestbot","label":"Pinterest","role":"social","allowed_homepage":true,"allowed_path":true,"has_own_group":false,"wildcard_rules_shadowed":false},{"agent":"Slackbot-LinkExpanding","label":"Slack unfurl","role":"social","allowed_homepage":true,"allowed_path":true,"has_own_group":false,"wildcard_rules_shadowed":false},{"agent":"OAI-SearchBot","label":"ChatGPT search index","role":"answer","allowed_homepage":true,"allowed_path":true,"has_own_group":false,"wildcard_rules_shadowed":false},{"agent":"ChatGPT-User","label":"ChatGPT live fetch","role":"answer","allowed_homepage":true,"allowed_path":true,"has_own_group":false,"wildcard_rules_shadowed":false},{"agent":"Claude-SearchBot","label":"Claude search index","role":"answer","allowed_homepage":true,"allowed_path":true,"has_own_group":false,"wildcard_rules_shadowed":false},{"agent":"Claude-User","label":"Claude live fetch","role":"answer","allowed_homepage":true,"allowed_path":true,"has_own_group":false,"wildcard_rules_shadowed":false},{"agent":"PerplexityBot","label":"Perplexity index","role":"answer","allowed_homepage":true,"allowed_path":true,"has_own_group":false,"wildcard_rules_shadowed":false},{"agent":"Perplexity-User","label":"Perplexity live fetch","role":"answer","allowed_homepage":true,"allowed_path":true,"has_own_group":false,"wildcard_rules_shadowed":false},{"agent":"Gemini-Deep-Research","label":"Gemini research agent","role":"answer","allowed_homepage":true,"allowed_path":true,"has_own_group":false,"wildcard_rules_shadowed":false},{"agent":"MistralAI-User","label":"Le Chat live fetch","role":"answer","allowed_homepage":true,"allowed_path":true,"has_own_group":false,"wildcard_rules_shadowed":false},{"agent":"DuckAssistBot","label":"DuckDuckGo AI assist","role":"answer","allowed_homepage":true,"allowed_path":true,"has_own_group":false,"wildcard_rules_shadowed":false},{"agent":"YouBot","label":"You.com","role":"answer","allowed_homepage":true,"allowed_path":true,"has_own_group":false,"wildcard_rules_shadowed":false},{"agent":"PhindBot","label":"Phind","role":"answer","allowed_homepage":true,"allowed_path":true,"has_own_group":false,"wildcard_rules_shadowed":false},{"agent":"Kagibot","label":"Kagi","role":"answer","allowed_homepage":true,"allowed_path":true,"has_own_group":false,"wildcard_rules_shadowed":false},{"agent":"Copilot-User","label":"Microsoft Copilot fetch","role":"answer","allowed_homepage":true,"allowed_path":true,"has_own_group":false,"wildcard_rules_shadowed":false},{"agent":"Meta-ExternalFetcher","label":"Meta AI live fetch","role":"answer","allowed_homepage":true,"allowed_path":true,"has_own_group":false,"wildcard_rules_shadowed":false},{"agent":"Googlebot","label":"Google Search and AI Overviews","role":"search","allowed_homepage":true,"allowed_path":true,"has_own_group":false,"wildcard_rules_shadowed":false},{"agent":"bingbot","label":"Bing and Copilot index","role":"search","allowed_homepage":true,"allowed_path":true,"has_own_group":false,"wildcard_rules_shadowed":false},{"agent":"Applebot","label":"Apple and Siri","role":"search","allowed_homepage":true,"allowed_path":true,"has_own_group":false,"wildcard_rules_shadowed":false},{"agent":"Amazonbot","label":"Amazon","role":"search","allowed_homepage":true,"allowed_path":true,"has_own_group":false,"wildcard_rules_shadowed":false},{"agent":"DuckDuckBot","label":"DuckDuckGo","role":"search","allowed_homepage":true,"allowed_path":true,"has_own_group":false,"wildcard_rules_shadowed":false},{"agent":"YandexBot","label":"Yandex","role":"search","allowed_homepage":true,"allowed_path":true,"has_own_group":false,"wildcard_rules_shadowed":false},{"agent":"Baiduspider","label":"Baidu","role":"search","allowed_homepage":true,"allowed_path":true,"has_own_group":false,"wildcard_rules_shadowed":false},{"agent":"Seznambot","label":"Seznam","role":"search","allowed_homepage":true,"allowed_path":true,"has_own_group":false,"wildcard_rules_shadowed":false},{"agent":"Neevabot","label":"Neeva","role":"search","allowed_homepage":true,"allowed_path":true,"has_own_group":false,"wildcard_rules_shadowed":false},{"agent":"PetalBot","label":"Huawei Petal","role":"search","allowed_homepage":true,"allowed_path":true,"has_own_group":false,"wildcard_rules_shadowed":false},{"agent":"GPTBot","label":"OpenAI training and index","role":"train","allowed_homepage":true,"allowed_path":true,"has_own_group":false,"wildcard_rules_shadowed":false},{"agent":"ClaudeBot","label":"Anthropic training","role":"train","allowed_homepage":true,"allowed_path":true,"has_own_group":false,"wildcard_rules_shadowed":false},{"agent":"anthropic-ai","label":"Anthropic legacy agent","role":"train","allowed_homepage":true,"allowed_path":true,"has_own_group":false,"wildcard_rules_shadowed":false},{"agent":"Claude-Web","label":"Anthropic legacy agent","role":"train","allowed_homepage":true,"allowed_path":true,"has_own_group":false,"wildcard_rules_shadowed":false},{"agent":"Google-Extended","label":"Gemini training","role":"train","allowed_homepage":true,"allowed_path":true,"has_own_group":false,"wildcard_rules_shadowed":false},{"agent":"Applebot-Extended","label":"Apple training","role":"train","allowed_homepage":true,"allowed_path":true,"has_own_group":false,"wildcard_rules_shadowed":false},{"agent":"CCBot","label":"Common Crawl","role":"train","allowed_homepage":true,"allowed_path":true,"has_own_group":false,"wildcard_rules_shadowed":false},{"agent":"Bytespider","label":"ByteDance","role":"train","allowed_homepage":true,"allowed_path":true,"has_own_group":false,"wildcard_rules_shadowed":false},{"agent":"meta-externalagent","label":"Meta AI training","role":"train","allowed_homepage":true,"allowed_path":true,"has_own_group":false,"wildcard_rules_shadowed":false},{"agent":"FacebookBot","label":"Meta legacy","role":"train","allowed_homepage":true,"allowed_path":true,"has_own_group":false,"wildcard_rules_shadowed":false},{"agent":"cohere-ai","label":"Cohere","role":"train","allowed_homepage":true,"allowed_path":true,"has_own_group":false,"wildcard_rules_shadowed":false},{"agent":"Diffbot","label":"Diffbot knowledge graph","role":"train","allowed_homepage":true,"allowed_path":true,"has_own_group":false,"wildcard_rules_shadowed":false},{"agent":"Omgilibot","label":"Webz.io","role":"train","allowed_homepage":true,"allowed_path":true,"has_own_group":false,"wildcard_rules_shadowed":false},{"agent":"ImagesiftBot","label":"Imagesift","role":"train","allowed_homepage":true,"allowed_path":true,"has_own_group":false,"wildcard_rules_shadowed":false},{"agent":"Timpibot","label":"Timpi","role":"train","allowed_homepage":true,"allowed_path":true,"has_own_group":false,"wildcard_rules_shadowed":false},{"agent":"AI2Bot","label":"Allen Institute","role":"train","allowed_homepage":true,"allowed_path":true,"has_own_group":false,"wildcard_rules_shadowed":false},{"agent":"Scrapy","label":"Generic scraper framework","role":"train","allowed_homepage":true,"allowed_path":true,"has_own_group":false,"wildcard_rules_shadowed":false},{"agent":"SemrushBot-OCOB","label":"Semrush AI corpus","role":"train","allowed_homepage":true,"allowed_path":true,"has_own_group":false,"wildcard_rules_shadowed":false},{"agent":"Applebot-Extended-Ads","label":"Apple ads corpus","role":"train","allowed_homepage":true,"allowed_path":true,"has_own_group":false,"wildcard_rules_shadowed":false},{"agent":"TikTokSpider","label":"TikTok","role":"train","allowed_homepage":true,"allowed_path":true,"has_own_group":false,"wildcard_rules_shadowed":false},{"agent":"QuillBot","label":"QuillBot","role":"train","allowed_homepage":true,"allowed_path":true,"has_own_group":false,"wildcard_rules_shadowed":false},{"agent":"Webzio-Extended","label":"Webz.io extended","role":"train","allowed_homepage":true,"allowed_path":true,"has_own_group":false,"wildcard_rules_shadowed":false},{"agent":"ProRataInc","label":"ProRata","role":"train","allowed_homepage":true,"allowed_path":true,"has_own_group":false,"wildcard_rules_shadowed":false},{"agent":"AwarioBot","label":"Awario","role":"train","allowed_homepage":true,"allowed_path":true,"has_own_group":false,"wildcard_rules_shadowed":false},{"agent":"Amzn-SearchBot","label":"Amazon search eligibility, separate from Amazonbot","role":"search","allowed_homepage":true,"allowed_path":true,"has_own_group":false,"wildcard_rules_shadowed":false},{"agent":"Amzn-User","label":"Amazon live fetch for Alexa questions","role":"answer","allowed_homepage":true,"allowed_path":true,"has_own_group":false,"wildcard_rules_shadowed":false},{"agent":"MistralAI-Index","label":"Mistral search index for Vibe","role":"search","allowed_homepage":true,"allowed_path":true,"has_own_group":false,"wildcard_rules_shadowed":false},{"agent":"MistralAI-Training","label":"Mistral training crawl","role":"train","allowed_homepage":true,"allowed_path":true,"has_own_group":false,"wildcard_rules_shadowed":false},{"agent":"OAI-AdsBot","label":"OpenAI ChatGPT ads page validation","role":"tool","allowed_homepage":true,"allowed_path":true,"has_own_group":false,"wildcard_rules_shadowed":false},{"agent":"Google-Agent","label":"Google user-triggered agent, acts on the web","role":"answer","allowed_homepage":true,"allowed_path":true,"has_own_group":false,"wildcard_rules_shadowed":false},{"agent":"Google-GeminiNotebook","label":"Gemini Notebook user-supplied URL fetch","role":"answer","allowed_homepage":true,"allowed_path":true,"has_own_group":false,"wildcard_rules_shadowed":false},{"agent":"GoogleOther-Image","label":"Google common crawler, public image URLs","role":"train","allowed_homepage":true,"allowed_path":true,"has_own_group":false,"wildcard_rules_shadowed":false},{"agent":"GoogleOther-Video","label":"Google common crawler, public video URLs","role":"train","allowed_homepage":true,"allowed_path":true,"has_own_group":false,"wildcard_rules_shadowed":false},{"agent":"ExaSearchBot","label":"Exa AI search index, Web Bot Auth signed","role":"search","allowed_homepage":true,"allowed_path":true,"has_own_group":false,"wildcard_rules_shadowed":false}],"signal_conflicts":[],"note":"robots.txt governs access and Content-Signal governs use; both are what the site declares, not what it serves"},"delivery":{"state":"observed","observed_at":"2026-10-17T21:09:03.098Z","url":"https://www.scrapingbee.com/","identity_basis":"every identity was sent by CrawlCheck from its own network; none is the named operator's own crawler","identities":[{"identity":"anon","label":"Unnamed client","role":"control","status":200,"words":2986,"blocker":null,"redirected_to":null,"x_robots_tag":null},{"identity":"gptbot","label":"GPTBot","role":"train","status":200,"words":2986,"blocker":null,"redirected_to":null,"x_robots_tag":null},{"identity":"claudebot","label":"ClaudeBot","role":"train","status":200,"words":2986,"blocker":null,"redirected_to":null,"x_robots_tag":null},{"identity":"oaisearch","label":"OAI-SearchBot","role":"answer","status":200,"words":2986,"blocker":null,"redirected_to":null,"x_robots_tag":null},{"identity":"claudesearch","label":"Claude-SearchBot","role":"answer","status":200,"words":2986,"blocker":null,"redirected_to":null,"x_robots_tag":null},{"identity":"perplexity","label":"PerplexityBot","role":"answer","status":200,"words":2986,"blocker":null,"redirected_to":null,"x_robots_tag":null},{"identity":"googlebot","label":"Googlebot","role":"search","status":200,"words":2986,"blocker":null,"redirected_to":null,"x_robots_tag":null},{"identity":"browser","label":"Mobile browser","role":"browser","status":200,"words":2986,"blocker":null,"redirected_to":null,"x_robots_tag":null},{"identity":"bingbot","label":"Bingbot","role":"search","status":200,"words":2986,"blocker":null,"redirected_to":null,"x_robots_tag":null},{"identity":"applebot","label":"Applebot","role":"search","status":200,"words":2986,"blocker":null,"redirected_to":null,"x_robots_tag":null},{"identity":"amazonbot","label":"Amazonbot","role":"train","status":200,"words":2986,"blocker":null,"redirected_to":null,"x_robots_tag":null},{"identity":"bytespider","label":"Bytespider","role":"train","status":200,"words":2986,"blocker":null,"redirected_to":null,"x_robots_tag":null},{"identity":"metaexternal","label":"Meta-ExternalAgent","role":"train","status":200,"words":2986,"blocker":null,"redirected_to":null,"x_robots_tag":null},{"identity":"ccbot","label":"CCBot","role":"train","status":200,"words":2986,"blocker":null,"redirected_to":null,"x_robots_tag":null}]},"machine_files":{"state":"observed","observed_at":"2026-10-17T21:09:03.098Z","files":[{"path":"/robots.txt","status":200,"content_type":"text/plain; charset=utf-8","bytes":72,"verdict":"ok"},{"path":"/sitemap.xml","status":200,"content_type":"application/xml","bytes":155181,"verdict":"ok"},{"path":"/sitemap_index.xml","status":404,"content_type":"text/html; charset=utf-8","bytes":198604,"verdict":"absent"},{"path":"/wp-sitemap.xml","status":404,"content_type":"text/html; charset=utf-8","bytes":198604,"verdict":"absent"},{"path":"/llms.txt","status":200,"content_type":"text/plain; charset=utf-8","bytes":6204,"verdict":"ok"}],"read_before_page":[{"path":"/sitemap.xml","bytes":155181,"kind":"machine"},{"path":"/llms.txt","bytes":6204,"kind":"machine"},{"path":"/robots.txt","bytes":72,"kind":"machine"}],"hashes":"every fetched body's SHA-256 is in the sealed manifest: https://crawlcheck.io/api/artifact/b29a7ddbdcdd75ab2edab1b0e1e7018c0ae598029c6a886d8e968052b70cf523?body=1"},"capabilities":{"state":"observed","observed_at":"2026-10-17T21:09:03.098Z","checked":[{"kind":"a2a_agent_card","url":"https://www.scrapingbee.com/.well-known/agent-card.json","status":404}],"declared":0,"by_safety_class":{},"verified":0,"mismatches":0,"invocation_rule":"capabilities are read from declarations only; nothing is invoked. Later verification may call read_only_public operations alone; every other class needs the site owner's authorisation and a test account, and unknown is never called","certificate":{"id":"ccap1:92a2a7a237c5c83d23da9e3c3c52c6cd67596d0881a1fc318ba165b497042a2a","signed":true,"entry":"https://crawlcheck.io/api/registry/capabilities?domain=scrapingbee.com","verify":"https://crawlcheck.io/api/registry/capabilities/verify?domain=scrapingbee.com"},"items":"https://crawlcheck.io/api/capabilities?domain=scrapingbee.com"},"entity":{"state":"observed","observed_at":"2026-10-17T21:09:03.098Z","name":"ScrapingBee","wikidata_item_found":true,"links":[],"states":"corroborated = the linked page links back; unreciprocated = readable and does not; unverifiable = could not be read from here"},"mcp_registry":{"state":"observed","listed":1,"observed":1,"initialized":1,"auth_required":0,"drifted":0,"shadowing":{"flagged":0,"servers":[],"shadowed_by":0,"shadowed_by_servers":[]},"auth_audit":null,"last_observed_at":"2026-10-17T12:47:13.612Z","servers":[{"name":"com.scrapingbee/mcp","version":"2.2.1","endpoint":"https://mcp.scrapingbee.com/mcp","outcome":"initialized","tools":18}],"url":"https://crawlcheck.io/api/mcp/servers?host=scrapingbee.com","reading":"listed by the registry (the publisher’s claim); outcome is what the endpoint answered to the MCP handshake"},"content_clock":{"homepage":{"url":"https://www.scrapingbee.com/","observed":true,"content_sha256":"b3d24aa88e827f7e695d70602dc0ffcc62e79019a13bc3a24572f478c85cdbaf","first_observed":"2026-10-10T20:56:20.736Z","last_read":"2026-10-15T20:28:42.511Z","changes_observed":1,"main_content_last_changed":{"between":"2026-10-10T20:56:20.736Z","and":"2026-10-15T20:28:42.511Z"},"unchanged_since_first_observed":null,"declared":{},"date_moved_words_did_not":null,"api":"https://crawlcheck.io/api/v1/content-clock?url=https%3A%2F%2Fscrapingbee.com%2F"},"reading":"When the main content (not the bytes) of these URLs last changed, from CrawlCheck's own reads. Full history and the page's declared dates: the api link."},"self_claims":{"observed":true,"first_read":"2026-10-10T07:29:41.370Z","last_read":"2026-10-15T20:28:36.801Z","reads":5,"claims":[{"type":"address","value":"66 Avenue des Champs Elysées, OCB Business Center 4, Paris, France, F-75008","status":"current","first_seen":"2026-10-10T07:29:41.370Z","last_seen":"2026-10-15T20:28:36.801Z"},{"type":"price","value":"$49/month","status":"current","first_seen":"2026-10-10T07:29:41.370Z","last_seen":"2026-10-15T20:28:36.801Z"},{"type":"price","value":"$99/month","status":"current","first_seen":"2026-10-10T07:29:41.370Z","last_seen":"2026-10-15T20:28:36.801Z"},{"type":"price","value":"$249/month","status":"current","first_seen":"2026-10-10T07:29:41.370Z","last_seen":"2026-10-15T20:28:36.801Z"},{"type":"price","value":"$599/month","status":"current","first_seen":"2026-10-10T07:29:41.370Z","last_seen":"2026-10-15T20:28:36.801Z"},{"type":"price","value":"$999/month","status":"current","first_seen":"2026-10-10T07:29:41.370Z","last_seen":"2026-10-15T20:28:36.801Z"},{"type":"price","value":"$19/month","status":"current","first_seen":"2026-10-14T15:20:05.760Z","last_seen":"2026-10-15T20:28:36.801Z"}],"api":"https://crawlcheck.io/api/v1/claims?domain=scrapingbee.com","reading":"what the site says about itself on its own pages, with when CrawlCheck first and last saw each value; not a statement that the claim is true"},"read_equivalence":{"url":"https://scrapingbee.com/","observed":true,"verdict":"equivalent","checked_at":"2026-10-15T20:28:37.022Z","first_observed":"2026-10-10T17:13:06.646Z","ai_identities_served":"3 of 3","identities":[{"identity":"gptbot","verdict":"equivalent","text_ratio":1,"instruction_like_text":false},{"identity":"claudebot","verdict":"equivalent","text_ratio":1,"instruction_like_text":false},{"identity":"perplexity","verdict":"equivalent","text_ratio":1,"instruction_like_text":false}],"api":"https://crawlcheck.io/api/v1/read-equivalence?url=https%3A%2F%2Fscrapingbee.com%2F","reading":"whether the main text served to AI identities is the same text a person sees in a browser, from one simultaneous read of the homepage; evidence and the differing sentences at the api link"},"rights":{"observed":true,"path":"/","read_at":"2026-10-17T18:38:52.953Z","api":"https://crawlcheck.io/api/v1/rights?domain=scrapingbee.com","reading":"one value per use for the homepage, with the signal that decided it; signals considered, conflicts and per-path answers at the api link","train":{"value":"not_declared","decided_by":{"signal":"robots.txt","url":"https://scrapingbee.com/robots.txt","evidence":"access is open to GPTBot, ClaudeBot, Google-Extended, CCBot, Applebot-Extended, meta-externalagent on / and no signal states whether train is allowed"},"conflicts":0},"ground":{"value":"not_declared","decided_by":{"signal":"robots.txt","url":"https://scrapingbee.com/robots.txt","evidence":"access is open to OAI-SearchBot, ChatGPT-User, Claude-SearchBot, Claude-User, PerplexityBot, Perplexity-User on / and no signal states whether ground is allowed"},"conflicts":0},"summarize":{"value":"not_declared","decided_by":{"signal":"robots.txt","url":"https://scrapingbee.com/robots.txt","evidence":"access is open to OAI-SearchBot, ChatGPT-User, Claude-SearchBot, Claude-User, PerplexityBot, Perplexity-User on / and no signal states whether summarize is allowed"},"conflicts":0},"cite":{"value":"not_declared","decided_by":{"signal":"robots.txt","url":"https://scrapingbee.com/robots.txt","evidence":"access is open to OAI-SearchBot, Claude-SearchBot, PerplexityBot, Googlebot, Bingbot on / and no signal states whether cite is allowed"},"conflicts":0},"transact":{"value":"not_declared","decided_by":{"signal":"robots.txt","url":"https://scrapingbee.com/robots.txt","evidence":"access is open to ChatGPT-User, Claude-User, Perplexity-User on / and no signal states whether transact is allowed"},"conflicts":0}},"findings":{"access":"public","rule":"the top finding in full; the rest by title, severity and path (a licence for this domain opens them)","items":[{"fid":"f1:bf7d64f49831811292c7","decision_id":"d1:f05c59aa8032e6964233","code":"PAGE_IS_MOSTLY_CODE","severity":3,"path":"/","title":"Almost nothing a machine receives from this page is readable text","rule_revision":1,"span_id":"4633bf8e51145c5d","state":"persisted","valid_time":{"valid_from":{"after":null,"by":"2026-10-04T21:22:45.063Z"},"valid_to":null,"recorded_at":"2026-10-04T21:22:53.556Z","superseded_at":null,"superseded_by":null,"earlier_intervals":0,"before_bt1":false},"observations_linked":1,"trust":{"freshness":{"observed_at":"2026-10-17T21:09:03.098Z","last_observed":"2026-10-17T21:09:03.098Z","held_since":"2026-10-04T21:22:53.556Z","state":"persisted"},"confidence":{"level":"reproduced","means":"seen again: repeated on CrawlCheck's own network, or in more than one scan","basis":["seen in 4 scans since 2026-10-04","the decision names the 1 fetch it read"],"sample":{"cross_network":"not_applicable","why":"this finding does not depend on the requesting identity or network, so other networks' readings neither confirm nor contradict it","readings":1,"diversity":{"readings":1,"observers":1,"distinct_networks":1,"networks":{"cloudflare":1},"distinct_countries":0,"countries":{}},"scans":{"seen":4,"could_have_seen":null,"interval":null,"why":"scans that could have seen this finding are counted from 2026-10-08; this finding was first seen before that, so only the scans that saw it are known"},"interval":null,"interval_of":null,"note":"interval: the 95% Wilson score interval for the share, from k of n; it is wide when n is small, and that width is the point. networks: autonomous systems the readings arrived from (two observers on one network are one network). countries: where the connection was measured; CrawlCheck's own reading leaves from a Cloudflare data centre and carries no country here."}},"contradictions":[]},"score_effect":{"decision_id":"d1:f05c59aa8032e6964233","code":"PAGE_IS_MOSTLY_CODE","path":"/","severity":3,"level":"medium","points":0,"cap":"C","binding":true,"letter_now":"C","letter_without_it":"B","same_level_others":0,"says":"Caps the letter at C. Without it the letter would be B (score 79). Moves no points; the score is built from the measured rows."},"root_cause":null,"corroboration":{"status":"not_applicable","reading":"this finding does not depend on the requesting identity or network"},"explain":{"mcp_tool":"explain_finding","arguments":{"id":"d1:f05c59aa8032e6964233"},"url":"https://crawlcheck.io/api/explain?id=d1:f05c59aa8032e6964233"},"disputes":[]},{"title":"HTTPS is served without a Strict-Transport-Security header","severity":1,"path":"/","locked":true},{"title":"A missing agent file answers with a full page","severity":1,"path":"/","locked":true},{"title":"A business record gives an address but no coordinates","severity":1,"path":"/","locked":true}],"summary":{"total":4,"serious":1,"by_severity":{"Medium":1,"Note":3}},"withheld":"Every finding is listed by title, severity and path. The top one carries its code, detail and evidence; the other 3 are locked (locked: true) and need a licence or the fix service: https://crawlcheck.io/pricing"},"confidence":{"identities_answered":{"n":14,"of":14,"ratio":1},"vantages":1,"vantage_state":{"state":"single_vantage","means":"only one network read this site near the scan; nothing here is cross-checked","networks":1,"readings":1,"countries":0,"findings_contested":0,"findings_cross_confirmed":0,"states":{"single_vantage":"only one network read this site near the scan; nothing here is cross-checked","insufficient_samples":"more than one network read it, but fewer than 3 readings: too few to state agreement","cross_confirmed":"two or more networks read it, 3 or more readings, and no finding is contested between them","network_divergence":"networks disagree on at least one finding","regional_divergence":"networks in more than one country disagree on at least one finding: what is served depends on where the request comes from"}},"note":"counts and ratios only - there is no composite confidence number, because the parts do not combine into one honestly","sample":{"readings":1,"observers":1,"distinct_networks":1,"networks":{"cloudflare":1},"distinct_countries":0,"countries":{},"window_hours":6,"all_time":{"independent_readings":0,"networks":[]},"reading":"readings of this domain within 6 hours of the scan: CrawlCheck's own plus every independent observer's, with the networks and countries they arrived from","note":"interval: the 95% Wilson score interval for the share, from k of n; it is wide when n is small, and that width is the point. networks: autonomous systems the readings arrived from (two observers on one network are one network). countries: where the connection was measured; CrawlCheck's own reading leaves from a Cloudflare data centre and carries no country here."}},"contradictions":{"site":[],"between_networks_open":[],"locked_findings_not_shown":3},"evidence":{"report":"https://crawlcheck.io/r/d0s3d4e8hi","machine_record":"https://crawlcheck.io/api/v1/machine-record?id=d0s3d4e8hi","manifest_sha256":"b29a7ddbdcdd75ab2edab1b0e1e7018c0ae598029c6a886d8e968052b70cf523","verify":"https://crawlcheck.io/api/verify?id=d0s3d4e8hi","bundle":"https://crawlcheck.io/api/bundle?id=d0s3d4e8hi","offline_verifier":"https://crawlcheck.io/crawlcheck-verify.mjs?v=b22547e95463d03e","verify_this_answer":"save it as answer.json, then: node crawlcheck-verify.mjs answer.json"},"impersonation":{"state":"no_lookalike","verdict":"no_lookalike","brand_frame":"Tranco top 10,000 (Y83YG)"},"actions":{"v":1,"decisions":"allow | warn | require_confirmation | block | unsupported","rule":"decided from this answer's own sections at signing time; an agent's own policy may be stricter, never looser","read":{"decision":"allow","minimum_assurance":"reachable","assurance_found":["declared","reachable"],"needs":"robots.txt lets AI agents read the path, and the page was served to at least one AI-crawler identity","why":"9 of 9 AI-crawler identities were served the page","doubts":[],"max_age_hours":168},"cite":{"decision":"allow","minimum_assurance":"reachable + entity observed","assurance_found":["declared","reachable","entity observed","read_equivalent"],"needs":"everything read needs, plus an observed entity and no contradiction the site makes with itself","scope":{"covers":["access: robots.txt and what was actually served to AI-crawler identities","permissions: the rights and content signals the site declares","identity: the entity the site declares and who has proven control of the domain","provenance: when, from where and by whom it was observed, signed and logged"],"does_not_cover":"whether the content is true, accurate, current or complete; a cite decision says the source may be cited and attributed, not that what it says is right"},"why":"9 of 9 AI-crawler identities were served the page","doubts":[],"max_age_hours":168},"connect":{"decision":"require_confirmation","minimum_assurance":"conformant","assurance_found":["listed","reachable"],"needs":"a declared capability that CrawlCheck invoked read-only and found to behave as declared","why":"the site declares no capability, but the official MCP Registry lists 1 MCP server(s) here and 1 completed the MCP handshake; tools were listed, not invoked, so a human confirms","doubts":[],"max_age_hours":72},"transact":{"decision":"require_confirmation","minimum_assurance":"action_ready","assurance_found":[],"needs":"owner-verified identity and an action-ready capability; CrawlCheck does not certify transactions, so a human confirms","why":"CrawlCheck does not certify transactions; a human confirms every financial, identity or irreversible action","doubts":[],"max_age_hours":24},"administer":{"decision":"block","minimum_assurance":"never automatic","assurance_found":[],"needs":"high-privilege operations are never authorised by a public record","why":"never authorised by a public record","doubts":[],"max_age_hours":0}},"absences":[{"what":"sitemap","path":"/sitemap_index.xml","result":"absent","at":"2026-10-17T21:09:03.098Z","from":{"observer":"crawlcheck-worker","network":"cloudflare","directory":"https://crawlcheck.io/.well-known/crawlcheck-observers.json"},"checked":[{"method":"GET","url":"https://scrapingbee.com/sitemap_index.xml","status":404,"redirects":"followed"}],"scope":"GET https://scrapingbee.com/sitemap_index.xml answered HTTP 404 at 2026-10-17T21:09:03.098Z. The file was not served at this path then; this says nothing about the same content at another path or a later time."},{"what":"sitemap","path":"/wp-sitemap.xml","result":"absent","at":"2026-10-17T21:09:03.098Z","from":{"observer":"crawlcheck-worker","network":"cloudflare","directory":"https://crawlcheck.io/.well-known/crawlcheck-observers.json"},"checked":[{"method":"GET","url":"https://scrapingbee.com/wp-sitemap.xml","status":404,"redirects":"followed"}],"scope":"GET https://scrapingbee.com/wp-sitemap.xml answered HTTP 404 at 2026-10-17T21:09:03.098Z. The file was not served at this path then; this says nothing about the same content at another path or a later time."},{"what":"a2a_agent_card","result":"absent","at":"2026-10-17T21:09:03.098Z","from":{"observer":"crawlcheck-worker","network":"cloudflare","directory":"https://crawlcheck.io/.well-known/crawlcheck-observers.json"},"checked":[{"method":"GET","url":"https://www.scrapingbee.com/.well-known/agent-card.json","status":404,"redirects":"followed"}],"scope":"Every location CrawlCheck checks for a a2a agent card answered not found (404). No declaration of this kind was served at those URLs at that time; one served elsewhere, or added later, is not covered."}],"control":{"state":"unproven","proofs":[],"organization":null,"claim":"https://crawlcheck.io/registry/scrapingbee.com/claim","why":"nobody has proven control of scrapingbee.com to CrawlCheck, so who stands behind the site is unconfirmed","note":"control says who answers for the domain, not whether it is safe; it does not change any decision in this answer"},"transparency":{"state":"included","log_id":"crawlcheck-resolve-log-v1","leaf_hash":"b0da9cac6886a064fc616cd1404c2e7dcb1867661e07720015e7690c5e58136f","batch":686,"index":13,"batch_size":17,"path":["ade1a4730047c7dc4ae7b300a169de40ba4af01b8a0f4767ef32082bd4744180","4ff252b6a6a618d80318452b6f9c3c47e2463f21084f38176ce75d44af11235a","fbf7171900784673843a6ecc7ff61f2c8647bd888b69327db64cd370fe97b0ea","940adcd69ea8cf90d48a191b8a11fbb84a4cc845703532a4100b794fd33c89db","f73ecbbf7ee1a2dc3114aa0c196e24da4332e69b5cdf91c20e5e423729cdf8c4"],"sth":{"kind":"crawlcheck-log-sth","v":1,"log_id":"crawlcheck-resolve-log-v1","batch":686,"first_index":144600,"batch_size":17,"tree_size":144617,"batch_root":"019ad738b4df9d5b66d5e760930e5ec60450c955389610c5d8690b05f720fe1f","prev_sth_sha256":"a80bd8a5eaf31ca054860845598c6aad270d280df6298c8adefd581752ec0105","created_at":"2026-10-17T21:32:36.599Z","rules":{"leaf":"SHA-256(0x00 || canonical JSON of the answer's decision content)","tree":"RFC 9162 Merkle tree hash over this batch's leaf hashes, sorted","chain":"prev_sth_sha256 is the sha256 of the previous batch's signed tree head; batch numbers are consecutive and tree_size grows by batch_size"},"code":{"version_id":"3eb13abe-8004-4e70-910d-f8bc5be9d9d7","script_sha16":"236ea88378fe6658","state":"attested","deployed":{"at":"2026-10-17T20:35:36.358Z","commit":"35b8b3e6d574","run":"39067307540"},"build_log_entry":526,"score_version":32,"scoring_policy_sha256":"d3c9c95d409dae4e90b8e868ff113e631e445e33c14a0cc008dda144293821b3","decision_rules_sha256":"18dd5636e0896692ab518050cb401744edea2e6f1086dff94fead314959bdc3e","check":"https://crawlcheck.io/api/v1/builds/3eb13abe-8004-4e70-910d-f8bc5be9d9d7"},"sha256":"45747003919795cb451dcd8f9d727197ed143da58867b02d3eca642aba8d4b3e","signature":{"v":1,"alg":"Ed25519","kid":"1hHmE3fo1rg5Ig70wuFtJ5wBc7G-sUTgUJkxyPWFRps","over":"sha256","message":"crawlcheck-data-v1\n45747003919795cb451dcd8f9d727197ed143da58867b02d3eca642aba8d4b3e","sig":"I1IECZ8K1e_K3fPut5lKyKjTiNZacpRSyXTNwvke1Gx8ebsYzqa1PNCysQ31XmJ977C-af_cKBFOPe34sIeoBQ","signed_at":"2026-10-17T21:32:37.146Z","key":{"kid":"1hHmE3fo1rg5Ig70wuFtJ5wBc7G-sUTgUJkxyPWFRps","jwk":{"kty":"OKP","crv":"Ed25519","x":"5TmKCOx0TTHepq7HwyXBHvGQpVCSBd2lV8cLzYVqFxQ"},"published_in":"https://crawlcheck.io/.well-known/http-message-signatures-directory"},"canonical":"sha256 over canonical JSON of every field except sha256 and signature"}},"witnesses":"https://crawlcheck.io/api/v1/log/sth/686","content_rule":"leaf = SHA-256(0x00 || canonical JSON of the answer's decision content); see https://crawlcheck.io/api/v1/log"},"assurance_ladder":{"v":1,"level":"reachable","order":["declared","discovered","reachable","conformant","owner_verified","authorized","reliable","action_ready"],"rungs":[{"rung":"declared","state":"reached","why":"robots.txt policy declared; 1 MCP server(s) in the registry"},{"rung":"discovered","state":"reached","why":"0 declared operation(s), 1 registry server(s)"},{"rung":"reachable","state":"reached","why":"AI-crawler identities were served the page; 1 MCP server(s) completed the handshake"},{"rung":"conformant","state":"not_measured","why":"no same-origin, login-free MCP server or A2A agent to run the read-only suite against, and no capability verified by invocation"},{"rung":"owner_verified","state":"not_reached","why":"nobody has proven control of scrapingbee.com to CrawlCheck, so who stands behind the site is unconfirmed"},{"rung":"authorized","state":"not_reached","why":"the domain's owner has published no authorization for agents"},{"rung":"reliable","state":"not_measured","why":"reliability is measured for MCP endpoints over time; none on this domain is on the panel"},{"rung":"action_ready","state":"not_reached","why":"not yet: conformant, owner_verified, authorized, reliable"}],"rule":"level is the highest rung with every rung under it reached; a rung is reached only on measured evidence, not_reached when measured and short, not_measured when there was nothing to measure. The ladder describes evidence; each action's decision is still made in actions"},"code":{"version_id":"6929574e-df21-45a5-8a92-5b8ba41cc9aa","script_sha16":"85c9b9b6c9a512ed","state":"attested","deployed":{"at":"2026-10-18T20:02:49.319Z","commit":"2bf491f55991","run":"39161431635"},"build_log_entry":597,"score_version":32,"scoring_policy_sha256":"d3c9c95d409dae4e90b8e868ff113e631e445e33c14a0cc008dda144293821b3","decision_rules_sha256":"18dd5636e0896692ab518050cb401744edea2e6f1086dff94fead314959bdc3e","check":"https://crawlcheck.io/api/v1/builds/6929574e-df21-45a5-8a92-5b8ba41cc9aa"},"sha256":"57640a8636a0dd74c7520efd4131c7ce48869c5a5af1eee81ae5b216cbcd91cf","signature":{"v":1,"alg":"Ed25519","kid":"1hHmE3fo1rg5Ig70wuFtJ5wBc7G-sUTgUJkxyPWFRps","over":"sha256","message":"crawlcheck-data-v1\n57640a8636a0dd74c7520efd4131c7ce48869c5a5af1eee81ae5b216cbcd91cf","sig":"7OttnFj1vL6e69Ibwj-W0CnspADFAFQI84YSLIKvZYJl_pgsBKqTZB8JCg6EhGyXgDgLlNhUz7Vx3P9F0NwzAw","signed_at":"2026-10-19T02:26:13.123Z","key":{"kid":"1hHmE3fo1rg5Ig70wuFtJ5wBc7G-sUTgUJkxyPWFRps","jwk":{"kty":"OKP","crv":"Ed25519","x":"5TmKCOx0TTHepq7HwyXBHvGQpVCSBd2lV8cLzYVqFxQ"},"published_in":"https://crawlcheck.io/.well-known/http-message-signatures-directory"},"canonical":"sha256 over canonical JSON of every field except sha256 and signature"}}