{"kind":"crawlcheck-resolve","v":1,"schema":"https://crawlcheck.io/schemas/resolve.json","domain":"you.com","answered_at":"2026-10-19T02:25:23.680Z","reading":"Each section is declared (what the site says about itself), observed (what CrawlCheck was served) or not_measured, with when it was observed. There is no overall score: decide on the section your action depends on.","status":"measured","subject":{"domain":"you.com","host":"you.com","path":"/","report_id":"btfg2dmw14","scanned_at":"2026-10-18T14:02:37.678Z","score_version":32},"freshness":{"observed_at":"2026-10-18T14:02:37.678Z","age_hours":12.4,"latest_report_id":"btfg2dmw14","is_latest":true,"newer_scan_exists":false,"last_second_reading_at":null,"note":"observed_at is when CrawlCheck fetched the site; nothing here says the site is unchanged since","last_verified_at":"2026-10-18T14:02:37.678Z","fresh_until":"2026-10-25T14:02:37.678Z","stale":false,"fresh_policy":"an answer is fresh for 168 hours after observed_at; each action below has its own maximum age. last_verified_at is the latest reading of this domain by any enrolled observer."},"observers":{"primary":{"version_tag":"6dea1861abf85ecc","vantage":{"kind":"cloudflare-worker","colo":null}},"independent_readings":0,"by_field":"https://crawlcheck.io/api/corroboration?domain=you.com","directory":"https://crawlcheck.io/.well-known/crawlcheck-observers.json","samples_by_network":{"networks":0,"samples":0,"groups":[],"window":"the latest 12 independent readings kept for this domain; the primary scan (Cloudflare) is not counted here"}},"crawl_policy":{"state":"declared","observed_at":"2026-10-18T14:02:37.678Z","source":"https://you.com/robots.txt","http_status":200,"path_tested":"/","agents":[{"agent":"Googlebot-Image","label":"Google image crawler","role":"search","allowed_homepage":true,"allowed_path":true,"has_own_group":false,"wildcard_rules_shadowed":false},{"agent":"Googlebot-News","label":"Google News","role":"search","allowed_homepage":true,"allowed_path":true,"has_own_group":false,"wildcard_rules_shadowed":false},{"agent":"Googlebot-Video","label":"Google video crawler","role":"search","allowed_homepage":true,"allowed_path":true,"has_own_group":false,"wildcard_rules_shadowed":false},{"agent":"Storebot-Google","label":"Google Shopping","role":"search","allowed_homepage":true,"allowed_path":true,"has_own_group":false,"wildcard_rules_shadowed":false},{"agent":"Mediapartners-Google","label":"Google AdSense","role":"search","allowed_homepage":true,"allowed_path":true,"has_own_group":false,"wildcard_rules_shadowed":false},{"agent":"APIs-Google","label":"Google push delivery","role":"search","allowed_homepage":true,"allowed_path":true,"has_own_group":false,"wildcard_rules_shadowed":false},{"agent":"BingPreview","label":"Bing page preview","role":"search","allowed_homepage":true,"allowed_path":true,"has_own_group":false,"wildcard_rules_shadowed":false},{"agent":"msnbot","label":"Microsoft legacy crawler","role":"search","allowed_homepage":true,"allowed_path":true,"has_own_group":false,"wildcard_rules_shadowed":false},{"agent":"Slurp","label":"Yahoo Slurp","role":"search","allowed_homepage":true,"allowed_path":true,"has_own_group":false,"wildcard_rules_shadowed":false},{"agent":"Yandex","label":"Yandex (umbrella token)","role":"search","allowed_homepage":true,"allowed_path":true,"has_own_group":false,"wildcard_rules_shadowed":false},{"agent":"coccocbot-web","label":"Coc Coc","role":"search","allowed_homepage":true,"allowed_path":true,"has_own_group":false,"wildcard_rules_shadowed":false},{"agent":"Facebot","label":"Meta link crawler","role":"social","allowed_homepage":true,"allowed_path":true,"has_own_group":false,"wildcard_rules_shadowed":false},{"agent":"Magpie-crawler","label":"Magpie AI","role":"train","allowed_homepage":true,"allowed_path":true,"has_own_group":false,"wildcard_rules_shadowed":false},{"agent":"img2dataset","label":"img2dataset image corpus","role":"train","allowed_homepage":true,"allowed_path":true,"has_own_group":false,"wildcard_rules_shadowed":false},{"agent":"AwarioRssBot","label":"Awario RSS","role":"train","allowed_homepage":true,"allowed_path":true,"has_own_group":false,"wildcard_rules_shadowed":false},{"agent":"AwarioSmartBot","label":"Awario smart","role":"train","allowed_homepage":true,"allowed_path":true,"has_own_group":false,"wildcard_rules_shadowed":false},{"agent":"TurnitinBot","label":"Turnitin","role":"train","allowed_homepage":true,"allowed_path":true,"has_own_group":false,"wildcard_rules_shadowed":false},{"agent":"archive.org_bot","label":"Internet Archive","role":"train","allowed_homepage":true,"allowed_path":true,"has_own_group":false,"wildcard_rules_shadowed":false},{"agent":"ia_archiver","label":"Internet Archive (legacy)","role":"train","allowed_homepage":true,"allowed_path":true,"has_own_group":false,"wildcard_rules_shadowed":false},{"agent":"AhrefsBot","label":"Ahrefs link index","role":"tool","allowed_homepage":true,"allowed_path":true,"has_own_group":false,"wildcard_rules_shadowed":false},{"agent":"SemrushBot","label":"Semrush crawler","role":"tool","allowed_homepage":true,"allowed_path":true,"has_own_group":false,"wildcard_rules_shadowed":false},{"agent":"MJ12bot","label":"Majestic link index","role":"tool","allowed_homepage":true,"allowed_path":true,"has_own_group":false,"wildcard_rules_shadowed":false},{"agent":"DotBot","label":"Moz link index","role":"tool","allowed_homepage":true,"allowed_path":true,"has_own_group":false,"wildcard_rules_shadowed":false},{"agent":"rogerbot","label":"Moz site crawler","role":"tool","allowed_homepage":true,"allowed_path":true,"has_own_group":false,"wildcard_rules_shadowed":false},{"agent":"DataForSeoBot","label":"DataForSEO","role":"tool","allowed_homepage":true,"allowed_path":true,"has_own_group":false,"wildcard_rules_shadowed":false},{"agent":"BLEXBot","label":"WebMeUp link index","role":"tool","allowed_homepage":true,"allowed_path":true,"has_own_group":false,"wildcard_rules_shadowed":false},{"agent":"Yeti","label":"Naver","role":"regional","allowed_homepage":true,"allowed_path":true,"has_own_group":false,"wildcard_rules_shadowed":false},{"agent":"YoudaoBot","label":"Youdao","role":"regional","allowed_homepage":true,"allowed_path":true,"has_own_group":false,"wildcard_rules_shadowed":false},{"agent":"Exabot","label":"Exalead","role":"regional","allowed_homepage":true,"allowed_path":true,"has_own_group":false,"wildcard_rules_shadowed":false},{"agent":"Sogou web spider","label":"Sogou","role":"regional","allowed_homepage":true,"allowed_path":true,"has_own_group":false,"wildcard_rules_shadowed":false},{"agent":"YisouSpider","label":"Yisou","role":"regional","allowed_homepage":true,"allowed_path":true,"has_own_group":false,"wildcard_rules_shadowed":false},{"agent":"360Spider","label":"360 Search","role":"regional","allowed_homepage":true,"allowed_path":true,"has_own_group":false,"wildcard_rules_shadowed":false},{"agent":"Sosospider","label":"Soso","role":"regional","allowed_homepage":true,"allowed_path":true,"has_own_group":false,"wildcard_rules_shadowed":false},{"agent":"meta-webindexer","label":"Meta web index","role":"train","allowed_homepage":true,"allowed_path":true,"has_own_group":false,"wildcard_rules_shadowed":false},{"agent":"omgili","label":"Webz.io omgili","role":"train","allowed_homepage":true,"allowed_path":true,"has_own_group":false,"wildcard_rules_shadowed":false},{"agent":"cohere-training-data-crawler","label":"Cohere training corpus","role":"train","allowed_homepage":true,"allowed_path":true,"has_own_group":false,"wildcard_rules_shadowed":false},{"agent":"PanguBot","label":"Huawei PanGu","role":"train","allowed_homepage":true,"allowed_path":true,"has_own_group":false,"wildcard_rules_shadowed":false},{"agent":"Ai2Bot-Dolma","label":"Allen Institute Dolma","role":"train","allowed_homepage":true,"allowed_path":true,"has_own_group":false,"wildcard_rules_shadowed":false},{"agent":"FriendlyCrawler","label":"FriendlyCrawler ML","role":"train","allowed_homepage":true,"allowed_path":true,"has_own_group":false,"wildcard_rules_shadowed":false},{"agent":"VelenPublicWebCrawler","label":"Velen","role":"train","allowed_homepage":true,"allowed_path":true,"has_own_group":false,"wildcard_rules_shadowed":false},{"agent":"MyCentralAIScraperBot","label":"MyCentral AI","role":"train","allowed_homepage":true,"allowed_path":true,"has_own_group":false,"wildcard_rules_shadowed":false},{"agent":"DeepSeekBot","label":"DeepSeek","role":"train","allowed_homepage":true,"allowed_path":true,"has_own_group":false,"wildcard_rules_shadowed":false},{"agent":"ICC-Crawler","label":"NICT ICC","role":"train","allowed_homepage":true,"allowed_path":true,"has_own_group":false,"wildcard_rules_shadowed":false},{"agent":"GoogleOther","label":"Google non-search fetch","role":"train","allowed_homepage":true,"allowed_path":true,"has_own_group":false,"wildcard_rules_shadowed":false},{"agent":"Google-CloudVertexBot","label":"Vertex AI agent build","role":"train","allowed_homepage":true,"allowed_path":true,"has_own_group":false,"wildcard_rules_shadowed":false},{"agent":"CloudflareBrowserRenderingCrawler","label":"Cloudflare Browser Run /crawl","role":"tool","allowed_homepage":true,"allowed_path":true,"has_own_group":false,"wildcard_rules_shadowed":false},{"agent":"Cloudflare-AutoRAG","label":"Cloudflare AutoRAG","role":"tool","allowed_homepage":true,"allowed_path":true,"has_own_group":false,"wildcard_rules_shadowed":false},{"agent":"Peer39_Crawler","label":"Peer39 ad context","role":"tool","allowed_homepage":true,"allowed_path":true,"has_own_group":false,"wildcard_rules_shadowed":false},{"agent":"AdsBot-Google","label":"Google Ads quality","role":"tool","allowed_homepage":true,"allowed_path":true,"has_own_group":false,"wildcard_rules_shadowed":false},{"agent":"AmazonAdBot","label":"Amazon Ads","role":"tool","allowed_homepage":true,"allowed_path":true,"has_own_group":false,"wildcard_rules_shadowed":false},{"agent":"AdIdxBot","label":"Microsoft Ads","role":"tool","allowed_homepage":true,"allowed_path":true,"has_own_group":false,"wildcard_rules_shadowed":false},{"agent":"Twitterbot","label":"X link preview","role":"social","allowed_homepage":true,"allowed_path":true,"has_own_group":false,"wildcard_rules_shadowed":false},{"agent":"LinkedInBot","label":"LinkedIn link preview","role":"social","allowed_homepage":true,"allowed_path":true,"has_own_group":false,"wildcard_rules_shadowed":false},{"agent":"facebookexternalhit","label":"Facebook link preview","role":"social","allowed_homepage":true,"allowed_path":true,"has_own_group":false,"wildcard_rules_shadowed":false},{"agent":"Pinterestbot","label":"Pinterest","role":"social","allowed_homepage":true,"allowed_path":true,"has_own_group":false,"wildcard_rules_shadowed":false},{"agent":"Slackbot-LinkExpanding","label":"Slack unfurl","role":"social","allowed_homepage":true,"allowed_path":true,"has_own_group":false,"wildcard_rules_shadowed":false},{"agent":"OAI-SearchBot","label":"ChatGPT search index","role":"answer","allowed_homepage":true,"allowed_path":true,"has_own_group":false,"wildcard_rules_shadowed":false},{"agent":"ChatGPT-User","label":"ChatGPT live fetch","role":"answer","allowed_homepage":true,"allowed_path":true,"has_own_group":false,"wildcard_rules_shadowed":false},{"agent":"Claude-SearchBot","label":"Claude search index","role":"answer","allowed_homepage":true,"allowed_path":true,"has_own_group":false,"wildcard_rules_shadowed":false},{"agent":"Claude-User","label":"Claude live fetch","role":"answer","allowed_homepage":true,"allowed_path":true,"has_own_group":false,"wildcard_rules_shadowed":false},{"agent":"PerplexityBot","label":"Perplexity index","role":"answer","allowed_homepage":true,"allowed_path":true,"has_own_group":false,"wildcard_rules_shadowed":false},{"agent":"Perplexity-User","label":"Perplexity live fetch","role":"answer","allowed_homepage":true,"allowed_path":true,"has_own_group":false,"wildcard_rules_shadowed":false},{"agent":"Gemini-Deep-Research","label":"Gemini research agent","role":"answer","allowed_homepage":true,"allowed_path":true,"has_own_group":false,"wildcard_rules_shadowed":false},{"agent":"MistralAI-User","label":"Le Chat live fetch","role":"answer","allowed_homepage":true,"allowed_path":true,"has_own_group":false,"wildcard_rules_shadowed":false},{"agent":"DuckAssistBot","label":"DuckDuckGo AI assist","role":"answer","allowed_homepage":true,"allowed_path":true,"has_own_group":false,"wildcard_rules_shadowed":false},{"agent":"YouBot","label":"You.com","role":"answer","allowed_homepage":true,"allowed_path":true,"has_own_group":false,"wildcard_rules_shadowed":false},{"agent":"PhindBot","label":"Phind","role":"answer","allowed_homepage":true,"allowed_path":true,"has_own_group":false,"wildcard_rules_shadowed":false},{"agent":"Kagibot","label":"Kagi","role":"answer","allowed_homepage":true,"allowed_path":true,"has_own_group":false,"wildcard_rules_shadowed":false},{"agent":"Copilot-User","label":"Microsoft Copilot fetch","role":"answer","allowed_homepage":true,"allowed_path":true,"has_own_group":false,"wildcard_rules_shadowed":false},{"agent":"Meta-ExternalFetcher","label":"Meta AI live fetch","role":"answer","allowed_homepage":true,"allowed_path":true,"has_own_group":false,"wildcard_rules_shadowed":false},{"agent":"Googlebot","label":"Google Search and AI Overviews","role":"search","allowed_homepage":true,"allowed_path":true,"has_own_group":false,"wildcard_rules_shadowed":false},{"agent":"bingbot","label":"Bing and Copilot index","role":"search","allowed_homepage":true,"allowed_path":true,"has_own_group":false,"wildcard_rules_shadowed":false},{"agent":"Applebot","label":"Apple and Siri","role":"search","allowed_homepage":true,"allowed_path":true,"has_own_group":false,"wildcard_rules_shadowed":false},{"agent":"Amazonbot","label":"Amazon","role":"search","allowed_homepage":true,"allowed_path":true,"has_own_group":false,"wildcard_rules_shadowed":false},{"agent":"DuckDuckBot","label":"DuckDuckGo","role":"search","allowed_homepage":true,"allowed_path":true,"has_own_group":false,"wildcard_rules_shadowed":false},{"agent":"YandexBot","label":"Yandex","role":"search","allowed_homepage":true,"allowed_path":true,"has_own_group":false,"wildcard_rules_shadowed":false},{"agent":"Baiduspider","label":"Baidu","role":"search","allowed_homepage":true,"allowed_path":true,"has_own_group":false,"wildcard_rules_shadowed":false},{"agent":"Seznambot","label":"Seznam","role":"search","allowed_homepage":true,"allowed_path":true,"has_own_group":false,"wildcard_rules_shadowed":false},{"agent":"Neevabot","label":"Neeva","role":"search","allowed_homepage":true,"allowed_path":true,"has_own_group":false,"wildcard_rules_shadowed":false},{"agent":"PetalBot","label":"Huawei Petal","role":"search","allowed_homepage":true,"allowed_path":true,"has_own_group":false,"wildcard_rules_shadowed":false},{"agent":"GPTBot","label":"OpenAI training and index","role":"train","allowed_homepage":true,"allowed_path":true,"has_own_group":false,"wildcard_rules_shadowed":false},{"agent":"ClaudeBot","label":"Anthropic training","role":"train","allowed_homepage":true,"allowed_path":true,"has_own_group":false,"wildcard_rules_shadowed":false},{"agent":"anthropic-ai","label":"Anthropic legacy agent","role":"train","allowed_homepage":true,"allowed_path":true,"has_own_group":false,"wildcard_rules_shadowed":false},{"agent":"Claude-Web","label":"Anthropic legacy agent","role":"train","allowed_homepage":true,"allowed_path":true,"has_own_group":false,"wildcard_rules_shadowed":false},{"agent":"Google-Extended","label":"Gemini training","role":"train","allowed_homepage":true,"allowed_path":true,"has_own_group":false,"wildcard_rules_shadowed":false},{"agent":"Applebot-Extended","label":"Apple training","role":"train","allowed_homepage":true,"allowed_path":true,"has_own_group":false,"wildcard_rules_shadowed":false},{"agent":"CCBot","label":"Common Crawl","role":"train","allowed_homepage":true,"allowed_path":true,"has_own_group":false,"wildcard_rules_shadowed":false},{"agent":"Bytespider","label":"ByteDance","role":"train","allowed_homepage":true,"allowed_path":true,"has_own_group":false,"wildcard_rules_shadowed":false},{"agent":"meta-externalagent","label":"Meta AI training","role":"train","allowed_homepage":true,"allowed_path":true,"has_own_group":false,"wildcard_rules_shadowed":false},{"agent":"FacebookBot","label":"Meta legacy","role":"train","allowed_homepage":true,"allowed_path":true,"has_own_group":false,"wildcard_rules_shadowed":false},{"agent":"cohere-ai","label":"Cohere","role":"train","allowed_homepage":true,"allowed_path":true,"has_own_group":false,"wildcard_rules_shadowed":false},{"agent":"Diffbot","label":"Diffbot knowledge graph","role":"train","allowed_homepage":true,"allowed_path":true,"has_own_group":false,"wildcard_rules_shadowed":false},{"agent":"Omgilibot","label":"Webz.io","role":"train","allowed_homepage":true,"allowed_path":true,"has_own_group":false,"wildcard_rules_shadowed":false},{"agent":"ImagesiftBot","label":"Imagesift","role":"train","allowed_homepage":true,"allowed_path":true,"has_own_group":false,"wildcard_rules_shadowed":false},{"agent":"Timpibot","label":"Timpi","role":"train","allowed_homepage":true,"allowed_path":true,"has_own_group":false,"wildcard_rules_shadowed":false},{"agent":"AI2Bot","label":"Allen Institute","role":"train","allowed_homepage":true,"allowed_path":true,"has_own_group":false,"wildcard_rules_shadowed":false},{"agent":"Scrapy","label":"Generic scraper framework","role":"train","allowed_homepage":true,"allowed_path":true,"has_own_group":false,"wildcard_rules_shadowed":false},{"agent":"SemrushBot-OCOB","label":"Semrush AI corpus","role":"train","allowed_homepage":true,"allowed_path":true,"has_own_group":false,"wildcard_rules_shadowed":false},{"agent":"Applebot-Extended-Ads","label":"Apple ads corpus","role":"train","allowed_homepage":true,"allowed_path":true,"has_own_group":false,"wildcard_rules_shadowed":false},{"agent":"TikTokSpider","label":"TikTok","role":"train","allowed_homepage":true,"allowed_path":true,"has_own_group":false,"wildcard_rules_shadowed":false},{"agent":"QuillBot","label":"QuillBot","role":"train","allowed_homepage":true,"allowed_path":true,"has_own_group":false,"wildcard_rules_shadowed":false},{"agent":"Webzio-Extended","label":"Webz.io extended","role":"train","allowed_homepage":true,"allowed_path":true,"has_own_group":false,"wildcard_rules_shadowed":false},{"agent":"ProRataInc","label":"ProRata","role":"train","allowed_homepage":true,"allowed_path":true,"has_own_group":false,"wildcard_rules_shadowed":false},{"agent":"AwarioBot","label":"Awario","role":"train","allowed_homepage":true,"allowed_path":true,"has_own_group":false,"wildcard_rules_shadowed":false},{"agent":"Amzn-SearchBot","label":"Amazon search eligibility, separate from Amazonbot","role":"search","allowed_homepage":true,"allowed_path":true,"has_own_group":false,"wildcard_rules_shadowed":false},{"agent":"Amzn-User","label":"Amazon live fetch for Alexa questions","role":"answer","allowed_homepage":true,"allowed_path":true,"has_own_group":false,"wildcard_rules_shadowed":false},{"agent":"MistralAI-Index","label":"Mistral search index for Vibe","role":"search","allowed_homepage":true,"allowed_path":true,"has_own_group":false,"wildcard_rules_shadowed":false},{"agent":"MistralAI-Training","label":"Mistral training crawl","role":"train","allowed_homepage":true,"allowed_path":true,"has_own_group":false,"wildcard_rules_shadowed":false},{"agent":"OAI-AdsBot","label":"OpenAI ChatGPT ads page validation","role":"tool","allowed_homepage":true,"allowed_path":true,"has_own_group":false,"wildcard_rules_shadowed":false},{"agent":"Google-Agent","label":"Google user-triggered agent, acts on the web","role":"answer","allowed_homepage":true,"allowed_path":true,"has_own_group":false,"wildcard_rules_shadowed":false},{"agent":"Google-GeminiNotebook","label":"Gemini Notebook user-supplied URL fetch","role":"answer","allowed_homepage":true,"allowed_path":true,"has_own_group":false,"wildcard_rules_shadowed":false},{"agent":"GoogleOther-Image","label":"Google common crawler, public image URLs","role":"train","allowed_homepage":true,"allowed_path":true,"has_own_group":false,"wildcard_rules_shadowed":false},{"agent":"GoogleOther-Video","label":"Google common crawler, public video URLs","role":"train","allowed_homepage":true,"allowed_path":true,"has_own_group":false,"wildcard_rules_shadowed":false},{"agent":"ExaSearchBot","label":"Exa AI search index, Web Bot Auth signed","role":"search","allowed_homepage":true,"allowed_path":true,"has_own_group":false,"wildcard_rules_shadowed":false}],"signal_conflicts":[],"note":"robots.txt governs access and Content-Signal governs use; both are what the site declares, not what it serves"},"delivery":{"state":"observed","observed_at":"2026-10-18T14:02:37.678Z","url":"https://you.com/","identity_basis":"every identity was sent by CrawlCheck from its own network; none is the named operator's own crawler","identities":[{"identity":"anon","label":"Unnamed client","role":"control","status":200,"words":1050,"blocker":null,"redirected_to":null,"x_robots_tag":null},{"identity":"gptbot","label":"GPTBot","role":"train","status":200,"words":1050,"blocker":null,"redirected_to":null,"x_robots_tag":null},{"identity":"claudebot","label":"ClaudeBot","role":"train","status":200,"words":1050,"blocker":null,"redirected_to":null,"x_robots_tag":null},{"identity":"oaisearch","label":"OAI-SearchBot","role":"answer","status":200,"words":1050,"blocker":null,"redirected_to":null,"x_robots_tag":null},{"identity":"claudesearch","label":"Claude-SearchBot","role":"answer","status":200,"words":1050,"blocker":null,"redirected_to":null,"x_robots_tag":null},{"identity":"perplexity","label":"PerplexityBot","role":"answer","status":200,"words":1050,"blocker":null,"redirected_to":null,"x_robots_tag":null},{"identity":"googlebot","label":"Googlebot","role":"search","status":200,"words":1050,"blocker":null,"redirected_to":null,"x_robots_tag":null},{"identity":"browser","label":"Mobile browser","role":"browser","status":200,"words":1050,"blocker":null,"redirected_to":null,"x_robots_tag":null},{"identity":"bingbot","label":"Bingbot","role":"search","status":200,"words":1050,"blocker":null,"redirected_to":null,"x_robots_tag":null},{"identity":"applebot","label":"Applebot","role":"search","status":200,"words":1050,"blocker":null,"redirected_to":null,"x_robots_tag":null},{"identity":"amazonbot","label":"Amazonbot","role":"train","status":200,"words":1050,"blocker":null,"redirected_to":null,"x_robots_tag":null},{"identity":"bytespider","label":"Bytespider","role":"train","status":200,"words":1050,"blocker":null,"redirected_to":null,"x_robots_tag":null},{"identity":"metaexternal","label":"Meta-ExternalAgent","role":"train","status":200,"words":1050,"blocker":null,"redirected_to":null,"x_robots_tag":null},{"identity":"ccbot","label":"CCBot","role":"train","status":200,"words":1050,"blocker":null,"redirected_to":null,"x_robots_tag":null}]},"machine_files":{"state":"observed","observed_at":"2026-10-18T14:02:37.678Z","files":[{"path":"/robots.txt","status":200,"content_type":"text/plain; charset=utf-8","bytes":59,"verdict":"ok"},{"path":"/sitemap.xml","status":404,"content_type":"text/html; charset=utf-8","bytes":11525,"verdict":"absent"},{"path":"/sitemap_index.xml","status":200,"content_type":"text/html; charset=utf-8","bytes":72685,"verdict":"absent"},{"path":"/wp-sitemap.xml","status":200,"content_type":"text/html; charset=utf-8","bytes":72672,"verdict":"absent"},{"path":"/llms.txt","status":200,"content_type":"text/plain; charset=utf-8","bytes":22649,"verdict":"ok"}],"read_before_page":[{"path":"/llms.txt","bytes":22649,"kind":"machine"},{"path":"/.well-known/api-catalog","bytes":200,"kind":"agent"},{"path":"/robots.txt","bytes":59,"kind":"machine"}],"hashes":"every fetched body's SHA-256 is in the sealed manifest: https://crawlcheck.io/api/artifact/b7aca2f75e39afc5d716cd745ea521641fcd27d3c0d267e5bd073959974ed38b?body=1"},"capabilities":{"state":"observed","observed_at":"2026-10-18T14:02:37.678Z","checked":[{"kind":"api_catalog","url":"https://you.com/.well-known/api-catalog","status":200},{"kind":"service_desc","url":"https://api.you.com/openapi.json","status":null},{"kind":"a2a_agent_card","url":"https://you.com/.well-known/agent-card.json","status":404}],"declared":0,"by_safety_class":{},"verified":0,"mismatches":0,"invocation_rule":"capabilities are read from declarations only; nothing is invoked. Later verification may call read_only_public operations alone; every other class needs the site owner's authorisation and a test account, and unknown is never called","certificate":{"id":"ccap1:23dfc2acb3cbe6ef80291fb69447d9d5eb68bb8b57598d3172c0073421a29073","signed":true,"entry":"https://crawlcheck.io/api/registry/capabilities?domain=you.com","verify":"https://crawlcheck.io/api/registry/capabilities/verify?domain=you.com"},"items":"https://crawlcheck.io/api/capabilities?domain=you.com"},"entity":{"state":"observed","observed_at":"2026-10-18T14:02:37.678Z","name":"You.com","wikidata_item_found":true,"links":[{"url":"https://x.com/youdotcom","state":"corroborated","why":"the target page references you.com back"},{"url":"https://www.linkedin.com/company/youdotcom/","state":"corroborated","why":"the target page references you.com back"}],"states":"corroborated = the linked page links back; unreciprocated = readable and does not; unverifiable = could not be read from here"},"mcp_registry":{"state":"observed","listed":1,"observed":0,"initialized":0,"auth_required":0,"drifted":0,"shadowing":{"flagged":0,"servers":[],"shadowed_by":0,"shadowed_by_servers":[]},"auth_audit":null,"last_observed_at":null,"servers":[{"name":"io.github.youdotcom-oss/mcp","version":"4.0.4","endpoint":"https://api.you.com/mcp","outcome":"not_observed_yet","tools":null}],"url":"https://crawlcheck.io/api/mcp/servers?host=you.com","reading":"listed by the registry (the publisher’s claim); outcome is what the endpoint answered to the MCP handshake"},"content_clock":{"homepage":{"url":"https://you.com/","observed":true,"content_sha256":"28875bdf92f16616f6b4cd7f0aa2c5d54a4e61c8662a00c60842cb889f7a2e3e","first_observed":"2026-10-10T22:49:08.100Z","last_read":"2026-10-15T21:43:48.656Z","changes_observed":0,"main_content_last_changed":null,"unchanged_since_first_observed":"2026-10-10T22:49:08.100Z","declared":{},"date_moved_words_did_not":null,"api":"https://crawlcheck.io/api/v1/content-clock?url=https%3A%2F%2Fyou.com%2F"},"reading":"When the main content (not the bytes) of these URLs last changed, from CrawlCheck's own reads. Full history and the page's declared dates: the api link."},"self_claims":{"observed":true,"first_read":"2026-10-10T04:10:04.495Z","last_read":"2026-10-15T21:43:53.435Z","reads":5,"claims":[{"type":"years_in_business","value":"20 years","status":"current","first_seen":"2026-10-10T04:10:04.495Z","last_seen":"2026-10-15T21:43:53.435Z"}],"api":"https://crawlcheck.io/api/v1/claims?domain=you.com","reading":"what the site says about itself on its own pages, with when CrawlCheck first and last saw each value; not a statement that the claim is true"},"read_equivalence":{"url":"https://you.com/","observed":true,"verdict":"equivalent","checked_at":"2026-10-15T21:43:49.833Z","first_observed":"2026-10-10T15:03:14.571Z","ai_identities_served":"3 of 3","identities":[{"identity":"gptbot","verdict":"equivalent","text_ratio":1,"instruction_like_text":false},{"identity":"claudebot","verdict":"equivalent","text_ratio":1,"instruction_like_text":false},{"identity":"perplexity","verdict":"equivalent","text_ratio":1,"instruction_like_text":false}],"api":"https://crawlcheck.io/api/v1/read-equivalence?url=https%3A%2F%2Fyou.com%2F","reading":"whether the main text served to AI identities is the same text a person sees in a browser, from one simultaneous read of the homepage; evidence and the differing sentences at the api link"},"rights":{"observed":true,"path":"/","read_at":"2026-10-17T23:53:02.631Z","api":"https://crawlcheck.io/api/v1/rights?domain=you.com","reading":"one value per use for the homepage, with the signal that decided it; signals considered, conflicts and per-path answers at the api link","train":{"value":"restricted_by_terms","decided_by":{"signal":"terms page","url":"https://you.com/terms","evidence":"You acknowledge and agree that due to the nature of our Services and artificial intelligence generally, your Outputs may not be unique and other users may receive similar output from our Services."},"conflicts":0},"ground":{"value":"restricted_by_terms","decided_by":{"signal":"terms page","url":"https://you.com/terms","evidence":"APIs Web Search API Contents API Answer API Research API Finance Research API MCP server Agent skills Company About Careers Partnerships Events Grant program Resources Developer docs Benchmarks MCP server Trust center Blog Terms & policies Privacy policy Cookie notice Do not sell or share my personal information Privacy requests Terms & conditions Master service agreement Data processing addendum © 2026 You.com / All rights reserved llms.txt for.you.com"},"conflicts":0},"summarize":{"value":"restricted_by_terms","decided_by":{"signal":"terms page","url":"https://you.com/terms","evidence":"APIs Web Search API Contents API Answer API Research API Finance Research API MCP server Agent skills Company About Careers Partnerships Events Grant program Resources Developer docs Benchmarks MCP server Trust center Blog Terms & policies Privacy policy Cookie notice Do not sell or share my personal information Privacy requests Terms & conditions Master service agreement Data processing addendum © 2026 You.com / All rights reserved llms.txt for.you.com"},"conflicts":0},"cite":{"value":"not_declared","decided_by":{"signal":"robots.txt","url":"https://you.com/robots.txt","evidence":"access is open to OAI-SearchBot, Claude-SearchBot, PerplexityBot, Googlebot, Bingbot on / and no signal states whether cite is allowed"},"conflicts":0},"transact":{"value":"not_declared","decided_by":{"signal":"robots.txt","url":"https://you.com/robots.txt","evidence":"access is open to ChatGPT-User, Claude-User, Perplexity-User on / and no signal states whether transact is allowed"},"conflicts":0}},"findings":{"access":"public","rule":"the top finding in full; the rest by title, severity and path (a licence for this domain opens them)","items":[{"fid":"f1:8541fdd06c61ae0b030b","decision_id":"d1:16789f04178dfbc582ea","code":"DECLARED_SITEMAP_BROKEN","severity":4,"path":"/robots.txt","title":"robots.txt points at a sitemap that fails","rule_revision":1,"span_id":"59d63938db882a18","state":"new","valid_time":{"valid_from":{"after":"2026-10-14T21:03:02.069Z","by":"2026-10-18T14:02:37.678Z"},"valid_to":null,"recorded_at":"2026-10-18T14:03:34.584Z","superseded_at":null,"superseded_by":null,"earlier_intervals":0,"before_bt1":false},"observations_linked":4,"trust":{"freshness":{"observed_at":"2026-10-18T14:02:37.678Z","last_observed":"2026-10-18T14:02:37.678Z","held_since":"2026-10-18T14:03:34.584Z","state":"new"},"confidence":{"level":"single_reading","means":"one reading from one network","basis":["seen in 1 scan since 2026-10-18","the decision names the 4 fetches it read"],"sample":{"cross_network":"not_applicable","why":"this finding does not depend on the requesting identity or network, so other networks' readings neither confirm nor contradict it","readings":1,"diversity":{"readings":1,"observers":1,"distinct_networks":1,"networks":{"cloudflare":1},"distinct_countries":0,"countries":{}},"scans":{"seen":1,"could_have_seen":1,"interval":{"k":1,"n":1,"share":1,"low":0.207,"high":1,"level":0.95,"method":"Wilson score interval"},"counts":"every scan of this domain since it was first seen"},"interval":{"k":1,"n":1,"share":1,"low":0.207,"high":1,"level":0.95,"method":"Wilson score interval"},"interval_of":"scans that saw it, of scans that could have","note":"interval: the 95% Wilson score interval for the share, from k of n; it is wide when n is small, and that width is the point. networks: autonomous systems the readings arrived from (two observers on one network are one network). countries: where the connection was measured; CrawlCheck's own reading leaves from a Cloudflare data centre and carries no country here."}},"contradictions":[]},"score_effect":{"decision_id":"d1:16789f04178dfbc582ea","code":"DECLARED_SITEMAP_BROKEN","path":"/robots.txt","severity":4,"level":"high","points":0,"cap":"D","binding":true,"letter_now":"D","letter_without_it":"C","same_level_others":0,"says":"Caps the letter at D. Without it the letter would be C (score 76). Moves no points; the score is built from the measured rows."},"root_cause":null,"corroboration":{"status":"not_applicable","reading":"this finding does not depend on the requesting identity or network"},"explain":{"mcp_tool":"explain_finding","arguments":{"id":"d1:16789f04178dfbc582ea"},"url":"https://crawlcheck.io/api/explain?id=d1:16789f04178dfbc582ea"},"disputes":[]},{"title":"Almost nothing a machine receives from this page is readable text","severity":3,"path":"/","locked":true},{"title":"Links send crawlers to the same pages under many different addresses","severity":2,"path":"/","locked":true}],"summary":{"total":3,"serious":2,"by_severity":{"High":1,"Medium":1,"Low":1}},"withheld":"Every finding is listed by title, severity and path. The top one carries its code, detail and evidence; the other 2 are locked (locked: true) and need a licence or the fix service: https://crawlcheck.io/pricing"},"confidence":{"identities_answered":{"n":14,"of":14,"ratio":1},"vantages":1,"vantage_state":{"state":"single_vantage","means":"only one network read this site near the scan; nothing here is cross-checked","networks":1,"readings":1,"countries":0,"findings_contested":0,"findings_cross_confirmed":0,"states":{"single_vantage":"only one network read this site near the scan; nothing here is cross-checked","insufficient_samples":"more than one network read it, but fewer than 3 readings: too few to state agreement","cross_confirmed":"two or more networks read it, 3 or more readings, and no finding is contested between them","network_divergence":"networks disagree on at least one finding","regional_divergence":"networks in more than one country disagree on at least one finding: what is served depends on where the request comes from"}},"note":"counts and ratios only - there is no composite confidence number, because the parts do not combine into one honestly","sample":{"readings":1,"observers":1,"distinct_networks":1,"networks":{"cloudflare":1},"distinct_countries":0,"countries":{},"window_hours":6,"all_time":{"independent_readings":0,"networks":[]},"reading":"readings of this domain within 6 hours of the scan: CrawlCheck's own plus every independent observer's, with the networks and countries they arrived from","note":"interval: the 95% Wilson score interval for the share, from k of n; it is wide when n is small, and that width is the point. networks: autonomous systems the readings arrived from (two observers on one network are one network). countries: where the connection was measured; CrawlCheck's own reading leaves from a Cloudflare data centre and carries no country here."}},"contradictions":{"site":[],"between_networks_open":[],"locked_findings_not_shown":2},"evidence":{"report":"https://crawlcheck.io/r/btfg2dmw14","machine_record":"https://crawlcheck.io/api/v1/machine-record?id=btfg2dmw14","manifest_sha256":"b7aca2f75e39afc5d716cd745ea521641fcd27d3c0d267e5bd073959974ed38b","verify":"https://crawlcheck.io/api/verify?id=btfg2dmw14","bundle":"https://crawlcheck.io/api/bundle?id=btfg2dmw14","offline_verifier":"https://crawlcheck.io/crawlcheck-verify.mjs?v=b22547e95463d03e","verify_this_answer":"save it as answer.json, then: node crawlcheck-verify.mjs answer.json"},"impersonation":{"state":"no_lookalike","verdict":"no_lookalike","brand_frame":"Tranco top 10,000 (Y83YG)"},"actions":{"v":1,"decisions":"allow | warn | require_confirmation | block | unsupported","rule":"decided from this answer's own sections at signing time; an agent's own policy may be stricter, never looser","read":{"decision":"allow","minimum_assurance":"reachable","assurance_found":["declared","reachable"],"needs":"robots.txt lets AI agents read the path, and the page was served to at least one AI-crawler identity","why":"9 of 9 AI-crawler identities were served the page","doubts":[],"max_age_hours":168},"cite":{"decision":"allow","minimum_assurance":"reachable + entity observed","assurance_found":["declared","reachable","entity observed","read_equivalent"],"needs":"everything read needs, plus an observed entity and no contradiction the site makes with itself","scope":{"covers":["access: robots.txt and what was actually served to AI-crawler identities","permissions: the rights and content signals the site declares","identity: the entity the site declares and who has proven control of the domain","provenance: when, from where and by whom it was observed, signed and logged"],"does_not_cover":"whether the content is true, accurate, current or complete; a cite decision says the source may be cited and attributed, not that what it says is right"},"why":"9 of 9 AI-crawler identities were served the page","doubts":[],"max_age_hours":168},"connect":{"decision":"warn","minimum_assurance":"conformant","assurance_found":["listed"],"needs":"a declared capability that CrawlCheck invoked read-only and found to behave as declared","why":"the official MCP Registry lists 1 MCP server(s) here; none has completed the MCP handshake when CrawlCheck tried","doubts":["registry_unverified"],"max_age_hours":72},"transact":{"decision":"require_confirmation","minimum_assurance":"action_ready","assurance_found":[],"needs":"owner-verified identity and an action-ready capability; CrawlCheck does not certify transactions, so a human confirms","why":"CrawlCheck does not certify transactions; a human confirms every financial, identity or irreversible action","doubts":[],"max_age_hours":24},"administer":{"decision":"block","minimum_assurance":"never automatic","assurance_found":[],"needs":"high-privilege operations are never authorised by a public record","why":"never authorised by a public record","doubts":[],"max_age_hours":0}},"absences":[{"what":"sitemap","path":"/sitemap.xml","result":"absent","at":"2026-10-18T14:02:37.678Z","from":{"observer":"crawlcheck-worker","network":"cloudflare","directory":"https://crawlcheck.io/.well-known/crawlcheck-observers.json"},"checked":[{"method":"GET","url":"https://you.com/sitemap.xml","status":404,"redirects":"followed"}],"scope":"GET https://you.com/sitemap.xml answered HTTP 404 at 2026-10-18T14:02:37.678Z. The file was not served at this path then; this says nothing about the same content at another path or a later time."},{"what":"a2a_agent_card","result":"absent","at":"2026-10-18T14:02:37.678Z","from":{"observer":"crawlcheck-worker","network":"cloudflare","directory":"https://crawlcheck.io/.well-known/crawlcheck-observers.json"},"checked":[{"method":"GET","url":"https://you.com/.well-known/agent-card.json","status":404,"redirects":"followed"}],"scope":"Every location CrawlCheck checks for a a2a agent card answered not found (404). No declaration of this kind was served at those URLs at that time; one served elsewhere, or added later, is not covered."}],"control":{"state":"unproven","proofs":[],"organization":null,"claim":"https://crawlcheck.io/registry/you.com/claim","why":"nobody has proven control of you.com to CrawlCheck, so who stands behind the site is unconfirmed","note":"control says who answers for the domain, not whether it is safe; it does not change any decision in this answer"},"transparency":{"state":"included","log_id":"crawlcheck-resolve-log-v1","leaf_hash":"37884d15418fb373127617b122c4695146de5e6a9acffde6ad93fc093d09638a","batch":736,"index":116,"batch_size":500,"path":["37c4aced80bd317443211c9ee8a42deb51899cdce191dbe345ee4d9445e4f2d9","246abef56c7ce59338b5038a73bc41c05781d2e57bad5b7188320cb1fc7abebc","8e508ed175b810d29aad4b20aa1cc0d265fed30bce096f15e84586d6cc179588","cbc30f5c96fe9b9b0cb933b555fbb7f0b393147499a83250b63bac50b849cf41","2f1f6ac80b880ff32254eaf159cc25566449962db61eba1c93a78d6c5accd7d4","3381db7b955827a8353e93feca3962b69e1090e010cb7989c6d98750cab906ca","f8b23e1c2c86472886fb4b339cfd660806cb3c70e1f5396584e22ea836d9c999","ab3fcba03ba31067449b59c978c9eac1a6a9d403e8d6b5e1ea6aa57223e10003","c94385284c2e06e1b07722af6081f7e8d5886dcd0a67c51472b48d8c7251b0ff"],"sth":{"kind":"crawlcheck-log-sth","v":1,"log_id":"crawlcheck-resolve-log-v1","batch":736,"first_index":162007,"batch_size":500,"tree_size":162507,"batch_root":"ecb8db7b01831110a2a0e148d99b33755f64ba2455dca6a551e76f15639f19dc","prev_sth_sha256":"9c0f0abe29a0ff5407c414cbe4ce8fc5218e9e69accf6642b7f8dae256cbbf1d","created_at":"2026-10-18T14:32:51.440Z","rules":{"leaf":"SHA-256(0x00 || canonical JSON of the answer's decision content)","tree":"RFC 9162 Merkle tree hash over this batch's leaf hashes, sorted","chain":"prev_sth_sha256 is the sha256 of the previous batch's signed tree head; batch numbers are consecutive and tree_size grows by batch_size"},"code":{"version_id":"b3cf8c17-2fdd-4134-979f-8e44241c5744","script_sha16":"6dea1861abf85ecc","state":"attested","deployed":{"at":"2026-10-18T13:45:50.312Z","commit":"a0b9f79e8766","run":"39132905982"},"build_log_entry":596,"score_version":32,"scoring_policy_sha256":"d3c9c95d409dae4e90b8e868ff113e631e445e33c14a0cc008dda144293821b3","decision_rules_sha256":"18dd5636e0896692ab518050cb401744edea2e6f1086dff94fead314959bdc3e","check":"https://crawlcheck.io/api/v1/builds/b3cf8c17-2fdd-4134-979f-8e44241c5744"},"sha256":"ed76c42c91b6aea25f3d3c648ccedf39d053594d67fe49eef79615b5d7e1d2d2","signature":{"v":1,"alg":"Ed25519","kid":"1hHmE3fo1rg5Ig70wuFtJ5wBc7G-sUTgUJkxyPWFRps","over":"sha256","message":"crawlcheck-data-v1\ned76c42c91b6aea25f3d3c648ccedf39d053594d67fe49eef79615b5d7e1d2d2","sig":"Bz4SRULsz51BwN9c5AuBkJLcx97MhagFhMflNghdYuXwkWPcYaNVOfxhV-vuDecq5abMf_tyhJaf7MTv7EdqAg","signed_at":"2026-10-18T14:32:51.440Z","key":{"kid":"1hHmE3fo1rg5Ig70wuFtJ5wBc7G-sUTgUJkxyPWFRps","jwk":{"kty":"OKP","crv":"Ed25519","x":"5TmKCOx0TTHepq7HwyXBHvGQpVCSBd2lV8cLzYVqFxQ"},"published_in":"https://crawlcheck.io/.well-known/http-message-signatures-directory"},"canonical":"sha256 over canonical JSON of every field except sha256 and signature"}},"witnesses":"https://crawlcheck.io/api/v1/log/sth/736","content_rule":"leaf = SHA-256(0x00 || canonical JSON of the answer's decision content); see https://crawlcheck.io/api/v1/log"},"assurance_ladder":{"v":1,"level":"reachable","order":["declared","discovered","reachable","conformant","owner_verified","authorized","reliable","action_ready"],"rungs":[{"rung":"declared","state":"reached","why":"robots.txt policy declared; 1 MCP server(s) in the registry"},{"rung":"discovered","state":"reached","why":"0 declared operation(s), 1 registry server(s)"},{"rung":"reachable","state":"reached","why":"AI-crawler identities were served the page"},{"rung":"conformant","state":"not_measured","why":"no same-origin, login-free MCP server or A2A agent to run the read-only suite against, and no capability verified by invocation"},{"rung":"owner_verified","state":"not_reached","why":"nobody has proven control of you.com to CrawlCheck, so who stands behind the site is unconfirmed"},{"rung":"authorized","state":"not_reached","why":"the domain's owner has published no authorization for agents"},{"rung":"reliable","state":"not_measured","why":"reliability is measured for MCP endpoints over time; none on this domain is on the panel"},{"rung":"action_ready","state":"not_reached","why":"not yet: conformant, owner_verified, authorized, reliable"}],"rule":"level is the highest rung with every rung under it reached; a rung is reached only on measured evidence, not_reached when measured and short, not_measured when there was nothing to measure. The ladder describes evidence; each action's decision is still made in actions"},"code":{"version_id":"6929574e-df21-45a5-8a92-5b8ba41cc9aa","script_sha16":"85c9b9b6c9a512ed","state":"attested","deployed":{"at":"2026-10-18T20:02:49.319Z","commit":"2bf491f55991","run":"39161431635"},"build_log_entry":597,"score_version":32,"scoring_policy_sha256":"d3c9c95d409dae4e90b8e868ff113e631e445e33c14a0cc008dda144293821b3","decision_rules_sha256":"18dd5636e0896692ab518050cb401744edea2e6f1086dff94fead314959bdc3e","check":"https://crawlcheck.io/api/v1/builds/6929574e-df21-45a5-8a92-5b8ba41cc9aa"},"sha256":"649e6ebf3452d3e1599900d6c879d51e63eb2cd09006b5bb620841bae14d7959","signature":{"v":1,"alg":"Ed25519","kid":"1hHmE3fo1rg5Ig70wuFtJ5wBc7G-sUTgUJkxyPWFRps","over":"sha256","message":"crawlcheck-data-v1\n649e6ebf3452d3e1599900d6c879d51e63eb2cd09006b5bb620841bae14d7959","sig":"d4RORvjO696I3lNRBVQQ9uLjBQjDVAx7wU3da8kX8P3No2I-DyKmapH9WFd-xf_yAxYlw6LbF9ESuE3xz8xBBw","signed_at":"2026-10-19T02:25:23.680Z","key":{"kid":"1hHmE3fo1rg5Ig70wuFtJ5wBc7G-sUTgUJkxyPWFRps","jwk":{"kty":"OKP","crv":"Ed25519","x":"5TmKCOx0TTHepq7HwyXBHvGQpVCSBd2lV8cLzYVqFxQ"},"published_in":"https://crawlcheck.io/.well-known/http-message-signatures-directory"},"canonical":"sha256 over canonical JSON of every field except sha256 and signature"}}