{"kind":"crawlcheck-census-weekly","v":1,"week":"2026-W47","state":"week_to_date","from":"2026-11-16T00:00:00.000Z","to":"2026-11-22T14:11:57.092Z","built_at":"2026-11-22T14:11:57.092Z","frames":[{"frame":"top1k","label":"top 1,000, daily","runs_this_week":7,"run":{"run":52,"finished_at":"2026-11-22T09:34:51.738Z"},"compared_with":{"run":45,"finished_at":"2026-11-15T08:59:28.147Z"},"sites":991,"figures":[{"id":"reachable","group":"Reach","label":"Homepage answers CrawlCheck's census fetch (2xx after redirects)","count":518,"of":991,"pct":52.3,"pct_before":52.3,"points_all":0,"points":0,"sites_gained":11,"sites_lost":11,"compared":991,"denominator_entered":0,"denominator_left":0},{"id":"refused_us","group":"Reach","label":"Homepage refuses CrawlCheck's census fetch (400, 401, 403 or 429): blocked or rate-limited, not down","count":121,"of":991,"pct":12.2,"pct_before":11.9,"points_all":0.3,"points":0.3,"sites_gained":7,"sites_lost":4,"compared":991,"denominator_entered":0,"denominator_left":0},{"id":"no_site","group":"Reach","label":"No website answers at that name (5xx, 530, timeout or no answer)","count":313,"of":991,"pct":31.6,"pct_before":31.8,"points_all":-0.2,"points":-0.2,"sites_gained":1,"sites_lost":3,"compared":991,"denominator_entered":0,"denominator_left":0},{"id":"robots_served","group":"Crawler policy","label":"Serves a robots.txt (200, not HTML)","count":408,"of":518,"pct":78.8,"pct_before":79.2,"points_all":-0.4,"points":-0.2,"sites_gained":1,"sites_lost":2,"compared":507,"denominator_entered":11,"denominator_left":11},{"id":"robots_sitemap","group":"Crawler policy","label":"robots.txt names a sitemap","count":292,"of":408,"pct":71.6,"pct_before":71.5,"points_all":0.1,"points":0,"sites_gained":1,"sites_lost":1,"compared":397,"denominator_entered":11,"denominator_left":13},{"id":"content_signal","group":"Crawler policy","label":"robots.txt carries a Content-Signal line","count":19,"of":408,"pct":4.7,"pct_before":4.4,"points_all":0.3,"points":0.3,"sites_gained":1,"sites_lost":0,"compared":397,"denominator_entered":11,"denominator_left":13},{"id":"cs_ai_train_no","group":"Crawler policy","label":"Content-Signal says ai-train=no","count":12,"of":408,"pct":2.9,"pct_before":3.2,"points_all":-0.3,"points":0,"sites_gained":0,"sites_lost":0,"compared":397,"denominator_entered":11,"denominator_left":13},{"id":"cs_ai_input_no","group":"Crawler policy","label":"Content-Signal says ai-input=no or search=no","count":4,"of":408,"pct":1,"pct_before":1,"points_all":0,"points":0,"sites_gained":0,"sites_lost":0,"compared":397,"denominator_entered":11,"denominator_left":13},{"id":"aipref_train","group":"Crawler policy","label":"robots.txt names the AI-training policy token (__usage_ai-training, IETF AIPREF draft)","count":8,"of":408,"pct":2,"pct_before":1.2,"points_all":0.8,"points":0.5,"sites_gained":2,"sites_lost":0,"compared":397,"denominator_entered":11,"denominator_left":13},{"id":"aipref_train_no","group":"Crawler policy","label":"robots.txt states a preference against AI training on / with that token","count":5,"of":408,"pct":1.2,"pct_before":1.2,"points_all":0,"points":0,"sites_gained":0,"sites_lost":0,"compared":397,"denominator_entered":11,"denominator_left":13},{"id":"blocks_any_ai","group":"Crawler policy","label":"robots.txt closes / to at least one of the 12 AI agents","count":102,"of":408,"pct":25,"pct_before":25.4,"points_all":-0.4,"points":0.3,"sites_gained":1,"sites_lost":0,"compared":397,"denominator_entered":11,"denominator_left":13},{"id":"blocks_GPTBot","group":"Crawler policy","label":"robots.txt closes / to GPTBot","count":66,"of":408,"pct":16.2,"pct_before":16.3,"points_all":-0.1,"points":0,"sites_gained":0,"sites_lost":0,"compared":397,"denominator_entered":11,"denominator_left":13},{"id":"blocks_OAI-SearchBot","group":"Crawler policy","label":"robots.txt closes / to OAI-SearchBot","count":32,"of":408,"pct":7.8,"pct_before":7.6,"points_all":0.2,"points":0,"sites_gained":0,"sites_lost":0,"compared":397,"denominator_entered":11,"denominator_left":13},{"id":"blocks_ChatGPT-User","group":"Crawler policy","label":"robots.txt closes / to ChatGPT-User","count":39,"of":408,"pct":9.6,"pct_before":9.3,"points_all":0.3,"points":0,"sites_gained":0,"sites_lost":0,"compared":397,"denominator_entered":11,"denominator_left":13},{"id":"blocks_ClaudeBot","group":"Crawler policy","label":"robots.txt closes / to ClaudeBot","count":63,"of":408,"pct":15.4,"pct_before":16.1,"points_all":-0.7,"points":0,"sites_gained":0,"sites_lost":0,"compared":397,"denominator_entered":11,"denominator_left":13},{"id":"blocks_Claude-SearchBot","group":"Crawler policy","label":"robots.txt closes / to Claude-SearchBot","count":38,"of":408,"pct":9.3,"pct_before":9.3,"points_all":0,"points":0,"sites_gained":0,"sites_lost":0,"compared":397,"denominator_entered":11,"denominator_left":13},{"id":"blocks_PerplexityBot","group":"Crawler policy","label":"robots.txt closes / to PerplexityBot","count":49,"of":408,"pct":12,"pct_before":12.4,"points_all":-0.4,"points":0,"sites_gained":0,"sites_lost":0,"compared":397,"denominator_entered":11,"denominator_left":13},{"id":"blocks_Google-Extended","group":"Crawler policy","label":"robots.txt closes / to Google-Extended","count":57,"of":408,"pct":14,"pct_before":13.9,"points_all":0.1,"points":0,"sites_gained":0,"sites_lost":0,"compared":397,"denominator_entered":11,"denominator_left":13},{"id":"blocks_Applebot-Extended","group":"Crawler policy","label":"robots.txt closes / to Applebot-Extended","count":51,"of":408,"pct":12.5,"pct_before":12.7,"points_all":-0.2,"points":0,"sites_gained":0,"sites_lost":0,"compared":397,"denominator_entered":11,"denominator_left":13},{"id":"blocks_CCBot","group":"Crawler policy","label":"robots.txt closes / to CCBot","count":78,"of":408,"pct":19.1,"pct_before":19.8,"points_all":-0.7,"points":0,"sites_gained":0,"sites_lost":0,"compared":397,"denominator_entered":11,"denominator_left":13},{"id":"blocks_Bytespider","group":"Crawler policy","label":"robots.txt closes / to Bytespider","count":78,"of":408,"pct":19.1,"pct_before":19.8,"points_all":-0.7,"points":0.3,"sites_gained":1,"sites_lost":0,"compared":397,"denominator_entered":11,"denominator_left":13},{"id":"blocks_Meta-ExternalAgent","group":"Crawler policy","label":"robots.txt closes / to Meta-ExternalAgent","count":62,"of":408,"pct":15.2,"pct_before":15.6,"points_all":-0.4,"points":0.3,"sites_gained":1,"sites_lost":0,"compared":397,"denominator_entered":11,"denominator_left":13},{"id":"blocks_Amazonbot","group":"Crawler policy","label":"robots.txt closes / to Amazonbot","count":54,"of":408,"pct":13.2,"pct_before":13.7,"points_all":-0.5,"points":0,"sites_gained":0,"sites_lost":0,"compared":397,"denominator_entered":11,"denominator_left":13},{"id":"llms_txt","group":"Machine files","label":"Serves /llms.txt","count":119,"of":518,"pct":23,"pct_before":22,"points_all":1,"points":0.8,"sites_gained":4,"sites_lost":0,"compared":507,"denominator_entered":11,"denominator_left":11},{"id":"agents_md","group":"Machine files","label":"Serves /agents.md","count":16,"of":518,"pct":3.1,"pct_before":3.1,"points_all":0,"points":0,"sites_gained":0,"sites_lost":0,"compared":507,"denominator_entered":11,"denominator_left":11},{"id":"agent_skills","group":"Capabilities","label":"Publishes an agent-skills index","count":13,"of":518,"pct":2.5,"pct_before":2.7,"points_all":-0.2,"points":-0.2,"sites_gained":0,"sites_lost":1,"compared":507,"denominator_entered":11,"denominator_left":11},{"id":"mcp_card","group":"Capabilities","label":"Publishes an MCP server card","count":13,"of":518,"pct":2.5,"pct_before":2.5,"points_all":0,"points":0,"sites_gained":1,"sites_lost":1,"compared":507,"denominator_entered":11,"denominator_left":11},{"id":"api_catalog","group":"Capabilities","label":"Publishes an API catalog (RFC 9727)","count":13,"of":518,"pct":2.5,"pct_before":2.3,"points_all":0.2,"points":0.2,"sites_gained":1,"sites_lost":0,"compared":507,"denominator_entered":11,"denominator_left":11},{"id":"a2a_card","group":"Capabilities","label":"Publishes an A2A agent card","count":3,"of":518,"pct":0.6,"pct_before":0.6,"points_all":0,"points":0,"sites_gained":0,"sites_lost":0,"compared":507,"denominator_entered":11,"denominator_left":11},{"id":"any_capability","group":"Capabilities","label":"Declares any agent capability (any of the four above)","count":17,"of":518,"pct":3.3,"pct_before":3.3,"points_all":0,"points":0,"sites_gained":1,"sites_lost":1,"compared":507,"denominator_entered":11,"denominator_left":11},{"id":"ai_refused_any","group":"Delivery parity","label":"At least one AI identity is refused (non-2xx) where the browser got 2xx","count":93,"of":518,"pct":18,"pct_before":19.1,"points_all":-1.1,"points":-1,"sites_gained":2,"sites_lost":7,"compared":507,"denominator_entered":11,"denominator_left":11},{"id":"ai_thin_any","group":"Delivery parity","label":"At least one AI identity gets under 70% of the browser's bytes (both 2xx)","count":20,"of":518,"pct":3.9,"pct_before":3.7,"points_all":0.2,"points":0.2,"sites_gained":3,"sites_lost":2,"compared":507,"denominator_entered":11,"denominator_left":11},{"id":"mcp_declared","group":"Endpoint rings","label":"Names an MCP endpoint (server card, llms.txt, agents.md, a homepage link, or /mcp answers like one)","count":55,"of":518,"pct":10.6,"pct_before":10.8,"points_all":-0.2,"points":-0.2,"sites_gained":0,"sites_lost":1,"compared":507,"denominator_entered":11,"denominator_left":11},{"id":"mcp_live","group":"Endpoint rings","label":"An MCP endpoint completes the handshake (initialize and tools/list; no tool called)","count":7,"of":518,"pct":1.4,"pct_before":1.4,"points_all":0,"points":0,"sites_gained":0,"sites_lost":0,"compared":507,"denominator_entered":11,"denominator_left":11},{"id":"mcp_live_of_declared","group":"Endpoint rings","label":"Of sites naming an MCP endpoint, one completes the handshake or asks for credentials","count":43,"of":55,"pct":78.2,"pct_before":76.8,"points_all":1.4,"points":0,"sites_gained":0,"sites_lost":0,"compared":55,"denominator_entered":0,"denominator_left":1},{"id":"a2a_declared","group":"Endpoint rings","label":"The A2A agent card names an endpoint url","count":2,"of":518,"pct":0.4,"pct_before":0.4,"points_all":0,"points":0,"sites_gained":0,"sites_lost":0,"compared":507,"denominator_entered":11,"denominator_left":11},{"id":"a2a_answers","group":"Endpoint rings","label":"Of A2A cards naming a url, the url answers (any non-5xx, not an HTML page)","count":1,"of":2,"pct":50,"pct_before":50,"points_all":0,"points":0,"sites_gained":0,"sites_lost":0,"compared":2,"denominator_entered":0,"denominator_left":0},{"id":"webmcp","group":"Endpoint rings","label":"Homepage carries WebMCP (navigator.modelContext, a WebMCP bridge, or an mcp+json block)","count":1,"of":518,"pct":0.2,"pct_before":0.2,"points_all":0,"points":0,"sites_gained":0,"sites_lost":0,"compared":507,"denominator_entered":11,"denominator_left":11},{"id":"sb_listed","group":"Safety","label":"Listed by Google Safe Browsing (all or some pages): browsers show a red warning","count":4,"of":319,"pct":1.3,"pct_before":1,"points_all":0.3,"points":null,"note":"not measured in the earlier run"},{"id":"cf_listed","group":"Safety","label":"Listed as malware or phishing by a public malware-blocking DNS resolver","count":0,"of":532,"pct":0,"pct_before":0,"points_all":0,"points":null,"note":"not measured in the earlier run"},{"id":"hsts_preloaded","group":"Safety","label":"On the browsers' HSTS preload list (HTTPS-only from the first visit)","count":93,"of":532,"pct":17.5,"pct_before":17.1,"points_all":0.4,"points":null,"note":"not measured in the earlier run"},{"id":"outside_scripts","group":"Safety","label":"Homepage loads scripts or styles from another domain","count":331,"of":532,"pct":62.2,"pct_before":62,"points_all":0.2,"points":null,"note":"not measured in the earlier run"},{"id":"outside_5plus","group":"Safety","label":"Homepage loads scripts or styles from 5 or more other domains","count":34,"of":532,"pct":6.4,"pct_before":6.4,"points_all":0,"points":null,"note":"not measured in the earlier run"},{"id":"any_integrity","group":"Safety","label":"Pins at least one outside script or style with integrity=","count":29,"of":331,"pct":8.8,"pct_before":9.4,"points_all":-0.6,"points":null,"note":"not measured in the earlier run"},{"id":"unpinned_cdn","group":"Safety","label":"Loads a library from a public CDN without integrity=","count":48,"of":532,"pct":9,"pct_before":9,"points_all":0,"points":null,"note":"not measured in the earlier run"},{"id":"hijacked_cdn","group":"Safety","label":"Still loads scripts from a CDN taken over in 2024 (polyfill.io and its sister CDNs)","count":0,"of":532,"pct":0,"pct_before":0,"points_all":0,"points":null,"note":"not measured in the earlier run"},{"id":"hack_signs","group":"Safety","label":"Homepage shows hidden spam links or cryptominer code","count":0,"of":532,"pct":0,"pct_before":0,"points_all":0,"points":null,"note":"not measured in the earlier run"},{"id":"refused_gptbot","group":"Delivery parity","label":"GPTBot refused where the browser got 2xx","count":79,"of":518,"pct":15.3,"pct_before":15.8,"points_all":-0.5,"points":-0.6,"sites_gained":2,"sites_lost":5,"compared":507,"denominator_entered":11,"denominator_left":11},{"id":"refused_claudebot","group":"Delivery parity","label":"ClaudeBot refused where the browser got 2xx","count":77,"of":518,"pct":14.9,"pct_before":15.6,"points_all":-0.7,"points":-0.4,"sites_gained":4,"sites_lost":6,"compared":507,"denominator_entered":11,"denominator_left":11},{"id":"refused_oaisearch","group":"Delivery parity","label":"OAI-SearchBot refused where the browser got 2xx","count":53,"of":518,"pct":10.2,"pct_before":11.2,"points_all":-1,"points":-0.6,"sites_gained":2,"sites_lost":5,"compared":507,"denominator_entered":11,"denominator_left":11},{"id":"refused_perplexity","group":"Delivery parity","label":"PerplexityBot refused where the browser got 2xx","count":71,"of":518,"pct":13.7,"pct_before":13.3,"points_all":0.4,"points":0.4,"sites_gained":8,"sites_lost":6,"compared":507,"denominator_entered":11,"denominator_left":11}],"movers":[{"id":"ai_refused_any","label":"At least one AI identity is refused (non-2xx) where the browser got 2xx","points":-1,"sites_gained":2,"sites_lost":7},{"id":"llms_txt","label":"Serves /llms.txt","points":0.8,"sites_gained":4,"sites_lost":0},{"id":"refused_gptbot","label":"GPTBot refused where the browser got 2xx","points":-0.6,"sites_gained":2,"sites_lost":5},{"id":"refused_oaisearch","label":"OAI-SearchBot refused where the browser got 2xx","points":-0.6,"sites_gained":2,"sites_lost":5},{"id":"aipref_train","label":"robots.txt names the AI-training policy token (__usage_ai-training, IETF AIPREF draft)","points":0.5,"sites_gained":2,"sites_lost":0}]},{"frame":"top10k","label":"top 10,000, daily","runs_this_week":7,"run":{"run":26,"finished_at":"2026-11-22T04:26:13.243Z"},"compared_with":{"run":19,"finished_at":"2026-11-15T03:08:26.484Z"},"sites":9776,"figures":[{"id":"reachable","group":"Reach","label":"Homepage answers CrawlCheck's census fetch (2xx after redirects)","count":5653,"of":9776,"pct":57.8,"pct_before":58,"points_all":-0.2,"points":-0.2,"sites_gained":148,"sites_lost":166,"compared":9776,"denominator_entered":0,"denominator_left":0},{"id":"refused_us","group":"Reach","label":"Homepage refuses CrawlCheck's census fetch (400, 401, 403 or 429): blocked or rate-limited, not down","count":1356,"of":9776,"pct":13.9,"pct_before":13.7,"points_all":0.2,"points":0.2,"sites_gained":95,"sites_lost":78,"compared":9776,"denominator_entered":0,"denominator_left":0},{"id":"no_site","group":"Reach","label":"No website answers at that name (5xx, 530, timeout or no answer)","count":2567,"of":9776,"pct":26.3,"pct_before":26.2,"points_all":0.1,"points":0,"sites_gained":83,"sites_lost":79,"compared":9776,"denominator_entered":0,"denominator_left":0},{"id":"robots_served","group":"Crawler policy","label":"Serves a robots.txt (200, not HTML)","count":4627,"of":5653,"pct":81.9,"pct_before":81.7,"points_all":0.2,"points":0,"sites_gained":29,"sites_lost":27,"compared":5505,"denominator_entered":148,"denominator_left":166},{"id":"robots_sitemap","group":"Crawler policy","label":"robots.txt names a sitemap","count":3252,"of":4627,"pct":70.3,"pct_before":70.4,"points_all":-0.1,"points":0.1,"sites_gained":9,"sites_lost":4,"compared":4480,"denominator_entered":147,"denominator_left":156},{"id":"content_signal","group":"Crawler policy","label":"robots.txt carries a Content-Signal line","count":302,"of":4627,"pct":6.5,"pct_before":6.5,"points_all":0,"points":0.1,"sites_gained":6,"sites_lost":1,"compared":4480,"denominator_entered":147,"denominator_left":156},{"id":"cs_ai_train_no","group":"Crawler policy","label":"Content-Signal says ai-train=no","count":230,"of":4627,"pct":5,"pct_before":5,"points_all":0,"points":0.1,"sites_gained":4,"sites_lost":1,"compared":4480,"denominator_entered":147,"denominator_left":156},{"id":"cs_ai_input_no","group":"Crawler policy","label":"Content-Signal says ai-input=no or search=no","count":21,"of":4627,"pct":0.5,"pct_before":0.5,"points_all":0,"points":0,"sites_gained":0,"sites_lost":1,"compared":4480,"denominator_entered":147,"denominator_left":156},{"id":"aipref_train","group":"Crawler policy","label":"robots.txt names the AI-training policy token (__usage_ai-training, IETF AIPREF draft)","count":51,"of":4623,"pct":1.1,"pct_before":0.4,"points_all":0.7,"points":0.3,"sites_gained":14,"sites_lost":0,"compared":4476,"denominator_entered":147,"denominator_left":156},{"id":"aipref_train_no","group":"Crawler policy","label":"robots.txt states a preference against AI training on / with that token","count":29,"of":4623,"pct":0.6,"pct_before":0.4,"points_all":0.2,"points":0.2,"sites_gained":10,"sites_lost":0,"compared":4476,"denominator_entered":147,"denominator_left":156},{"id":"blocks_any_ai","group":"Crawler policy","label":"robots.txt closes / to at least one of the 12 AI agents","count":1024,"of":4627,"pct":22.1,"pct_before":21.9,"points_all":0.2,"points":0.1,"sites_gained":7,"sites_lost":4,"compared":4480,"denominator_entered":147,"denominator_left":156},{"id":"blocks_GPTBot","group":"Crawler policy","label":"robots.txt closes / to GPTBot","count":749,"of":4627,"pct":16.2,"pct_before":16.1,"points_all":0.1,"points":0,"sites_gained":4,"sites_lost":5,"compared":4480,"denominator_entered":147,"denominator_left":156},{"id":"blocks_OAI-SearchBot","group":"Crawler policy","label":"robots.txt closes / to OAI-SearchBot","count":290,"of":4627,"pct":6.3,"pct_before":6,"points_all":0.3,"points":0,"sites_gained":3,"sites_lost":1,"compared":4480,"denominator_entered":147,"denominator_left":156},{"id":"blocks_ChatGPT-User","group":"Crawler policy","label":"robots.txt closes / to ChatGPT-User","count":372,"of":4627,"pct":8,"pct_before":7.8,"points_all":0.2,"points":0,"sites_gained":2,"sites_lost":2,"compared":4480,"denominator_entered":147,"denominator_left":156},{"id":"blocks_ClaudeBot","group":"Crawler policy","label":"robots.txt closes / to ClaudeBot","count":695,"of":4627,"pct":15,"pct_before":14.9,"points_all":0.1,"points":0,"sites_gained":4,"sites_lost":4,"compared":4480,"denominator_entered":147,"denominator_left":156},{"id":"blocks_Claude-SearchBot","group":"Crawler policy","label":"robots.txt closes / to Claude-SearchBot","count":277,"of":4627,"pct":6,"pct_before":5.8,"points_all":0.2,"points":0.1,"sites_gained":4,"sites_lost":1,"compared":4480,"denominator_entered":147,"denominator_left":156},{"id":"blocks_PerplexityBot","group":"Crawler policy","label":"robots.txt closes / to PerplexityBot","count":418,"of":4627,"pct":9,"pct_before":8.8,"points_all":0.2,"points":0,"sites_gained":1,"sites_lost":2,"compared":4480,"denominator_entered":147,"denominator_left":156},{"id":"blocks_Google-Extended","group":"Crawler policy","label":"robots.txt closes / to Google-Extended","count":619,"of":4627,"pct":13.4,"pct_before":13.3,"points_all":0.1,"points":-0.1,"sites_gained":1,"sites_lost":4,"compared":4480,"denominator_entered":147,"denominator_left":156},{"id":"blocks_Applebot-Extended","group":"Crawler policy","label":"robots.txt closes / to Applebot-Extended","count":615,"of":4627,"pct":13.3,"pct_before":13.1,"points_all":0.2,"points":0,"sites_gained":4,"sites_lost":4,"compared":4480,"denominator_entered":147,"denominator_left":156},{"id":"blocks_CCBot","group":"Crawler policy","label":"robots.txt closes / to CCBot","count":797,"of":4627,"pct":17.2,"pct_before":17.1,"points_all":0.1,"points":0,"sites_gained":4,"sites_lost":3,"compared":4480,"denominator_entered":147,"denominator_left":156},{"id":"blocks_Bytespider","group":"Crawler policy","label":"robots.txt closes / to Bytespider","count":793,"of":4627,"pct":17.1,"pct_before":17,"points_all":0.1,"points":0,"sites_gained":5,"sites_lost":3,"compared":4480,"denominator_entered":147,"denominator_left":156},{"id":"blocks_Meta-ExternalAgent","group":"Crawler policy","label":"robots.txt closes / to Meta-ExternalAgent","count":649,"of":4627,"pct":14,"pct_before":13.9,"points_all":0.1,"points":0.1,"sites_gained":8,"sites_lost":4,"compared":4480,"denominator_entered":147,"denominator_left":156},{"id":"blocks_Amazonbot","group":"Crawler policy","label":"robots.txt closes / to Amazonbot","count":632,"of":4627,"pct":13.7,"pct_before":13.4,"points_all":0.3,"points":0,"sites_gained":6,"sites_lost":4,"compared":4480,"denominator_entered":147,"denominator_left":156},{"id":"llms_txt","group":"Machine files","label":"Serves /llms.txt","count":960,"of":5653,"pct":17,"pct_before":17.1,"points_all":-0.1,"points":0.1,"sites_gained":13,"sites_lost":8,"compared":5505,"denominator_entered":148,"denominator_left":166},{"id":"agents_md","group":"Machine files","label":"Serves /agents.md","count":82,"of":5653,"pct":1.5,"pct_before":1.5,"points_all":0,"points":0,"sites_gained":3,"sites_lost":4,"compared":5505,"denominator_entered":148,"denominator_left":166},{"id":"agent_skills","group":"Capabilities","label":"Publishes an agent-skills index","count":60,"of":5653,"pct":1.1,"pct_before":1.1,"points_all":0,"points":0,"sites_gained":0,"sites_lost":1,"compared":5505,"denominator_entered":148,"denominator_left":166},{"id":"mcp_card","group":"Capabilities","label":"Publishes an MCP server card","count":68,"of":5653,"pct":1.2,"pct_before":1.2,"points_all":0,"points":0,"sites_gained":1,"sites_lost":1,"compared":5505,"denominator_entered":148,"denominator_left":166},{"id":"api_catalog","group":"Capabilities","label":"Publishes an API catalog (RFC 9727)","count":74,"of":5653,"pct":1.3,"pct_before":1.3,"points_all":0,"points":0,"sites_gained":1,"sites_lost":1,"compared":5505,"denominator_entered":148,"denominator_left":166},{"id":"a2a_card","group":"Capabilities","label":"Publishes an A2A agent card","count":16,"of":5653,"pct":0.3,"pct_before":0.3,"points_all":0,"points":0,"sites_gained":0,"sites_lost":2,"compared":5505,"denominator_entered":148,"denominator_left":166},{"id":"any_capability","group":"Capabilities","label":"Declares any agent capability (any of the four above)","count":89,"of":5653,"pct":1.6,"pct_before":1.6,"points_all":0,"points":0,"sites_gained":1,"sites_lost":1,"compared":5505,"denominator_entered":148,"denominator_left":166},{"id":"ai_refused_any","group":"Delivery parity","label":"At least one AI identity is refused (non-2xx) where the browser got 2xx","count":1268,"of":5653,"pct":22.4,"pct_before":22.8,"points_all":-0.4,"points":-0.3,"sites_gained":48,"sites_lost":66,"compared":5505,"denominator_entered":148,"denominator_left":166},{"id":"ai_thin_any","group":"Delivery parity","label":"At least one AI identity gets under 70% of the browser's bytes (both 2xx)","count":119,"of":5653,"pct":2.1,"pct_before":2.3,"points_all":-0.2,"points":-0.1,"sites_gained":8,"sites_lost":12,"compared":5505,"denominator_entered":148,"denominator_left":166},{"id":"mcp_declared","group":"Endpoint rings","label":"Names an MCP endpoint (server card, llms.txt, agents.md, a homepage link, or /mcp answers like one)","count":338,"of":5653,"pct":6,"pct_before":5.9,"points_all":0.1,"points":0.1,"sites_gained":9,"sites_lost":6,"compared":5505,"denominator_entered":148,"denominator_left":166},{"id":"mcp_live","group":"Endpoint rings","label":"An MCP endpoint completes the handshake (initialize and tools/list; no tool called)","count":55,"of":5653,"pct":1,"pct_before":1,"points_all":0,"points":0,"sites_gained":1,"sites_lost":0,"compared":5505,"denominator_entered":148,"denominator_left":166},{"id":"mcp_live_of_declared","group":"Endpoint rings","label":"Of sites naming an MCP endpoint, one completes the handshake or asks for credentials","count":264,"of":338,"pct":78.1,"pct_before":78.4,"points_all":-0.3,"points":-0.3,"sites_gained":0,"sites_lost":1,"compared":325,"denominator_entered":13,"denominator_left":8},{"id":"a2a_declared","group":"Endpoint rings","label":"The A2A agent card names an endpoint url","count":9,"of":5653,"pct":0.2,"pct_before":0.2,"points_all":0,"points":0,"sites_gained":0,"sites_lost":0,"compared":5505,"denominator_entered":148,"denominator_left":166},{"id":"a2a_answers","group":"Endpoint rings","label":"Of A2A cards naming a url, the url answers (any non-5xx, not an HTML page)","count":4,"of":9,"pct":44.4,"pct_before":44.4,"points_all":0,"points":0,"sites_gained":0,"sites_lost":0,"compared":9,"denominator_entered":0,"denominator_left":0},{"id":"webmcp","group":"Endpoint rings","label":"Homepage carries WebMCP (navigator.modelContext, a WebMCP bridge, or an mcp+json block)","count":5,"of":5653,"pct":0.1,"pct_before":0.1,"points_all":0,"points":0,"sites_gained":0,"sites_lost":0,"compared":5505,"denominator_entered":148,"denominator_left":166},{"id":"sb_listed","group":"Safety","label":"Listed by Google Safe Browsing (all or some pages): browsers show a red warning","count":34,"of":4143,"pct":0.8,"pct_before":0.9,"points_all":-0.1,"points":null,"note":"not measured in the earlier run"},{"id":"cf_listed","group":"Safety","label":"Listed as malware or phishing by a public malware-blocking DNS resolver","count":18,"of":5674,"pct":0.3,"pct_before":0.3,"points_all":0,"points":null,"note":"not measured in the earlier run"},{"id":"hsts_preloaded","group":"Safety","label":"On the browsers' HSTS preload list (HTTPS-only from the first visit)","count":452,"of":5671,"pct":8,"pct_before":8,"points_all":0,"points":null,"note":"not measured in the earlier run"},{"id":"outside_scripts","group":"Safety","label":"Homepage loads scripts or styles from another domain","count":3796,"of":5675,"pct":66.9,"pct_before":66.9,"points_all":0,"points":null,"note":"not measured in the earlier run"},{"id":"outside_5plus","group":"Safety","label":"Homepage loads scripts or styles from 5 or more other domains","count":596,"of":5675,"pct":10.5,"pct_before":10.8,"points_all":-0.3,"points":null,"note":"not measured in the earlier run"},{"id":"any_integrity","group":"Safety","label":"Pins at least one outside script or style with integrity=","count":374,"of":3796,"pct":9.9,"pct_before":9.8,"points_all":0.1,"points":null,"note":"not measured in the earlier run"},{"id":"unpinned_cdn","group":"Safety","label":"Loads a library from a public CDN without integrity=","count":729,"of":5675,"pct":12.8,"pct_before":13.2,"points_all":-0.4,"points":null,"note":"not measured in the earlier run"},{"id":"hijacked_cdn","group":"Safety","label":"Still loads scripts from a CDN taken over in 2024 (polyfill.io and its sister CDNs)","count":4,"of":5675,"pct":0.1,"pct_before":0.1,"points_all":0,"points":null,"note":"not measured in the earlier run"},{"id":"hack_signs","group":"Safety","label":"Homepage shows hidden spam links or cryptominer code","count":1,"of":5675,"pct":0,"pct_before":0,"points_all":0,"points":null,"note":"not measured in the earlier run"},{"id":"refused_gptbot","group":"Delivery parity","label":"GPTBot refused where the browser got 2xx","count":1080,"of":5653,"pct":19.1,"pct_before":19.1,"points_all":0,"points":0,"sites_gained":38,"sites_lost":39,"compared":5505,"denominator_entered":148,"denominator_left":166},{"id":"refused_claudebot","group":"Delivery parity","label":"ClaudeBot refused where the browser got 2xx","count":1123,"of":5653,"pct":19.9,"pct_before":19.7,"points_all":0.2,"points":0.2,"sites_gained":66,"sites_lost":55,"compared":5505,"denominator_entered":148,"denominator_left":166},{"id":"refused_oaisearch","group":"Delivery parity","label":"OAI-SearchBot refused where the browser got 2xx","count":699,"of":5653,"pct":12.4,"pct_before":12.6,"points_all":-0.2,"points":-0.1,"sites_gained":24,"sites_lost":31,"compared":5505,"denominator_entered":148,"denominator_left":166},{"id":"refused_perplexity","group":"Delivery parity","label":"PerplexityBot refused where the browser got 2xx","count":943,"of":5653,"pct":16.7,"pct_before":15.6,"points_all":1.1,"points":1.1,"sites_gained":88,"sites_lost":26,"compared":5505,"denominator_entered":148,"denominator_left":166}],"movers":[{"id":"refused_perplexity","label":"PerplexityBot refused where the browser got 2xx","points":1.1,"sites_gained":88,"sites_lost":26},{"id":"ai_refused_any","label":"At least one AI identity is refused (non-2xx) where the browser got 2xx","points":-0.3,"sites_gained":48,"sites_lost":66},{"id":"aipref_train","label":"robots.txt names the AI-training policy token (__usage_ai-training, IETF AIPREF draft)","points":0.3,"sites_gained":14,"sites_lost":0},{"id":"mcp_live_of_declared","label":"Of sites naming an MCP endpoint, one completes the handshake or asks for credentials","points":-0.3,"sites_gained":0,"sites_lost":1},{"id":"reachable","label":"Homepage answers CrawlCheck's census fetch (2xx after redirects)","points":-0.2,"sites_gained":148,"sites_lost":166}]}],"reading":"Each frame compares the last run that finished in this ISO week (UTC) with the last run that finished before it. Change is like-for-like: points = (sites gained - sites lost) / sites compared, counting only sites measured in both runs and inside the figure's denominator both times, so a site that did not answer in one run cannot move a figure. points_all is the plain difference of the two percentages, and denominator_entered / denominator_left count the sites that moved in or out of a figure's base (often a site that did not answer one run). No site is named; per-site records: https://crawlcheck.io/api/census/records.","method":"https://crawlcheck.io/api/census","page":"https://crawlcheck.io/data/census/weekly?week=2026-W47","editions":[{"week":"2026-W40","url":"https://crawlcheck.io/api/v1/census/weekly?week=2026-W40","archive":"https://crawlcheck.io/api/v1/census/weekly/2026-W40.json"},{"week":"2026-W41","url":"https://crawlcheck.io/api/v1/census/weekly?week=2026-W41","archive":"https://crawlcheck.io/api/v1/census/weekly/2026-W41.json"},{"week":"2026-W42","url":"https://crawlcheck.io/api/v1/census/weekly?week=2026-W42","archive":"https://crawlcheck.io/api/v1/census/weekly/2026-W42.json"},{"week":"2026-W43","url":"https://crawlcheck.io/api/v1/census/weekly?week=2026-W43","archive":"https://crawlcheck.io/api/v1/census/weekly/2026-W43.json"},{"week":"2026-W44","url":"https://crawlcheck.io/api/v1/census/weekly?week=2026-W44","archive":"https://crawlcheck.io/api/v1/census/weekly/2026-W44.json"},{"week":"2026-W45","url":"https://crawlcheck.io/api/v1/census/weekly?week=2026-W45","archive":"https://crawlcheck.io/api/v1/census/weekly/2026-W45.json"},{"week":"2026-W46","url":"https://crawlcheck.io/api/v1/census/weekly?week=2026-W46","archive":"https://crawlcheck.io/api/v1/census/weekly/2026-W46.json"}],"feed":"https://crawlcheck.io/api/v1/census/weekly/feed.xml","code":{"build_id":"9219fb519ff0eaf2","script_sha16":"b00e794a299c5a1b","state":"attested","deployed":{"at":"2026-11-21T18:41:28.011Z"},"build_log_entry":2899,"score_version":43,"scoring_policy_sha256":"3e08fccfbbce356dc82ecd302b32a226a36fc0fb3150a4ab72248fb5795372af","decision_rules_sha256":"18dd5636e0896692ab518050cb401744edea2e6f1086dff94fead314959bdc3e","check":"https://crawlcheck.io/api/v1/builds/9219fb519ff0eaf2"},"sha256":"76b782c63dbe32df15c6815a52bfb1c995e3bf0419a542b0d612064ff4ae819c","signature":{"v":1,"alg":"Ed25519","kid":"1hHmE3fo1rg5Ig70wuFtJ5wBc7G-sUTgUJkxyPWFRps","over":"sha256","message":"crawlcheck-data-v1\n76b782c63dbe32df15c6815a52bfb1c995e3bf0419a542b0d612064ff4ae819c","sig":"YxFkoQmPpUxgtRizjYK6AgBzQOpovANGAxtiocZA-XRyFvcVfOCoFIgd9h-AnMYldcmRLmrULMWyTIMLXUVgCg","signed_at":"2026-11-22T14:11:58.337Z","key":{"kid":"1hHmE3fo1rg5Ig70wuFtJ5wBc7G-sUTgUJkxyPWFRps","jwk":{"kty":"OKP","crv":"Ed25519","x":"5TmKCOx0TTHepq7HwyXBHvGQpVCSBd2lV8cLzYVqFxQ"},"published_in":"https://crawlcheck.io/.well-known/http-message-signatures-directory"},"canonical":"sha256 over canonical JSON of every field except sha256 and signature"}}