{"kind":"crawlcheck-census-weekly","v":1,"week":"2026-W44","state":"final","from":"2026-10-26T00:00:00.000Z","to":"2026-11-02T00:00:00.000Z","built_at":"2026-11-02T01:18:56.275Z","frames":[{"frame":"top1k","label":"top 1,000, daily","runs_this_week":7,"run":{"run":31,"finished_at":"2026-11-01T07:38:23.965Z"},"compared_with":{"run":24,"finished_at":"2026-10-25T07:16:41.288Z"},"sites":991,"figures":[{"id":"reachable","group":"Reach","label":"Homepage answers CrawlCheck's census fetch (2xx after redirects)","count":509,"of":991,"pct":51.4,"pct_before":51.4,"points_all":0,"points":0,"sites_gained":11,"sites_lost":11,"compared":991,"denominator_entered":0,"denominator_left":0},{"id":"refused_us","group":"Reach","label":"Homepage refuses CrawlCheck's census fetch (400, 401, 403 or 429): blocked or rate-limited, not down","count":122,"of":991,"pct":12.3,"pct_before":null,"points_all":null,"points":null,"note":"not measured in the earlier run"},{"id":"no_site","group":"Reach","label":"No website answers at that name (5xx, 530, timeout or no answer)","count":323,"of":991,"pct":32.6,"pct_before":null,"points_all":null,"points":null,"note":"not measured in the earlier run"},{"id":"robots_served","group":"Crawler policy","label":"Serves a robots.txt (200, not HTML)","count":399,"of":509,"pct":78.4,"pct_before":79,"points_all":-0.6,"points":-0.2,"sites_gained":4,"sites_lost":5,"compared":498,"denominator_entered":11,"denominator_left":11},{"id":"robots_sitemap","group":"Crawler policy","label":"robots.txt names a sitemap","count":288,"of":399,"pct":72.2,"pct_before":70.9,"points_all":1.3,"points":0.5,"sites_gained":2,"sites_lost":0,"compared":387,"denominator_entered":12,"denominator_left":15},{"id":"content_signal","group":"Crawler policy","label":"robots.txt carries a Content-Signal line","count":19,"of":399,"pct":4.8,"pct_before":4,"points_all":0.8,"points":0.5,"sites_gained":2,"sites_lost":0,"compared":387,"denominator_entered":12,"denominator_left":15},{"id":"cs_ai_train_no","group":"Crawler policy","label":"Content-Signal says ai-train=no","count":13,"of":399,"pct":3.3,"pct_before":2.7,"points_all":0.6,"points":0.3,"sites_gained":1,"sites_lost":0,"compared":387,"denominator_entered":12,"denominator_left":15},{"id":"cs_ai_input_no","group":"Crawler policy","label":"Content-Signal says ai-input=no or search=no","count":4,"of":399,"pct":1,"pct_before":1,"points_all":0,"points":0,"sites_gained":0,"sites_lost":0,"compared":387,"denominator_entered":12,"denominator_left":15},{"id":"blocks_any_ai","group":"Crawler policy","label":"robots.txt closes / to at least one of the 12 AI agents","count":101,"of":399,"pct":25.3,"pct_before":25.1,"points_all":0.2,"points":0.5,"sites_gained":2,"sites_lost":0,"compared":387,"denominator_entered":12,"denominator_left":15},{"id":"blocks_GPTBot","group":"Crawler policy","label":"robots.txt closes / to GPTBot","count":67,"of":399,"pct":16.8,"pct_before":16.9,"points_all":-0.1,"points":0.3,"sites_gained":1,"sites_lost":0,"compared":387,"denominator_entered":12,"denominator_left":15},{"id":"blocks_OAI-SearchBot","group":"Crawler policy","label":"robots.txt closes / to OAI-SearchBot","count":30,"of":399,"pct":7.5,"pct_before":8,"points_all":-0.5,"points":0,"sites_gained":0,"sites_lost":0,"compared":387,"denominator_entered":12,"denominator_left":15},{"id":"blocks_ChatGPT-User","group":"Crawler policy","label":"robots.txt closes / to ChatGPT-User","count":39,"of":399,"pct":9.8,"pct_before":10.2,"points_all":-0.4,"points":0,"sites_gained":0,"sites_lost":0,"compared":387,"denominator_entered":12,"denominator_left":15},{"id":"blocks_ClaudeBot","group":"Crawler policy","label":"robots.txt closes / to ClaudeBot","count":62,"of":399,"pct":15.5,"pct_before":15.9,"points_all":-0.4,"points":0.5,"sites_gained":2,"sites_lost":0,"compared":387,"denominator_entered":12,"denominator_left":15},{"id":"blocks_Claude-SearchBot","group":"Crawler policy","label":"robots.txt closes / to Claude-SearchBot","count":33,"of":399,"pct":8.3,"pct_before":8.5,"points_all":-0.2,"points":0.5,"sites_gained":2,"sites_lost":0,"compared":387,"denominator_entered":12,"denominator_left":15},{"id":"blocks_PerplexityBot","group":"Crawler policy","label":"robots.txt closes / to PerplexityBot","count":46,"of":399,"pct":11.5,"pct_before":12.2,"points_all":-0.7,"points":0,"sites_gained":0,"sites_lost":0,"compared":387,"denominator_entered":12,"denominator_left":15},{"id":"blocks_Google-Extended","group":"Crawler policy","label":"robots.txt closes / to Google-Extended","count":56,"of":399,"pct":14,"pct_before":13.7,"points_all":0.3,"points":0.3,"sites_gained":1,"sites_lost":0,"compared":387,"denominator_entered":12,"denominator_left":15},{"id":"blocks_Applebot-Extended","group":"Crawler policy","label":"robots.txt closes / to Applebot-Extended","count":50,"of":399,"pct":12.5,"pct_before":12.2,"points_all":0.3,"points":0.5,"sites_gained":2,"sites_lost":0,"compared":387,"denominator_entered":12,"denominator_left":15},{"id":"blocks_CCBot","group":"Crawler policy","label":"robots.txt closes / to CCBot","count":78,"of":399,"pct":19.5,"pct_before":19.9,"points_all":-0.4,"points":0.3,"sites_gained":1,"sites_lost":0,"compared":387,"denominator_entered":12,"denominator_left":15},{"id":"blocks_Bytespider","group":"Crawler policy","label":"robots.txt closes / to Bytespider","count":77,"of":399,"pct":19.3,"pct_before":19.2,"points_all":0.1,"points":0.5,"sites_gained":2,"sites_lost":0,"compared":387,"denominator_entered":12,"denominator_left":15},{"id":"blocks_Meta-ExternalAgent","group":"Crawler policy","label":"robots.txt closes / to Meta-ExternalAgent","count":60,"of":399,"pct":15,"pct_before":15.2,"points_all":-0.2,"points":0.5,"sites_gained":2,"sites_lost":0,"compared":387,"denominator_entered":12,"denominator_left":15},{"id":"blocks_Amazonbot","group":"Crawler policy","label":"robots.txt closes / to Amazonbot","count":56,"of":399,"pct":14,"pct_before":13.7,"points_all":0.3,"points":0.8,"sites_gained":3,"sites_lost":0,"compared":387,"denominator_entered":12,"denominator_left":15},{"id":"llms_txt","group":"Machine files","label":"Serves /llms.txt","count":110,"of":509,"pct":21.6,"pct_before":21.2,"points_all":0.4,"points":0,"sites_gained":1,"sites_lost":1,"compared":498,"denominator_entered":11,"denominator_left":11},{"id":"agents_md","group":"Machine files","label":"Serves /agents.md","count":12,"of":509,"pct":2.4,"pct_before":2.2,"points_all":0.2,"points":0,"sites_gained":0,"sites_lost":0,"compared":498,"denominator_entered":11,"denominator_left":11},{"id":"agent_skills","group":"Capabilities","label":"Publishes an agent-skills index","count":13,"of":509,"pct":2.6,"pct_before":2.4,"points_all":0.2,"points":0.2,"sites_gained":1,"sites_lost":0,"compared":498,"denominator_entered":11,"denominator_left":11},{"id":"mcp_card","group":"Capabilities","label":"Publishes an MCP server card","count":13,"of":509,"pct":2.6,"pct_before":2.6,"points_all":0,"points":0,"sites_gained":0,"sites_lost":0,"compared":498,"denominator_entered":11,"denominator_left":11},{"id":"api_catalog","group":"Capabilities","label":"Publishes an API catalog (RFC 9727)","count":12,"of":509,"pct":2.4,"pct_before":2.2,"points_all":0.2,"points":0.2,"sites_gained":1,"sites_lost":0,"compared":498,"denominator_entered":11,"denominator_left":11},{"id":"a2a_card","group":"Capabilities","label":"Publishes an A2A agent card","count":3,"of":509,"pct":0.6,"pct_before":0.6,"points_all":0,"points":0,"sites_gained":0,"sites_lost":0,"compared":498,"denominator_entered":11,"denominator_left":11},{"id":"any_capability","group":"Capabilities","label":"Declares any agent capability (any of the four above)","count":17,"of":509,"pct":3.3,"pct_before":2.9,"points_all":0.4,"points":0.4,"sites_gained":2,"sites_lost":0,"compared":498,"denominator_entered":11,"denominator_left":11},{"id":"ai_refused_any","group":"Delivery parity","label":"At least one AI identity is refused (non-2xx) where the browser got 2xx","count":96,"of":509,"pct":18.9,"pct_before":17.7,"points_all":1.2,"points":0.8,"sites_gained":9,"sites_lost":5,"compared":498,"denominator_entered":11,"denominator_left":11},{"id":"ai_thin_any","group":"Delivery parity","label":"At least one AI identity gets under 70% of the browser's bytes (both 2xx)","count":20,"of":509,"pct":3.9,"pct_before":3.7,"points_all":0.2,"points":0.2,"sites_gained":2,"sites_lost":1,"compared":498,"denominator_entered":11,"denominator_left":11},{"id":"mcp_declared","group":"Endpoint rings","label":"Names an MCP endpoint (server card, llms.txt, agents.md, a homepage link, or /mcp answers like one)","count":52,"of":509,"pct":10.2,"pct_before":10.8,"points_all":-0.6,"points":-0.4,"sites_gained":0,"sites_lost":2,"compared":498,"denominator_entered":11,"denominator_left":11},{"id":"mcp_live","group":"Endpoint rings","label":"An MCP endpoint completes the handshake (initialize and tools/list; no tool called)","count":8,"of":509,"pct":1.6,"pct_before":1.6,"points_all":0,"points":0,"sites_gained":0,"sites_lost":0,"compared":498,"denominator_entered":11,"denominator_left":11},{"id":"mcp_live_of_declared","group":"Endpoint rings","label":"Of sites naming an MCP endpoint, one completes the handshake or asks for credentials","count":42,"of":52,"pct":80.8,"pct_before":76.4,"points_all":4.4,"points":3.8,"sites_gained":2,"sites_lost":0,"compared":52,"denominator_entered":0,"denominator_left":3},{"id":"a2a_declared","group":"Endpoint rings","label":"The A2A agent card names an endpoint url","count":2,"of":509,"pct":0.4,"pct_before":0.4,"points_all":0,"points":0,"sites_gained":0,"sites_lost":0,"compared":498,"denominator_entered":11,"denominator_left":11},{"id":"a2a_answers","group":"Endpoint rings","label":"Of A2A cards naming a url, the url answers (any non-5xx, not an HTML page)","count":1,"of":2,"pct":50,"pct_before":50,"points_all":0,"points":0,"sites_gained":0,"sites_lost":0,"compared":2,"denominator_entered":0,"denominator_left":0},{"id":"webmcp","group":"Endpoint rings","label":"Homepage carries WebMCP (navigator.modelContext, a WebMCP bridge, or an mcp+json block)","count":1,"of":509,"pct":0.2,"pct_before":0.2,"points_all":0,"points":0,"sites_gained":0,"sites_lost":0,"compared":498,"denominator_entered":11,"denominator_left":11},{"id":"refused_gptbot","group":"Delivery parity","label":"GPTBot refused where the browser got 2xx","count":84,"of":509,"pct":16.5,"pct_before":14.9,"points_all":1.6,"points":1.2,"sites_gained":6,"sites_lost":0,"compared":498,"denominator_entered":11,"denominator_left":11},{"id":"refused_claudebot","group":"Delivery parity","label":"ClaudeBot refused where the browser got 2xx","count":76,"of":509,"pct":14.9,"pct_before":13.9,"points_all":1,"points":0.8,"sites_gained":7,"sites_lost":3,"compared":498,"denominator_entered":11,"denominator_left":11},{"id":"refused_oaisearch","group":"Delivery parity","label":"OAI-SearchBot refused where the browser got 2xx","count":55,"of":509,"pct":10.8,"pct_before":10.2,"points_all":0.6,"points":0.2,"sites_gained":3,"sites_lost":2,"compared":498,"denominator_entered":11,"denominator_left":11},{"id":"refused_perplexity","group":"Delivery parity","label":"PerplexityBot refused where the browser got 2xx","count":60,"of":509,"pct":11.8,"pct_before":11,"points_all":0.8,"points":0.4,"sites_gained":5,"sites_lost":3,"compared":498,"denominator_entered":11,"denominator_left":11}],"movers":[{"id":"mcp_live_of_declared","label":"Of sites naming an MCP endpoint, one completes the handshake or asks for credentials","points":3.8,"sites_gained":2,"sites_lost":0},{"id":"refused_gptbot","label":"GPTBot refused where the browser got 2xx","points":1.2,"sites_gained":6,"sites_lost":0},{"id":"ai_refused_any","label":"At least one AI identity is refused (non-2xx) where the browser got 2xx","points":0.8,"sites_gained":9,"sites_lost":5},{"id":"refused_claudebot","label":"ClaudeBot refused where the browser got 2xx","points":0.8,"sites_gained":7,"sites_lost":3},{"id":"blocks_Amazonbot","label":"robots.txt closes / to Amazonbot","points":0.8,"sites_gained":3,"sites_lost":0}]},{"frame":"top10k","label":"top 10,000, daily","runs_this_week":1,"run":{"run":5,"finished_at":"2026-10-31T21:02:55.469Z"},"compared_with":{"run":4,"finished_at":"2026-10-24T23:42:20.214Z"},"sites":9776,"figures":[{"id":"reachable","group":"Reach","label":"Homepage answers CrawlCheck's census fetch (2xx after redirects)","count":5621,"of":9776,"pct":57.5,"pct_before":56.8,"points_all":0.7,"points":0.7,"sites_gained":228,"sites_lost":160,"compared":9776,"denominator_entered":0,"denominator_left":0},{"id":"refused_us","group":"Reach","label":"Homepage refuses CrawlCheck's census fetch (400, 401, 403 or 429): blocked or rate-limited, not down","count":1338,"of":9776,"pct":13.7,"pct_before":null,"points_all":null,"points":null,"note":"not measured in the earlier run"},{"id":"no_site","group":"Reach","label":"No website answers at that name (5xx, 530, timeout or no answer)","count":2624,"of":9776,"pct":26.8,"pct_before":null,"points_all":null,"points":null,"note":"not measured in the earlier run"},{"id":"robots_served","group":"Crawler policy","label":"Serves a robots.txt (200, not HTML)","count":4594,"of":5621,"pct":81.7,"pct_before":81.1,"points_all":0.6,"points":0.5,"sites_gained":63,"sites_lost":36,"compared":5393,"denominator_entered":228,"denominator_left":160},{"id":"robots_sitemap","group":"Crawler policy","label":"robots.txt names a sitemap","count":3219,"of":4594,"pct":70.1,"pct_before":69.9,"points_all":0.2,"points":0.1,"sites_gained":15,"sites_lost":10,"compared":4353,"denominator_entered":241,"denominator_left":148},{"id":"content_signal","group":"Crawler policy","label":"robots.txt carries a Content-Signal line","count":283,"of":4594,"pct":6.2,"pct_before":4.3,"points_all":1.9,"points":1.7,"sites_gained":75,"sites_lost":0,"compared":4353,"denominator_entered":241,"denominator_left":148},{"id":"cs_ai_train_no","group":"Crawler policy","label":"Content-Signal says ai-train=no","count":222,"of":4594,"pct":4.8,"pct_before":3,"points_all":1.8,"points":1.7,"sites_gained":74,"sites_lost":0,"compared":4353,"denominator_entered":241,"denominator_left":148},{"id":"cs_ai_input_no","group":"Crawler policy","label":"Content-Signal says ai-input=no or search=no","count":21,"of":4594,"pct":0.5,"pct_before":0.5,"points_all":0,"points":0,"sites_gained":0,"sites_lost":0,"compared":4353,"denominator_entered":241,"denominator_left":148},{"id":"blocks_any_ai","group":"Crawler policy","label":"robots.txt closes / to at least one of the 12 AI agents","count":1004,"of":4594,"pct":21.9,"pct_before":19.8,"points_all":2.1,"points":1.8,"sites_gained":81,"sites_lost":1,"compared":4353,"denominator_entered":241,"denominator_left":148},{"id":"blocks_GPTBot","group":"Crawler policy","label":"robots.txt closes / to GPTBot","count":742,"of":4594,"pct":16.2,"pct_before":14.3,"points_all":1.9,"points":1.6,"sites_gained":72,"sites_lost":1,"compared":4353,"denominator_entered":241,"denominator_left":148},{"id":"blocks_OAI-SearchBot","group":"Crawler policy","label":"robots.txt closes / to OAI-SearchBot","count":284,"of":4594,"pct":6.2,"pct_before":6.2,"points_all":0,"points":0,"sites_gained":4,"sites_lost":3,"compared":4353,"denominator_entered":241,"denominator_left":148},{"id":"blocks_ChatGPT-User","group":"Crawler policy","label":"robots.txt closes / to ChatGPT-User","count":370,"of":4594,"pct":8.1,"pct_before":8.1,"points_all":0,"points":0,"sites_gained":5,"sites_lost":3,"compared":4353,"denominator_entered":241,"denominator_left":148},{"id":"blocks_ClaudeBot","group":"Crawler policy","label":"robots.txt closes / to ClaudeBot","count":685,"of":4594,"pct":14.9,"pct_before":13.1,"points_all":1.8,"points":1.8,"sites_gained":78,"sites_lost":1,"compared":4353,"denominator_entered":241,"denominator_left":148},{"id":"blocks_Claude-SearchBot","group":"Crawler policy","label":"robots.txt closes / to Claude-SearchBot","count":262,"of":4594,"pct":5.7,"pct_before":5.8,"points_all":-0.1,"points":0.1,"sites_gained":6,"sites_lost":2,"compared":4353,"denominator_entered":241,"denominator_left":148},{"id":"blocks_PerplexityBot","group":"Crawler policy","label":"robots.txt closes / to PerplexityBot","count":426,"of":4594,"pct":9.3,"pct_before":9.1,"points_all":0.2,"points":0.3,"sites_gained":17,"sites_lost":3,"compared":4353,"denominator_entered":241,"denominator_left":148},{"id":"blocks_Google-Extended","group":"Crawler policy","label":"robots.txt closes / to Google-Extended","count":616,"of":4594,"pct":13.4,"pct_before":11.5,"points_all":1.9,"points":1.9,"sites_gained":86,"sites_lost":4,"compared":4353,"denominator_entered":241,"denominator_left":148},{"id":"blocks_Applebot-Extended","group":"Crawler policy","label":"robots.txt closes / to Applebot-Extended","count":594,"of":4594,"pct":12.9,"pct_before":11.1,"points_all":1.8,"points":1.8,"sites_gained":78,"sites_lost":1,"compared":4353,"denominator_entered":241,"denominator_left":148},{"id":"blocks_CCBot","group":"Crawler policy","label":"robots.txt closes / to CCBot","count":788,"of":4594,"pct":17.2,"pct_before":15.1,"points_all":2.1,"points":2,"sites_gained":88,"sites_lost":1,"compared":4353,"denominator_entered":241,"denominator_left":148},{"id":"blocks_Bytespider","group":"Crawler policy","label":"robots.txt closes / to Bytespider","count":776,"of":4594,"pct":16.9,"pct_before":14.6,"points_all":2.3,"points":2,"sites_gained":89,"sites_lost":1,"compared":4353,"denominator_entered":241,"denominator_left":148},{"id":"blocks_Meta-ExternalAgent","group":"Crawler policy","label":"robots.txt closes / to Meta-ExternalAgent","count":631,"of":4594,"pct":13.7,"pct_before":11.7,"points_all":2,"points":1.8,"sites_gained":78,"sites_lost":1,"compared":4353,"denominator_entered":241,"denominator_left":148},{"id":"blocks_Amazonbot","group":"Crawler policy","label":"robots.txt closes / to Amazonbot","count":604,"of":4594,"pct":13.1,"pct_before":11.1,"points_all":2,"points":2,"sites_gained":87,"sites_lost":2,"compared":4353,"denominator_entered":241,"denominator_left":148},{"id":"llms_txt","group":"Machine files","label":"Serves /llms.txt","count":932,"of":5621,"pct":16.6,"pct_before":16.2,"points_all":0.4,"points":0.3,"sites_gained":18,"sites_lost":4,"compared":5393,"denominator_entered":228,"denominator_left":160},{"id":"agents_md","group":"Machine files","label":"Serves /agents.md","count":73,"of":5621,"pct":1.3,"pct_before":1.3,"points_all":0,"points":0,"sites_gained":0,"sites_lost":0,"compared":5393,"denominator_entered":228,"denominator_left":160},{"id":"agent_skills","group":"Capabilities","label":"Publishes an agent-skills index","count":56,"of":5621,"pct":1,"pct_before":1,"points_all":0,"points":0,"sites_gained":2,"sites_lost":1,"compared":5393,"denominator_entered":228,"denominator_left":160},{"id":"mcp_card","group":"Capabilities","label":"Publishes an MCP server card","count":57,"of":5621,"pct":1,"pct_before":1.1,"points_all":-0.1,"points":0,"sites_gained":1,"sites_lost":1,"compared":5393,"denominator_entered":228,"denominator_left":160},{"id":"api_catalog","group":"Capabilities","label":"Publishes an API catalog (RFC 9727)","count":63,"of":5621,"pct":1.1,"pct_before":1.2,"points_all":-0.1,"points":0,"sites_gained":2,"sites_lost":3,"compared":5393,"denominator_entered":228,"denominator_left":160},{"id":"a2a_card","group":"Capabilities","label":"Publishes an A2A agent card","count":14,"of":5621,"pct":0.2,"pct_before":0.3,"points_all":-0.1,"points":0,"sites_gained":0,"sites_lost":2,"compared":5393,"denominator_entered":228,"denominator_left":160},{"id":"any_capability","group":"Capabilities","label":"Declares any agent capability (any of the four above)","count":78,"of":5621,"pct":1.4,"pct_before":1.4,"points_all":0,"points":0,"sites_gained":3,"sites_lost":3,"compared":5393,"denominator_entered":228,"denominator_left":160},{"id":"ai_refused_any","group":"Delivery parity","label":"At least one AI identity is refused (non-2xx) where the browser got 2xx","count":1331,"of":5621,"pct":23.7,"pct_before":23.3,"points_all":0.4,"points":0.3,"sites_gained":82,"sites_lost":67,"compared":5393,"denominator_entered":228,"denominator_left":160},{"id":"ai_thin_any","group":"Delivery parity","label":"At least one AI identity gets under 70% of the browser's bytes (both 2xx)","count":143,"of":5621,"pct":2.5,"pct_before":2.5,"points_all":0,"points":0.1,"sites_gained":12,"sites_lost":9,"compared":5393,"denominator_entered":228,"denominator_left":160},{"id":"mcp_declared","group":"Endpoint rings","label":"Names an MCP endpoint (server card, llms.txt, agents.md, a homepage link, or /mcp answers like one)","count":315,"of":5621,"pct":5.6,"pct_before":5.6,"points_all":0,"points":0,"sites_gained":9,"sites_lost":7,"compared":5393,"denominator_entered":228,"denominator_left":160},{"id":"mcp_live","group":"Endpoint rings","label":"An MCP endpoint completes the handshake (initialize and tools/list; no tool called)","count":51,"of":5621,"pct":0.9,"pct_before":0.9,"points_all":0,"points":-0.1,"sites_gained":1,"sites_lost":4,"compared":5393,"denominator_entered":228,"denominator_left":160},{"id":"mcp_live_of_declared","group":"Endpoint rings","label":"Of sites naming an MCP endpoint, one completes the handshake or asks for credentials","count":250,"of":315,"pct":79.4,"pct_before":79.1,"points_all":0.3,"points":1,"sites_gained":3,"sites_lost":0,"compared":297,"denominator_entered":18,"denominator_left":14},{"id":"a2a_declared","group":"Endpoint rings","label":"The A2A agent card names an endpoint url","count":8,"of":5621,"pct":0.1,"pct_before":0.2,"points_all":-0.1,"points":0,"sites_gained":0,"sites_lost":1,"compared":5393,"denominator_entered":228,"denominator_left":160},{"id":"a2a_answers","group":"Endpoint rings","label":"Of A2A cards naming a url, the url answers (any non-5xx, not an HTML page)","count":4,"of":8,"pct":50,"pct_before":44.4,"points_all":5.6,"points":0,"sites_gained":0,"sites_lost":0,"compared":8,"denominator_entered":0,"denominator_left":1},{"id":"webmcp","group":"Endpoint rings","label":"Homepage carries WebMCP (navigator.modelContext, a WebMCP bridge, or an mcp+json block)","count":5,"of":5621,"pct":0.1,"pct_before":0.1,"points_all":0,"points":0,"sites_gained":0,"sites_lost":0,"compared":5393,"denominator_entered":228,"denominator_left":160},{"id":"refused_gptbot","group":"Delivery parity","label":"GPTBot refused where the browser got 2xx","count":1106,"of":5621,"pct":19.7,"pct_before":20,"points_all":-0.3,"points":-0.4,"sites_gained":37,"sites_lost":57,"compared":5393,"denominator_entered":228,"denominator_left":160},{"id":"refused_claudebot","group":"Delivery parity","label":"ClaudeBot refused where the browser got 2xx","count":1143,"of":5621,"pct":20.3,"pct_before":20.2,"points_all":0.1,"points":0.1,"sites_gained":58,"sites_lost":55,"compared":5393,"denominator_entered":228,"denominator_left":160},{"id":"refused_oaisearch","group":"Delivery parity","label":"OAI-SearchBot refused where the browser got 2xx","count":745,"of":5621,"pct":13.3,"pct_before":12.6,"points_all":0.7,"points":0.3,"sites_gained":46,"sites_lost":29,"compared":5393,"denominator_entered":228,"denominator_left":160},{"id":"refused_perplexity","group":"Delivery parity","label":"PerplexityBot refused where the browser got 2xx","count":751,"of":5621,"pct":13.4,"pct_before":13.2,"points_all":0.2,"points":-0.1,"sites_gained":27,"sites_lost":32,"compared":5393,"denominator_entered":228,"denominator_left":160}],"movers":[{"id":"blocks_Bytespider","label":"robots.txt closes / to Bytespider","points":2,"sites_gained":89,"sites_lost":1},{"id":"blocks_CCBot","label":"robots.txt closes / to CCBot","points":2,"sites_gained":88,"sites_lost":1},{"id":"blocks_Amazonbot","label":"robots.txt closes / to Amazonbot","points":2,"sites_gained":87,"sites_lost":2},{"id":"blocks_Google-Extended","label":"robots.txt closes / to Google-Extended","points":1.9,"sites_gained":86,"sites_lost":4},{"id":"blocks_any_ai","label":"robots.txt closes / to at least one of the 12 AI agents","points":1.8,"sites_gained":81,"sites_lost":1}]}],"reading":"Each frame compares the last run that finished in this ISO week (UTC) with the last run that finished before it. Change is like-for-like: points = (sites gained - sites lost) / sites compared, counting only sites measured in both runs and inside the figure's denominator both times, so a site that did not answer in one run cannot move a figure. points_all is the plain difference of the two percentages, and denominator_entered / denominator_left count the sites that moved in or out of a figure's base (often a site that did not answer one run). No site is named; per-site records: https://crawlcheck.io/api/census/records.","method":"https://crawlcheck.io/api/census","page":"https://crawlcheck.io/data/census/weekly?week=2026-W44","sealed_at":"2026-11-02T01:18:58.285Z","code":{"version_id":"c2b3ba44-bde5-45f3-bf8f-bc4dd92e08f6","script_sha16":"228ea33c1e3c84ca","state":"attested","deployed":{"at":"2026-11-02T01:14:02.163Z","commit":"c10fb44d6ac0","run":"41371305895"},"build_log_entry":1250,"score_version":34,"scoring_policy_sha256":"76ee7526c402d711f2c976ad88ae13d4d3411aaa9d5ca3dc87ab3102396b1370","decision_rules_sha256":"18dd5636e0896692ab518050cb401744edea2e6f1086dff94fead314959bdc3e","check":"https://crawlcheck.io/api/v1/builds/c2b3ba44-bde5-45f3-bf8f-bc4dd92e08f6"},"sha256":"45742c834da1315b273752e6df4097284be421d5e2734b1a37b6fded3a3daf8c","signature":{"v":1,"alg":"Ed25519","kid":"1hHmE3fo1rg5Ig70wuFtJ5wBc7G-sUTgUJkxyPWFRps","over":"sha256","message":"crawlcheck-data-v1\n45742c834da1315b273752e6df4097284be421d5e2734b1a37b6fded3a3daf8c","sig":"mqabExiYu3u2b8_LeC8I0qO057GMyAG95nLgFLV9MjBKpSgBS1fievwy9gRgiZbEupNUmaAT4eNv9_EVypTjBg","signed_at":"2026-11-02T01:19:00.496Z","key":{"kid":"1hHmE3fo1rg5Ig70wuFtJ5wBc7G-sUTgUJkxyPWFRps","jwk":{"kty":"OKP","crv":"Ed25519","x":"5TmKCOx0TTHepq7HwyXBHvGQpVCSBd2lV8cLzYVqFxQ"},"published_in":"https://crawlcheck.io/.well-known/http-message-signatures-directory"},"canonical":"sha256 over canonical JSON of every field except sha256 and signature"}}