{"kind":"crawlcheck-census-weekly","v":1,"week":"2026-W41","state":"final","from":"2026-10-05T00:00:00.000Z","to":"2026-10-12T00:00:00.000Z","built_at":"2026-10-12T00:17:14.383Z","frames":[{"frame":"top1k","label":"top 1,000, daily","runs_this_week":7,"run":{"run":10,"finished_at":"2026-10-11T06:04:35.685Z"},"compared_with":{"run":3,"finished_at":"2026-10-04T05:00:02.620Z"},"sites":991,"figures":[{"id":"reachable","group":"Reach","label":"Homepage answers a browser (2xx after redirects)","count":515,"of":991,"pct":52,"pct_before":51.5,"points_all":0.5,"points":0.5,"sites_gained":16,"sites_lost":11,"compared":991,"denominator_entered":0,"denominator_left":0},{"id":"robots_served","group":"Crawler policy","label":"Serves a robots.txt (200, not HTML)","count":414,"of":515,"pct":80.4,"pct_before":79,"points_all":1.4,"points":1,"sites_gained":5,"sites_lost":0,"compared":499,"denominator_entered":16,"denominator_left":11},{"id":"robots_sitemap","group":"Crawler policy","label":"robots.txt names a sitemap","count":288,"of":414,"pct":69.6,"pct_before":69.7,"points_all":-0.1,"points":-0.3,"sites_gained":0,"sites_lost":1,"compared":395,"denominator_entered":19,"denominator_left":8},{"id":"content_signal","group":"Crawler policy","label":"robots.txt carries a Content-Signal line","count":15,"of":414,"pct":3.6,"pct_before":3.7,"points_all":-0.1,"points":0,"sites_gained":0,"sites_lost":0,"compared":395,"denominator_entered":19,"denominator_left":8},{"id":"cs_ai_train_no","group":"Crawler policy","label":"Content-Signal says ai-train=no","count":9,"of":414,"pct":2.2,"pct_before":2.2,"points_all":0,"points":0,"sites_gained":0,"sites_lost":0,"compared":395,"denominator_entered":19,"denominator_left":8},{"id":"cs_ai_input_no","group":"Crawler policy","label":"Content-Signal says ai-input=no or search=no","count":4,"of":414,"pct":1,"pct_before":1,"points_all":0,"points":0,"sites_gained":0,"sites_lost":0,"compared":395,"denominator_entered":19,"denominator_left":8},{"id":"blocks_any_ai","group":"Crawler policy","label":"robots.txt closes / to at least one of the 12 AI agents","count":101,"of":414,"pct":24.4,"pct_before":24.6,"points_all":-0.2,"points":0,"sites_gained":0,"sites_lost":0,"compared":395,"denominator_entered":19,"denominator_left":8},{"id":"blocks_GPTBot","group":"Crawler policy","label":"robots.txt closes / to GPTBot","count":67,"of":414,"pct":16.2,"pct_before":16.6,"points_all":-0.4,"points":-0.3,"sites_gained":0,"sites_lost":1,"compared":395,"denominator_entered":19,"denominator_left":8},{"id":"blocks_OAI-SearchBot","group":"Crawler policy","label":"robots.txt closes / to OAI-SearchBot","count":33,"of":414,"pct":8,"pct_before":8.2,"points_all":-0.2,"points":0,"sites_gained":0,"sites_lost":0,"compared":395,"denominator_entered":19,"denominator_left":8},{"id":"blocks_ChatGPT-User","group":"Crawler policy","label":"robots.txt closes / to ChatGPT-User","count":42,"of":414,"pct":10.1,"pct_before":10.4,"points_all":-0.3,"points":0,"sites_gained":0,"sites_lost":0,"compared":395,"denominator_entered":19,"denominator_left":8},{"id":"blocks_ClaudeBot","group":"Crawler policy","label":"robots.txt closes / to ClaudeBot","count":63,"of":414,"pct":15.2,"pct_before":15.4,"points_all":-0.2,"points":-0.3,"sites_gained":0,"sites_lost":1,"compared":395,"denominator_entered":19,"denominator_left":8},{"id":"blocks_Claude-SearchBot","group":"Crawler policy","label":"robots.txt closes / to Claude-SearchBot","count":35,"of":414,"pct":8.5,"pct_before":8.4,"points_all":0.1,"points":0,"sites_gained":0,"sites_lost":0,"compared":395,"denominator_entered":19,"denominator_left":8},{"id":"blocks_PerplexityBot","group":"Crawler policy","label":"robots.txt closes / to PerplexityBot","count":49,"of":414,"pct":11.8,"pct_before":12.4,"points_all":-0.6,"points":-0.3,"sites_gained":0,"sites_lost":1,"compared":395,"denominator_entered":19,"denominator_left":8},{"id":"blocks_Google-Extended","group":"Crawler policy","label":"robots.txt closes / to Google-Extended","count":55,"of":414,"pct":13.3,"pct_before":13.9,"points_all":-0.6,"points":-0.3,"sites_gained":0,"sites_lost":1,"compared":395,"denominator_entered":19,"denominator_left":8},{"id":"blocks_Applebot-Extended","group":"Crawler policy","label":"robots.txt closes / to Applebot-Extended","count":48,"of":414,"pct":11.6,"pct_before":11.9,"points_all":-0.3,"points":-0.3,"sites_gained":0,"sites_lost":1,"compared":395,"denominator_entered":19,"denominator_left":8},{"id":"blocks_CCBot","group":"Crawler policy","label":"robots.txt closes / to CCBot","count":80,"of":414,"pct":19.3,"pct_before":19.6,"points_all":-0.3,"points":0,"sites_gained":0,"sites_lost":0,"compared":395,"denominator_entered":19,"denominator_left":8},{"id":"blocks_Bytespider","group":"Crawler policy","label":"robots.txt closes / to Bytespider","count":77,"of":414,"pct":18.6,"pct_before":18.9,"points_all":-0.3,"points":0,"sites_gained":0,"sites_lost":0,"compared":395,"denominator_entered":19,"denominator_left":8},{"id":"blocks_Meta-ExternalAgent","group":"Crawler policy","label":"robots.txt closes / to Meta-ExternalAgent","count":62,"of":414,"pct":15,"pct_before":15.1,"points_all":-0.1,"points":0,"sites_gained":0,"sites_lost":0,"compared":395,"denominator_entered":19,"denominator_left":8},{"id":"blocks_Amazonbot","group":"Crawler policy","label":"robots.txt closes / to Amazonbot","count":53,"of":414,"pct":12.8,"pct_before":12.9,"points_all":-0.1,"points":-0.3,"sites_gained":0,"sites_lost":1,"compared":395,"denominator_entered":19,"denominator_left":8},{"id":"llms_txt","group":"Machine files","label":"Serves /llms.txt","count":107,"of":515,"pct":20.8,"pct_before":20.6,"points_all":0.2,"points":0.4,"sites_gained":3,"sites_lost":1,"compared":499,"denominator_entered":16,"denominator_left":11},{"id":"agents_md","group":"Machine files","label":"Serves /agents.md","count":14,"of":515,"pct":2.7,"pct_before":2.7,"points_all":0,"points":0,"sites_gained":0,"sites_lost":0,"compared":499,"denominator_entered":16,"denominator_left":11},{"id":"agent_skills","group":"Capabilities","label":"Publishes an agent-skills index","count":12,"of":515,"pct":2.3,"pct_before":2,"points_all":0.3,"points":0.2,"sites_gained":1,"sites_lost":0,"compared":499,"denominator_entered":16,"denominator_left":11},{"id":"mcp_card","group":"Capabilities","label":"Publishes an MCP server card","count":12,"of":515,"pct":2.3,"pct_before":2.4,"points_all":-0.1,"points":0,"sites_gained":0,"sites_lost":0,"compared":499,"denominator_entered":16,"denominator_left":11},{"id":"api_catalog","group":"Capabilities","label":"Publishes an API catalog (RFC 9727)","count":11,"of":515,"pct":2.1,"pct_before":2.2,"points_all":-0.1,"points":0,"sites_gained":0,"sites_lost":0,"compared":499,"denominator_entered":16,"denominator_left":11},{"id":"a2a_card","group":"Capabilities","label":"Publishes an A2A agent card","count":2,"of":515,"pct":0.4,"pct_before":0.4,"points_all":0,"points":0,"sites_gained":0,"sites_lost":0,"compared":499,"denominator_entered":16,"denominator_left":11},{"id":"any_capability","group":"Capabilities","label":"Declares any agent capability (any of the four above)","count":15,"of":515,"pct":2.9,"pct_before":2.7,"points_all":0.2,"points":0,"sites_gained":0,"sites_lost":0,"compared":499,"denominator_entered":16,"denominator_left":11},{"id":"ai_refused_any","group":"Delivery parity","label":"At least one AI identity is refused (non-2xx) where the browser got 2xx","count":95,"of":515,"pct":18.4,"pct_before":17.1,"points_all":1.3,"points":0.6,"sites_gained":6,"sites_lost":3,"compared":499,"denominator_entered":16,"denominator_left":11},{"id":"ai_thin_any","group":"Delivery parity","label":"At least one AI identity gets under 70% of the browser's bytes (both 2xx)","count":21,"of":515,"pct":4.1,"pct_before":3.7,"points_all":0.4,"points":0.2,"sites_gained":2,"sites_lost":1,"compared":499,"denominator_entered":16,"denominator_left":11},{"id":"mcp_declared","group":"Endpoint rings","label":"Names an MCP endpoint (server card, llms.txt, agents.md, a homepage link, or /mcp answers like one)","count":56,"of":515,"pct":10.9,"pct_before":null,"points_all":null,"points":null,"note":"not measured in the earlier run"},{"id":"mcp_live","group":"Endpoint rings","label":"An MCP endpoint completes the handshake (initialize and tools/list; no tool called)","count":7,"of":515,"pct":1.4,"pct_before":null,"points_all":null,"points":null,"note":"not measured in the earlier run"},{"id":"mcp_live_of_declared","group":"Endpoint rings","label":"Of sites naming an MCP endpoint, one completes the handshake or asks for credentials","count":39,"of":56,"pct":69.6,"pct_before":null,"points_all":null,"points":null,"note":"not measured in the earlier run"},{"id":"a2a_declared","group":"Endpoint rings","label":"The A2A agent card names an endpoint url","count":1,"of":515,"pct":0.2,"pct_before":null,"points_all":null,"points":null,"note":"not measured in the earlier run"},{"id":"a2a_answers","group":"Endpoint rings","label":"Of A2A cards naming a url, the url answers (any non-5xx, not an HTML page)","count":0,"of":1,"pct":0,"pct_before":null,"points_all":null,"points":null,"note":"not measured in the earlier run"},{"id":"webmcp","group":"Endpoint rings","label":"Homepage carries WebMCP (navigator.modelContext, a WebMCP bridge, or an mcp+json block)","count":1,"of":515,"pct":0.2,"pct_before":null,"points_all":null,"points":null,"note":"not measured in the earlier run"},{"id":"refused_gptbot","group":"Delivery parity","label":"GPTBot refused where the browser got 2xx","count":83,"of":515,"pct":16.1,"pct_before":14.7,"points_all":1.4,"points":1,"sites_gained":6,"sites_lost":1,"compared":499,"denominator_entered":16,"denominator_left":11},{"id":"refused_claudebot","group":"Delivery parity","label":"ClaudeBot refused where the browser got 2xx","count":74,"of":515,"pct":14.4,"pct_before":13.9,"points_all":0.5,"points":-0.4,"sites_gained":2,"sites_lost":4,"compared":499,"denominator_entered":16,"denominator_left":11},{"id":"refused_oaisearch","group":"Delivery parity","label":"OAI-SearchBot refused where the browser got 2xx","count":53,"of":515,"pct":10.3,"pct_before":9.4,"points_all":0.9,"points":0.2,"sites_gained":2,"sites_lost":1,"compared":499,"denominator_entered":16,"denominator_left":11},{"id":"refused_perplexity","group":"Delivery parity","label":"PerplexityBot refused where the browser got 2xx","count":54,"of":515,"pct":10.5,"pct_before":10,"points_all":0.5,"points":0,"sites_gained":2,"sites_lost":2,"compared":499,"denominator_entered":16,"denominator_left":11}],"movers":[{"id":"refused_gptbot","label":"GPTBot refused where the browser got 2xx","points":1,"sites_gained":6,"sites_lost":1},{"id":"robots_served","label":"Serves a robots.txt (200, not HTML)","points":1,"sites_gained":5,"sites_lost":0},{"id":"ai_refused_any","label":"At least one AI identity is refused (non-2xx) where the browser got 2xx","points":0.6,"sites_gained":6,"sites_lost":3},{"id":"reachable","label":"Homepage answers a browser (2xx after redirects)","points":0.5,"sites_gained":16,"sites_lost":11},{"id":"refused_claudebot","label":"ClaudeBot refused where the browser got 2xx","points":-0.4,"sites_gained":2,"sites_lost":4}]},{"frame":"top10k","label":"top 10,000, weekly","runs_this_week":1,"run":{"run":2,"finished_at":"2026-10-10T23:13:15.648Z"},"compared_with":{"run":1,"finished_at":"2026-10-03T21:53:40.747Z"},"sites":9776,"figures":[{"id":"reachable","group":"Reach","label":"Homepage answers a browser (2xx after redirects)","count":5559,"of":9776,"pct":56.9,"pct_before":57.7,"points_all":-0.8,"points":-0.9,"sites_gained":151,"sites_lost":235,"compared":9776,"denominator_entered":0,"denominator_left":0},{"id":"robots_served","group":"Crawler policy","label":"Serves a robots.txt (200, not HTML)","count":4512,"of":5559,"pct":81.2,"pct_before":80.8,"points_all":0.4,"points":0,"sites_gained":34,"sites_lost":33,"compared":5408,"denominator_entered":151,"denominator_left":235},{"id":"robots_sitemap","group":"Crawler policy","label":"robots.txt names a sitemap","count":3145,"of":4512,"pct":69.7,"pct_before":69.7,"points_all":0,"points":0,"sites_gained":7,"sites_lost":6,"compared":4356,"denominator_entered":156,"denominator_left":206},{"id":"content_signal","group":"Crawler policy","label":"robots.txt carries a Content-Signal line","count":142,"of":4512,"pct":3.1,"pct_before":3.2,"points_all":-0.1,"points":0,"sites_gained":2,"sites_lost":0,"compared":4356,"denominator_entered":156,"denominator_left":206},{"id":"cs_ai_train_no","group":"Crawler policy","label":"Content-Signal says ai-train=no","count":85,"of":4512,"pct":1.9,"pct_before":2,"points_all":-0.1,"points":0,"sites_gained":2,"sites_lost":1,"compared":4356,"denominator_entered":156,"denominator_left":206},{"id":"cs_ai_input_no","group":"Crawler policy","label":"Content-Signal says ai-input=no or search=no","count":21,"of":4512,"pct":0.5,"pct_before":0.5,"points_all":0,"points":0,"sites_gained":0,"sites_lost":0,"compared":4356,"denominator_entered":156,"denominator_left":206},{"id":"blocks_any_ai","group":"Crawler policy","label":"robots.txt closes / to at least one of the 12 AI agents","count":846,"of":4512,"pct":18.8,"pct_before":19.1,"points_all":-0.3,"points":0,"sites_gained":3,"sites_lost":2,"compared":4356,"denominator_entered":156,"denominator_left":206},{"id":"blocks_GPTBot","group":"Crawler policy","label":"robots.txt closes / to GPTBot","count":602,"of":4512,"pct":13.3,"pct_before":13.7,"points_all":-0.4,"points":-0.1,"sites_gained":3,"sites_lost":7,"compared":4356,"denominator_entered":156,"denominator_left":206},{"id":"blocks_OAI-SearchBot","group":"Crawler policy","label":"robots.txt closes / to OAI-SearchBot","count":284,"of":4512,"pct":6.3,"pct_before":6.4,"points_all":-0.1,"points":0,"sites_gained":2,"sites_lost":4,"compared":4356,"denominator_entered":156,"denominator_left":206},{"id":"blocks_ChatGPT-User","group":"Crawler policy","label":"robots.txt closes / to ChatGPT-User","count":373,"of":4512,"pct":8.3,"pct_before":8.4,"points_all":-0.1,"points":0,"sites_gained":3,"sites_lost":4,"compared":4356,"denominator_entered":156,"denominator_left":206},{"id":"blocks_ClaudeBot","group":"Crawler policy","label":"robots.txt closes / to ClaudeBot","count":540,"of":4512,"pct":12,"pct_before":12.3,"points_all":-0.3,"points":-0.1,"sites_gained":3,"sites_lost":7,"compared":4356,"denominator_entered":156,"denominator_left":206},{"id":"blocks_Claude-SearchBot","group":"Crawler policy","label":"robots.txt closes / to Claude-SearchBot","count":260,"of":4512,"pct":5.8,"pct_before":5.9,"points_all":-0.1,"points":0,"sites_gained":2,"sites_lost":2,"compared":4356,"denominator_entered":156,"denominator_left":206},{"id":"blocks_PerplexityBot","group":"Crawler policy","label":"robots.txt closes / to PerplexityBot","count":412,"of":4512,"pct":9.1,"pct_before":9.4,"points_all":-0.3,"points":-0.1,"sites_gained":1,"sites_lost":5,"compared":4356,"denominator_entered":156,"denominator_left":206},{"id":"blocks_Google-Extended","group":"Crawler policy","label":"robots.txt closes / to Google-Extended","count":458,"of":4512,"pct":10.2,"pct_before":10.4,"points_all":-0.2,"points":-0.1,"sites_gained":3,"sites_lost":8,"compared":4356,"denominator_entered":156,"denominator_left":206},{"id":"blocks_Applebot-Extended","group":"Crawler policy","label":"robots.txt closes / to Applebot-Extended","count":443,"of":4512,"pct":9.8,"pct_before":10.1,"points_all":-0.3,"points":-0.1,"sites_gained":3,"sites_lost":6,"compared":4356,"denominator_entered":156,"denominator_left":206},{"id":"blocks_CCBot","group":"Crawler policy","label":"robots.txt closes / to CCBot","count":630,"of":4512,"pct":14,"pct_before":14.9,"points_all":-0.9,"points":-0.5,"sites_gained":3,"sites_lost":26,"compared":4356,"denominator_entered":156,"denominator_left":206},{"id":"blocks_Bytespider","group":"Crawler policy","label":"robots.txt closes / to Bytespider","count":611,"of":4512,"pct":13.5,"pct_before":14,"points_all":-0.5,"points":0,"sites_gained":3,"sites_lost":3,"compared":4356,"denominator_entered":156,"denominator_left":206},{"id":"blocks_Meta-ExternalAgent","group":"Crawler policy","label":"robots.txt closes / to Meta-ExternalAgent","count":469,"of":4512,"pct":10.4,"pct_before":11.1,"points_all":-0.7,"points":-0.5,"sites_gained":2,"sites_lost":24,"compared":4356,"denominator_entered":156,"denominator_left":206},{"id":"blocks_Amazonbot","group":"Crawler policy","label":"robots.txt closes / to Amazonbot","count":445,"of":4512,"pct":9.9,"pct_before":10.2,"points_all":-0.3,"points":-0.1,"sites_gained":2,"sites_lost":5,"compared":4356,"denominator_entered":156,"denominator_left":206},{"id":"llms_txt","group":"Machine files","label":"Serves /llms.txt","count":873,"of":5559,"pct":15.7,"pct_before":15.4,"points_all":0.3,"points":0.3,"sites_gained":20,"sites_lost":5,"compared":5408,"denominator_entered":151,"denominator_left":235},{"id":"agents_md","group":"Machine files","label":"Serves /agents.md","count":73,"of":5559,"pct":1.3,"pct_before":1.3,"points_all":0,"points":0,"sites_gained":3,"sites_lost":2,"compared":5408,"denominator_entered":151,"denominator_left":235},{"id":"agent_skills","group":"Capabilities","label":"Publishes an agent-skills index","count":53,"of":5559,"pct":1,"pct_before":0.9,"points_all":0.1,"points":0,"sites_gained":3,"sites_lost":2,"compared":5408,"denominator_entered":151,"denominator_left":235},{"id":"mcp_card","group":"Capabilities","label":"Publishes an MCP server card","count":53,"of":5559,"pct":1,"pct_before":0.9,"points_all":0.1,"points":0,"sites_gained":2,"sites_lost":2,"compared":5408,"denominator_entered":151,"denominator_left":235},{"id":"api_catalog","group":"Capabilities","label":"Publishes an API catalog (RFC 9727)","count":66,"of":5559,"pct":1.2,"pct_before":1.1,"points_all":0.1,"points":0.1,"sites_gained":4,"sites_lost":1,"compared":5408,"denominator_entered":151,"denominator_left":235},{"id":"a2a_card","group":"Capabilities","label":"Publishes an A2A agent card","count":17,"of":5559,"pct":0.3,"pct_before":0.3,"points_all":0,"points":0,"sites_gained":2,"sites_lost":1,"compared":5408,"denominator_entered":151,"denominator_left":235},{"id":"any_capability","group":"Capabilities","label":"Declares any agent capability (any of the four above)","count":77,"of":5559,"pct":1.4,"pct_before":1.3,"points_all":0.1,"points":0,"sites_gained":2,"sites_lost":2,"compared":5408,"denominator_entered":151,"denominator_left":235},{"id":"ai_refused_any","group":"Delivery parity","label":"At least one AI identity is refused (non-2xx) where the browser got 2xx","count":1287,"of":5559,"pct":23.2,"pct_before":22.9,"points_all":0.3,"points":0.1,"sites_gained":70,"sites_lost":65,"compared":5408,"denominator_entered":151,"denominator_left":235},{"id":"ai_thin_any","group":"Delivery parity","label":"At least one AI identity gets under 70% of the browser's bytes (both 2xx)","count":145,"of":5559,"pct":2.6,"pct_before":2.4,"points_all":0.2,"points":0.1,"sites_gained":19,"sites_lost":14,"compared":5408,"denominator_entered":151,"denominator_left":235},{"id":"mcp_declared","group":"Endpoint rings","label":"Names an MCP endpoint (server card, llms.txt, agents.md, a homepage link, or /mcp answers like one)","count":295,"of":5559,"pct":5.3,"pct_before":null,"points_all":null,"points":null,"note":"not measured in the earlier run"},{"id":"mcp_live","group":"Endpoint rings","label":"An MCP endpoint completes the handshake (initialize and tools/list; no tool called)","count":49,"of":5559,"pct":0.9,"pct_before":null,"points_all":null,"points":null,"note":"not measured in the earlier run"},{"id":"mcp_live_of_declared","group":"Endpoint rings","label":"Of sites naming an MCP endpoint, one completes the handshake or asks for credentials","count":229,"of":295,"pct":77.6,"pct_before":null,"points_all":null,"points":null,"note":"not measured in the earlier run"},{"id":"a2a_declared","group":"Endpoint rings","label":"The A2A agent card names an endpoint url","count":8,"of":5559,"pct":0.1,"pct_before":null,"points_all":null,"points":null,"note":"not measured in the earlier run"},{"id":"a2a_answers","group":"Endpoint rings","label":"Of A2A cards naming a url, the url answers (any non-5xx, not an HTML page)","count":3,"of":8,"pct":37.5,"pct_before":null,"points_all":null,"points":null,"note":"not measured in the earlier run"},{"id":"webmcp","group":"Endpoint rings","label":"Homepage carries WebMCP (navigator.modelContext, a WebMCP bridge, or an mcp+json block)","count":6,"of":5559,"pct":0.1,"pct_before":null,"points_all":null,"points":null,"note":"not measured in the earlier run"},{"id":"refused_gptbot","group":"Delivery parity","label":"GPTBot refused where the browser got 2xx","count":1097,"of":5559,"pct":19.7,"pct_before":19.6,"points_all":0.1,"points":0.1,"sites_gained":44,"sites_lost":39,"compared":5408,"denominator_entered":151,"denominator_left":235},{"id":"refused_claudebot","group":"Delivery parity","label":"ClaudeBot refused where the browser got 2xx","count":1106,"of":5559,"pct":19.9,"pct_before":19.8,"points_all":0.1,"points":0,"sites_gained":49,"sites_lost":47,"compared":5408,"denominator_entered":151,"denominator_left":235},{"id":"refused_oaisearch","group":"Delivery parity","label":"OAI-SearchBot refused where the browser got 2xx","count":712,"of":5559,"pct":12.8,"pct_before":12.3,"points_all":0.5,"points":0.4,"sites_gained":43,"sites_lost":22,"compared":5408,"denominator_entered":151,"denominator_left":235},{"id":"refused_perplexity","group":"Delivery parity","label":"PerplexityBot refused where the browser got 2xx","count":736,"of":5559,"pct":13.2,"pct_before":12.7,"points_all":0.5,"points":0.3,"sites_gained":46,"sites_lost":28,"compared":5408,"denominator_entered":151,"denominator_left":235}],"movers":[{"id":"reachable","label":"Homepage answers a browser (2xx after redirects)","points":-0.9,"sites_gained":151,"sites_lost":235},{"id":"blocks_CCBot","label":"robots.txt closes / to CCBot","points":-0.5,"sites_gained":3,"sites_lost":26},{"id":"blocks_Meta-ExternalAgent","label":"robots.txt closes / to Meta-ExternalAgent","points":-0.5,"sites_gained":2,"sites_lost":24},{"id":"refused_oaisearch","label":"OAI-SearchBot refused where the browser got 2xx","points":0.4,"sites_gained":43,"sites_lost":22},{"id":"refused_perplexity","label":"PerplexityBot refused where the browser got 2xx","points":0.3,"sites_gained":46,"sites_lost":28}]}],"reading":"Each frame compares the last run that finished in this ISO week (UTC) with the last run that finished before it. Change is like-for-like: points = (sites gained - sites lost) / sites compared, counting only sites measured in both runs and inside the figure's denominator both times, so a site that did not answer in one run cannot move a figure. points_all is the plain difference of the two percentages, and denominator_entered / denominator_left count the sites that moved in or out of a figure's base (often a site that did not answer one run). No site is named; per-site records: https://crawlcheck.io/api/census/records.","method":"https://crawlcheck.io/api/census","page":"https://crawlcheck.io/data/census/weekly?week=2026-W41","sealed_at":"2026-10-12T00:17:22.625Z","editions":[{"week":"2026-W40","url":"https://crawlcheck.io/api/v1/census/weekly?week=2026-W40","archive":"https://crawlcheck.io/api/v1/census/weekly/2026-W40.json"},{"week":"2026-W41","url":"https://crawlcheck.io/api/v1/census/weekly?week=2026-W41","archive":"https://crawlcheck.io/api/v1/census/weekly/2026-W41.json"}],"feed":"https://crawlcheck.io/api/v1/census/weekly/feed.xml","code":{"version_id":"a0b27a0e-2c79-4841-b2b1-093d95552111","script_sha16":"e7dadc5fed0bfc6a","state":"attested","deployed":{"at":"2026-10-11T18:18:04.747Z","commit":"09ee2035eaf1","run":"38163186367"},"build_log_entry":227,"score_version":31,"scoring_policy_sha256":"86898d2a8c88d6b56e6710f3aadf4375cbb5342d4a2f9aa017b536d95602d671","decision_rules_sha256":"18dd5636e0896692ab518050cb401744edea2e6f1086dff94fead314959bdc3e","check":"https://crawlcheck.io/api/v1/builds/a0b27a0e-2c79-4841-b2b1-093d95552111"},"sha256":"89b8f237fb3bfa8f1decfa3e03eec2d493a15db2d24f79751b913d1181fd8e33","signature":{"v":1,"alg":"Ed25519","kid":"1hHmE3fo1rg5Ig70wuFtJ5wBc7G-sUTgUJkxyPWFRps","over":"sha256","message":"crawlcheck-data-v1\n89b8f237fb3bfa8f1decfa3e03eec2d493a15db2d24f79751b913d1181fd8e33","sig":"jutRBVeUtxrBaY4OvTb45qg_Cg1-AVp9El6bxkLL3awISMv3wjv91c3Kja9IMruTT9dU8K9DhS2U0WA3XYSqBw","signed_at":"2026-10-13T10:07:02.095Z","key":{"kid":"1hHmE3fo1rg5Ig70wuFtJ5wBc7G-sUTgUJkxyPWFRps","jwk":{"kty":"OKP","crv":"Ed25519","x":"5TmKCOx0TTHepq7HwyXBHvGQpVCSBd2lV8cLzYVqFxQ"},"published_in":"https://crawlcheck.io/.well-known/http-message-signatures-directory"},"canonical":"sha256 over canonical JSON of every field except sha256 and signature"}}