{"kind":"crawlcheck-census-weekly","v":1,"week":"2026-W40","state":"final","from":"2026-09-28T00:00:00.000Z","to":"2026-10-05T00:00:00.000Z","built_at":"2026-10-09T23:17:50.441Z","frames":[{"frame":"top1k","label":"top 1,000, daily","runs_this_week":1,"run":{"run":3,"finished_at":"2026-10-04T05:00:02.620Z"},"compared_with":null,"sites":991,"figures":[{"id":"reachable","group":"Reach","label":"Homepage answers a browser (2xx after redirects)","count":510,"of":991,"pct":51.5},{"id":"robots_served","group":"Crawler policy","label":"Serves a robots.txt (200, not HTML)","count":403,"of":510,"pct":79},{"id":"robots_sitemap","group":"Crawler policy","label":"robots.txt names a sitemap","count":281,"of":403,"pct":69.7},{"id":"content_signal","group":"Crawler policy","label":"robots.txt carries a Content-Signal line","count":15,"of":403,"pct":3.7},{"id":"cs_ai_train_no","group":"Crawler policy","label":"Content-Signal says ai-train=no","count":9,"of":403,"pct":2.2},{"id":"cs_ai_input_no","group":"Crawler policy","label":"Content-Signal says ai-input=no or search=no","count":4,"of":403,"pct":1},{"id":"blocks_any_ai","group":"Crawler policy","label":"robots.txt closes / to at least one of the 12 AI agents","count":99,"of":403,"pct":24.6},{"id":"blocks_GPTBot","group":"Crawler policy","label":"robots.txt closes / to GPTBot","count":67,"of":403,"pct":16.6},{"id":"blocks_OAI-SearchBot","group":"Crawler policy","label":"robots.txt closes / to OAI-SearchBot","count":33,"of":403,"pct":8.2},{"id":"blocks_ChatGPT-User","group":"Crawler policy","label":"robots.txt closes / to ChatGPT-User","count":42,"of":403,"pct":10.4},{"id":"blocks_ClaudeBot","group":"Crawler policy","label":"robots.txt closes / to ClaudeBot","count":62,"of":403,"pct":15.4},{"id":"blocks_Claude-SearchBot","group":"Crawler policy","label":"robots.txt closes / to Claude-SearchBot","count":34,"of":403,"pct":8.4},{"id":"blocks_PerplexityBot","group":"Crawler policy","label":"robots.txt closes / to PerplexityBot","count":50,"of":403,"pct":12.4},{"id":"blocks_Google-Extended","group":"Crawler policy","label":"robots.txt closes / to Google-Extended","count":56,"of":403,"pct":13.9},{"id":"blocks_Applebot-Extended","group":"Crawler policy","label":"robots.txt closes / to Applebot-Extended","count":48,"of":403,"pct":11.9},{"id":"blocks_CCBot","group":"Crawler policy","label":"robots.txt closes / to CCBot","count":79,"of":403,"pct":19.6},{"id":"blocks_Bytespider","group":"Crawler policy","label":"robots.txt closes / to Bytespider","count":76,"of":403,"pct":18.9},{"id":"blocks_Meta-ExternalAgent","group":"Crawler policy","label":"robots.txt closes / to Meta-ExternalAgent","count":61,"of":403,"pct":15.1},{"id":"blocks_Amazonbot","group":"Crawler policy","label":"robots.txt closes / to Amazonbot","count":52,"of":403,"pct":12.9},{"id":"llms_txt","group":"Machine files","label":"Serves /llms.txt","count":105,"of":510,"pct":20.6},{"id":"agents_md","group":"Machine files","label":"Serves /agents.md","count":14,"of":510,"pct":2.7},{"id":"agent_skills","group":"Capabilities","label":"Publishes an agent-skills index","count":10,"of":510,"pct":2},{"id":"mcp_card","group":"Capabilities","label":"Publishes an MCP server card","count":12,"of":510,"pct":2.4},{"id":"api_catalog","group":"Capabilities","label":"Publishes an API catalog (RFC 9727)","count":11,"of":510,"pct":2.2},{"id":"a2a_card","group":"Capabilities","label":"Publishes an A2A agent card","count":2,"of":510,"pct":0.4},{"id":"any_capability","group":"Capabilities","label":"Declares any agent capability (any of the four above)","count":14,"of":510,"pct":2.7},{"id":"ai_refused_any","group":"Delivery parity","label":"At least one AI identity is refused (non-2xx) where the browser got 2xx","count":87,"of":510,"pct":17.1},{"id":"ai_thin_any","group":"Delivery parity","label":"At least one AI identity gets under 70% of the browser's bytes (both 2xx)","count":19,"of":510,"pct":3.7},{"id":"mcp_declared","group":"Endpoint rings","label":"Names an MCP endpoint (server card, llms.txt, agents.md, a homepage link, or /mcp answers like one)","count":0,"of":0,"pct":null},{"id":"mcp_live","group":"Endpoint rings","label":"An MCP endpoint completes the handshake (initialize and tools/list; no tool called)","count":0,"of":0,"pct":null},{"id":"mcp_live_of_declared","group":"Endpoint rings","label":"Of sites naming an MCP endpoint, one completes the handshake or asks for credentials","count":0,"of":0,"pct":null},{"id":"a2a_declared","group":"Endpoint rings","label":"The A2A agent card names an endpoint url","count":0,"of":0,"pct":null},{"id":"a2a_answers","group":"Endpoint rings","label":"Of A2A cards naming a url, the url answers (any non-5xx, not an HTML page)","count":0,"of":0,"pct":null},{"id":"webmcp","group":"Endpoint rings","label":"Homepage carries WebMCP (navigator.modelContext, a WebMCP bridge, or an mcp+json block)","count":0,"of":0,"pct":null},{"id":"refused_gptbot","group":"Delivery parity","label":"GPTBot refused where the browser got 2xx","count":75,"of":510,"pct":14.7},{"id":"refused_claudebot","group":"Delivery parity","label":"ClaudeBot refused where the browser got 2xx","count":71,"of":510,"pct":13.9},{"id":"refused_oaisearch","group":"Delivery parity","label":"OAI-SearchBot refused where the browser got 2xx","count":48,"of":510,"pct":9.4},{"id":"refused_perplexity","group":"Delivery parity","label":"PerplexityBot refused where the browser got 2xx","count":51,"of":510,"pct":10}]},{"frame":"top10k","label":"top 10,000, weekly","runs_this_week":1,"run":{"run":1,"finished_at":"2026-10-03T21:53:40.747Z"},"compared_with":null,"sites":9776,"figures":[{"id":"reachable","group":"Reach","label":"Homepage answers a browser (2xx after redirects)","count":5643,"of":9776,"pct":57.7},{"id":"robots_served","group":"Crawler policy","label":"Serves a robots.txt (200, not HTML)","count":4562,"of":5643,"pct":80.8},{"id":"robots_sitemap","group":"Crawler policy","label":"robots.txt names a sitemap","count":3180,"of":4562,"pct":69.7},{"id":"content_signal","group":"Crawler policy","label":"robots.txt carries a Content-Signal line","count":146,"of":4562,"pct":3.2},{"id":"cs_ai_train_no","group":"Crawler policy","label":"Content-Signal says ai-train=no","count":89,"of":4562,"pct":2},{"id":"cs_ai_input_no","group":"Crawler policy","label":"Content-Signal says ai-input=no or search=no","count":22,"of":4562,"pct":0.5},{"id":"blocks_any_ai","group":"Crawler policy","label":"robots.txt closes / to at least one of the 12 AI agents","count":873,"of":4562,"pct":19.1},{"id":"blocks_GPTBot","group":"Crawler policy","label":"robots.txt closes / to GPTBot","count":626,"of":4562,"pct":13.7},{"id":"blocks_OAI-SearchBot","group":"Crawler policy","label":"robots.txt closes / to OAI-SearchBot","count":292,"of":4562,"pct":6.4},{"id":"blocks_ChatGPT-User","group":"Crawler policy","label":"robots.txt closes / to ChatGPT-User","count":382,"of":4562,"pct":8.4},{"id":"blocks_ClaudeBot","group":"Crawler policy","label":"robots.txt closes / to ClaudeBot","count":563,"of":4562,"pct":12.3},{"id":"blocks_Claude-SearchBot","group":"Crawler policy","label":"robots.txt closes / to Claude-SearchBot","count":267,"of":4562,"pct":5.9},{"id":"blocks_PerplexityBot","group":"Crawler policy","label":"robots.txt closes / to PerplexityBot","count":431,"of":4562,"pct":9.4},{"id":"blocks_Google-Extended","group":"Crawler policy","label":"robots.txt closes / to Google-Extended","count":476,"of":4562,"pct":10.4},{"id":"blocks_Applebot-Extended","group":"Crawler policy","label":"robots.txt closes / to Applebot-Extended","count":460,"of":4562,"pct":10.1},{"id":"blocks_CCBot","group":"Crawler policy","label":"robots.txt closes / to CCBot","count":678,"of":4562,"pct":14.9},{"id":"blocks_Bytespider","group":"Crawler policy","label":"robots.txt closes / to Bytespider","count":640,"of":4562,"pct":14},{"id":"blocks_Meta-ExternalAgent","group":"Crawler policy","label":"robots.txt closes / to Meta-ExternalAgent","count":508,"of":4562,"pct":11.1},{"id":"blocks_Amazonbot","group":"Crawler policy","label":"robots.txt closes / to Amazonbot","count":466,"of":4562,"pct":10.2},{"id":"llms_txt","group":"Machine files","label":"Serves /llms.txt","count":871,"of":5643,"pct":15.4},{"id":"agents_md","group":"Machine files","label":"Serves /agents.md","count":74,"of":5643,"pct":1.3},{"id":"agent_skills","group":"Capabilities","label":"Publishes an agent-skills index","count":51,"of":5643,"pct":0.9},{"id":"mcp_card","group":"Capabilities","label":"Publishes an MCP server card","count":53,"of":5643,"pct":0.9},{"id":"api_catalog","group":"Capabilities","label":"Publishes an API catalog (RFC 9727)","count":62,"of":5643,"pct":1.1},{"id":"a2a_card","group":"Capabilities","label":"Publishes an A2A agent card","count":16,"of":5643,"pct":0.3},{"id":"any_capability","group":"Capabilities","label":"Declares any agent capability (any of the four above)","count":75,"of":5643,"pct":1.3},{"id":"ai_refused_any","group":"Delivery parity","label":"At least one AI identity is refused (non-2xx) where the browser got 2xx","count":1294,"of":5643,"pct":22.9},{"id":"ai_thin_any","group":"Delivery parity","label":"At least one AI identity gets under 70% of the browser's bytes (both 2xx)","count":138,"of":5643,"pct":2.4},{"id":"mcp_declared","group":"Endpoint rings","label":"Names an MCP endpoint (server card, llms.txt, agents.md, a homepage link, or /mcp answers like one)","count":0,"of":0,"pct":null},{"id":"mcp_live","group":"Endpoint rings","label":"An MCP endpoint completes the handshake (initialize and tools/list; no tool called)","count":0,"of":0,"pct":null},{"id":"mcp_live_of_declared","group":"Endpoint rings","label":"Of sites naming an MCP endpoint, one completes the handshake or asks for credentials","count":0,"of":0,"pct":null},{"id":"a2a_declared","group":"Endpoint rings","label":"The A2A agent card names an endpoint url","count":0,"of":0,"pct":null},{"id":"a2a_answers","group":"Endpoint rings","label":"Of A2A cards naming a url, the url answers (any non-5xx, not an HTML page)","count":0,"of":0,"pct":null},{"id":"webmcp","group":"Endpoint rings","label":"Homepage carries WebMCP (navigator.modelContext, a WebMCP bridge, or an mcp+json block)","count":0,"of":0,"pct":null},{"id":"refused_gptbot","group":"Delivery parity","label":"GPTBot refused where the browser got 2xx","count":1105,"of":5643,"pct":19.6},{"id":"refused_claudebot","group":"Delivery parity","label":"ClaudeBot refused where the browser got 2xx","count":1117,"of":5643,"pct":19.8},{"id":"refused_oaisearch","group":"Delivery parity","label":"OAI-SearchBot refused where the browser got 2xx","count":694,"of":5643,"pct":12.3},{"id":"refused_perplexity","group":"Delivery parity","label":"PerplexityBot refused where the browser got 2xx","count":718,"of":5643,"pct":12.7}]}],"reading":"Each frame compares the last run that finished in this ISO week (UTC) with the last run that finished before it. Change is like-for-like: points = (sites gained - sites lost) / sites compared, counting only sites measured in both runs and inside the figure's denominator both times, so a site that did not answer in one run cannot move a figure. points_all is the plain difference of the two percentages, and denominator_entered / denominator_left count the sites that moved in or out of a figure's base (often a site that did not answer one run). No site is named; per-site records: https://crawlcheck.io/api/census/records.","method":"https://crawlcheck.io/api/census","page":"https://crawlcheck.io/data/census/weekly?week=2026-W40","sealed_at":"2026-10-09T23:17:53.373Z","editions":[{"week":"2026-W40","url":"https://crawlcheck.io/api/v1/census/weekly?week=2026-W40","archive":"https://crawlcheck.io/api/v1/census/weekly/2026-W40.json"},{"week":"2026-W41","url":"https://crawlcheck.io/api/v1/census/weekly?week=2026-W41","archive":"https://crawlcheck.io/api/v1/census/weekly/2026-W41.json"}],"feed":"https://crawlcheck.io/api/v1/census/weekly/feed.xml","code":{"version_id":"a0b27a0e-2c79-4841-b2b1-093d95552111","script_sha16":"e7dadc5fed0bfc6a","state":"attested","deployed":{"at":"2026-10-11T18:18:04.747Z","commit":"09ee2035eaf1","run":"38163186367"},"build_log_entry":227,"score_version":31,"scoring_policy_sha256":"86898d2a8c88d6b56e6710f3aadf4375cbb5342d4a2f9aa017b536d95602d671","decision_rules_sha256":"18dd5636e0896692ab518050cb401744edea2e6f1086dff94fead314959bdc3e","check":"https://crawlcheck.io/api/v1/builds/a0b27a0e-2c79-4841-b2b1-093d95552111"},"sha256":"19672dab7aa05426414e275bb94f685c6a465c627ee4a6d519890f1efb245e2c","signature":{"v":1,"alg":"Ed25519","kid":"1hHmE3fo1rg5Ig70wuFtJ5wBc7G-sUTgUJkxyPWFRps","over":"sha256","message":"crawlcheck-data-v1\n19672dab7aa05426414e275bb94f685c6a465c627ee4a6d519890f1efb245e2c","sig":"My6dvZjO0nUuwx_mxQFyDVd6cjgfJ9AlcAYLBPt8zdwIpeeucoYtvzTbwdk6-KhObOQTunf77I5EZ7112fDIAQ","signed_at":"2026-10-13T10:07:01.953Z","key":{"kid":"1hHmE3fo1rg5Ig70wuFtJ5wBc7G-sUTgUJkxyPWFRps","jwk":{"kty":"OKP","crv":"Ed25519","x":"5TmKCOx0TTHepq7HwyXBHvGQpVCSBd2lV8cLzYVqFxQ"},"published_in":"https://crawlcheck.io/.well-known/http-message-signatures-directory"},"canonical":"sha256 over canonical JSON of every field except sha256 and signature"}}