{"kind":"crawlcheck-census-weekly","v":1,"week":"2026-W42","state":"week_to_date","from":"2026-10-12T00:00:00.000Z","to":"2026-10-16T09:47:25.737Z","built_at":"2026-10-16T09:47:25.737Z","frames":[{"frame":"top1k","label":"top 1,000, daily","runs_this_week":5,"run":{"run":15,"finished_at":"2026-10-16T05:55:17.231Z"},"compared_with":{"run":10,"finished_at":"2026-10-11T06:04:35.685Z"},"sites":991,"figures":[{"id":"reachable","group":"Reach","label":"Homepage answers a browser (2xx after redirects)","count":533,"of":991,"pct":53.8,"pct_before":52,"points_all":1.8,"points":1.8,"sites_gained":34,"sites_lost":16,"compared":991,"denominator_entered":0,"denominator_left":0},{"id":"robots_served","group":"Crawler policy","label":"Serves a robots.txt (200, not HTML)","count":415,"of":533,"pct":77.9,"pct_before":80.4,"points_all":-2.5,"points":-0.4,"sites_gained":0,"sites_lost":2,"compared":499,"denominator_entered":34,"denominator_left":16},{"id":"robots_sitemap","group":"Crawler policy","label":"robots.txt names a sitemap","count":291,"of":415,"pct":70.1,"pct_before":69.6,"points_all":0.5,"points":0,"sites_gained":0,"sites_lost":0,"compared":399,"denominator_entered":16,"denominator_left":15},{"id":"content_signal","group":"Crawler policy","label":"robots.txt carries a Content-Signal line","count":16,"of":415,"pct":3.9,"pct_before":3.6,"points_all":0.3,"points":0.3,"sites_gained":1,"sites_lost":0,"compared":399,"denominator_entered":16,"denominator_left":15},{"id":"cs_ai_train_no","group":"Crawler policy","label":"Content-Signal says ai-train=no","count":10,"of":415,"pct":2.4,"pct_before":2.2,"points_all":0.2,"points":0.3,"sites_gained":1,"sites_lost":0,"compared":399,"denominator_entered":16,"denominator_left":15},{"id":"cs_ai_input_no","group":"Crawler policy","label":"Content-Signal says ai-input=no or search=no","count":4,"of":415,"pct":1,"pct_before":1,"points_all":0,"points":0,"sites_gained":0,"sites_lost":0,"compared":399,"denominator_entered":16,"denominator_left":15},{"id":"blocks_any_ai","group":"Crawler policy","label":"robots.txt closes / to at least one of the 12 AI agents","count":99,"of":415,"pct":23.9,"pct_before":24.4,"points_all":-0.5,"points":0.3,"sites_gained":1,"sites_lost":0,"compared":399,"denominator_entered":16,"denominator_left":15},{"id":"blocks_GPTBot","group":"Crawler policy","label":"robots.txt closes / to GPTBot","count":65,"of":415,"pct":15.7,"pct_before":16.2,"points_all":-0.5,"points":0.3,"sites_gained":1,"sites_lost":0,"compared":399,"denominator_entered":16,"denominator_left":15},{"id":"blocks_OAI-SearchBot","group":"Crawler policy","label":"robots.txt closes / to OAI-SearchBot","count":32,"of":415,"pct":7.7,"pct_before":8,"points_all":-0.3,"points":0,"sites_gained":0,"sites_lost":0,"compared":399,"denominator_entered":16,"denominator_left":15},{"id":"blocks_ChatGPT-User","group":"Crawler policy","label":"robots.txt closes / to ChatGPT-User","count":41,"of":415,"pct":9.9,"pct_before":10.1,"points_all":-0.2,"points":0,"sites_gained":0,"sites_lost":0,"compared":399,"denominator_entered":16,"denominator_left":15},{"id":"blocks_ClaudeBot","group":"Crawler policy","label":"robots.txt closes / to ClaudeBot","count":61,"of":415,"pct":14.7,"pct_before":15.2,"points_all":-0.5,"points":0.3,"sites_gained":1,"sites_lost":0,"compared":399,"denominator_entered":16,"denominator_left":15},{"id":"blocks_Claude-SearchBot","group":"Crawler policy","label":"robots.txt closes / to Claude-SearchBot","count":33,"of":415,"pct":8,"pct_before":8.5,"points_all":-0.5,"points":0,"sites_gained":0,"sites_lost":0,"compared":399,"denominator_entered":16,"denominator_left":15},{"id":"blocks_PerplexityBot","group":"Crawler policy","label":"robots.txt closes / to PerplexityBot","count":48,"of":415,"pct":11.6,"pct_before":11.8,"points_all":-0.2,"points":0,"sites_gained":0,"sites_lost":0,"compared":399,"denominator_entered":16,"denominator_left":15},{"id":"blocks_Google-Extended","group":"Crawler policy","label":"robots.txt closes / to Google-Extended","count":55,"of":415,"pct":13.3,"pct_before":13.3,"points_all":0,"points":0.3,"sites_gained":2,"sites_lost":1,"compared":399,"denominator_entered":16,"denominator_left":15},{"id":"blocks_Applebot-Extended","group":"Crawler policy","label":"robots.txt closes / to Applebot-Extended","count":48,"of":415,"pct":11.6,"pct_before":11.6,"points_all":0,"points":0,"sites_gained":0,"sites_lost":0,"compared":399,"denominator_entered":16,"denominator_left":15},{"id":"blocks_CCBot","group":"Crawler policy","label":"robots.txt closes / to CCBot","count":79,"of":415,"pct":19,"pct_before":19.3,"points_all":-0.3,"points":0.3,"sites_gained":1,"sites_lost":0,"compared":399,"denominator_entered":16,"denominator_left":15},{"id":"blocks_Bytespider","group":"Crawler policy","label":"robots.txt closes / to Bytespider","count":76,"of":415,"pct":18.3,"pct_before":18.6,"points_all":-0.3,"points":0.3,"sites_gained":1,"sites_lost":0,"compared":399,"denominator_entered":16,"denominator_left":15},{"id":"blocks_Meta-ExternalAgent","group":"Crawler policy","label":"robots.txt closes / to Meta-ExternalAgent","count":60,"of":415,"pct":14.5,"pct_before":15,"points_all":-0.5,"points":0,"sites_gained":0,"sites_lost":0,"compared":399,"denominator_entered":16,"denominator_left":15},{"id":"blocks_Amazonbot","group":"Crawler policy","label":"robots.txt closes / to Amazonbot","count":53,"of":415,"pct":12.8,"pct_before":12.8,"points_all":0,"points":0.3,"sites_gained":1,"sites_lost":0,"compared":399,"denominator_entered":16,"denominator_left":15},{"id":"llms_txt","group":"Machine files","label":"Serves /llms.txt","count":112,"of":533,"pct":21,"pct_before":20.8,"points_all":0.2,"points":0.4,"sites_gained":2,"sites_lost":0,"compared":499,"denominator_entered":34,"denominator_left":16},{"id":"agents_md","group":"Machine files","label":"Serves /agents.md","count":14,"of":533,"pct":2.6,"pct_before":2.7,"points_all":-0.1,"points":0,"sites_gained":0,"sites_lost":0,"compared":499,"denominator_entered":34,"denominator_left":16},{"id":"agent_skills","group":"Capabilities","label":"Publishes an agent-skills index","count":13,"of":533,"pct":2.4,"pct_before":2.3,"points_all":0.1,"points":0.2,"sites_gained":1,"sites_lost":0,"compared":499,"denominator_entered":34,"denominator_left":16},{"id":"mcp_card","group":"Capabilities","label":"Publishes an MCP server card","count":13,"of":533,"pct":2.4,"pct_before":2.3,"points_all":0.1,"points":0.2,"sites_gained":1,"sites_lost":0,"compared":499,"denominator_entered":34,"denominator_left":16},{"id":"api_catalog","group":"Capabilities","label":"Publishes an API catalog (RFC 9727)","count":11,"of":533,"pct":2.1,"pct_before":2.1,"points_all":0,"points":0,"sites_gained":0,"sites_lost":0,"compared":499,"denominator_entered":34,"denominator_left":16},{"id":"a2a_card","group":"Capabilities","label":"Publishes an A2A agent card","count":3,"of":533,"pct":0.6,"pct_before":0.4,"points_all":0.2,"points":0.2,"sites_gained":1,"sites_lost":0,"compared":499,"denominator_entered":34,"denominator_left":16},{"id":"any_capability","group":"Capabilities","label":"Declares any agent capability (any of the four above)","count":16,"of":533,"pct":3,"pct_before":2.9,"points_all":0.1,"points":0.2,"sites_gained":1,"sites_lost":0,"compared":499,"denominator_entered":34,"denominator_left":16},{"id":"ai_refused_any","group":"Delivery parity","label":"At least one AI identity is refused (non-2xx) where the browser got 2xx","count":116,"of":533,"pct":21.8,"pct_before":18.4,"points_all":3.4,"points":1,"sites_gained":9,"sites_lost":4,"compared":499,"denominator_entered":34,"denominator_left":16},{"id":"ai_thin_any","group":"Delivery parity","label":"At least one AI identity gets under 70% of the browser's bytes (both 2xx)","count":21,"of":533,"pct":3.9,"pct_before":4.1,"points_all":-0.2,"points":0,"sites_gained":2,"sites_lost":2,"compared":499,"denominator_entered":34,"denominator_left":16},{"id":"mcp_declared","group":"Endpoint rings","label":"Names an MCP endpoint (server card, llms.txt, agents.md, a homepage link, or /mcp answers like one)","count":55,"of":533,"pct":10.3,"pct_before":10.9,"points_all":-0.6,"points":0.4,"sites_gained":3,"sites_lost":1,"compared":499,"denominator_entered":34,"denominator_left":16},{"id":"mcp_live","group":"Endpoint rings","label":"An MCP endpoint completes the handshake (initialize and tools/list; no tool called)","count":8,"of":533,"pct":1.5,"pct_before":1.4,"points_all":0.1,"points":0,"sites_gained":0,"sites_lost":0,"compared":499,"denominator_entered":34,"denominator_left":16},{"id":"mcp_live_of_declared","group":"Endpoint rings","label":"Of sites naming an MCP endpoint, one completes the handshake or asks for credentials","count":40,"of":55,"pct":72.7,"pct_before":69.6,"points_all":3.1,"points":0,"sites_gained":0,"sites_lost":0,"compared":51,"denominator_entered":4,"denominator_left":5},{"id":"a2a_declared","group":"Endpoint rings","label":"The A2A agent card names an endpoint url","count":1,"of":533,"pct":0.2,"pct_before":0.2,"points_all":0,"points":0,"sites_gained":0,"sites_lost":0,"compared":499,"denominator_entered":34,"denominator_left":16},{"id":"a2a_answers","group":"Endpoint rings","label":"Of A2A cards naming a url, the url answers (any non-5xx, not an HTML page)","count":0,"of":1,"pct":0,"pct_before":0,"points_all":0,"points":0,"sites_gained":0,"sites_lost":0,"compared":1,"denominator_entered":0,"denominator_left":0},{"id":"webmcp","group":"Endpoint rings","label":"Homepage carries WebMCP (navigator.modelContext, a WebMCP bridge, or an mcp+json block)","count":1,"of":533,"pct":0.2,"pct_before":0.2,"points_all":0,"points":0,"sites_gained":0,"sites_lost":0,"compared":499,"denominator_entered":34,"denominator_left":16},{"id":"refused_gptbot","group":"Delivery parity","label":"GPTBot refused where the browser got 2xx","count":99,"of":533,"pct":18.6,"pct_before":16.1,"points_all":2.5,"points":0,"sites_gained":2,"sites_lost":2,"compared":499,"denominator_entered":34,"denominator_left":16},{"id":"refused_claudebot","group":"Delivery parity","label":"ClaudeBot refused where the browser got 2xx","count":93,"of":533,"pct":17.4,"pct_before":14.4,"points_all":3,"points":0.6,"sites_gained":5,"sites_lost":2,"compared":499,"denominator_entered":34,"denominator_left":16},{"id":"refused_oaisearch","group":"Delivery parity","label":"OAI-SearchBot refused where the browser got 2xx","count":71,"of":533,"pct":13.3,"pct_before":10.3,"points_all":3,"points":0,"sites_gained":2,"sites_lost":2,"compared":499,"denominator_entered":34,"denominator_left":16},{"id":"refused_perplexity","group":"Delivery parity","label":"PerplexityBot refused where the browser got 2xx","count":75,"of":533,"pct":14.1,"pct_before":10.5,"points_all":3.6,"points":1,"sites_gained":5,"sites_lost":0,"compared":499,"denominator_entered":34,"denominator_left":16}],"movers":[{"id":"reachable","label":"Homepage answers a browser (2xx after redirects)","points":1.8,"sites_gained":34,"sites_lost":16},{"id":"ai_refused_any","label":"At least one AI identity is refused (non-2xx) where the browser got 2xx","points":1,"sites_gained":9,"sites_lost":4},{"id":"refused_perplexity","label":"PerplexityBot refused where the browser got 2xx","points":1,"sites_gained":5,"sites_lost":0},{"id":"refused_claudebot","label":"ClaudeBot refused where the browser got 2xx","points":0.6,"sites_gained":5,"sites_lost":2},{"id":"mcp_declared","label":"Names an MCP endpoint (server card, llms.txt, agents.md, a homepage link, or /mcp answers like one)","points":0.4,"sites_gained":3,"sites_lost":1}]},{"frame":"top10k","label":"top 10,000, weekly","runs_this_week":0,"run":null,"compared_with":{"run":2,"finished_at":"2026-10-10T23:13:15.648Z"},"note":"no run of this frame finished in the week"}],"reading":"Each frame compares the last run that finished in this ISO week (UTC) with the last run that finished before it. Change is like-for-like: points = (sites gained - sites lost) / sites compared, counting only sites measured in both runs and inside the figure's denominator both times, so a site that did not answer in one run cannot move a figure. points_all is the plain difference of the two percentages, and denominator_entered / denominator_left count the sites that moved in or out of a figure's base (often a site that did not answer one run). No site is named; per-site records: https://crawlcheck.io/api/census/records.","method":"https://crawlcheck.io/api/census","page":"https://crawlcheck.io/data/census/weekly?week=2026-W42","editions":[{"week":"2026-W40","url":"https://crawlcheck.io/api/v1/census/weekly?week=2026-W40","archive":"https://crawlcheck.io/api/v1/census/weekly/2026-W40.json"},{"week":"2026-W41","url":"https://crawlcheck.io/api/v1/census/weekly?week=2026-W41","archive":"https://crawlcheck.io/api/v1/census/weekly/2026-W41.json"}],"feed":"https://crawlcheck.io/api/v1/census/weekly/feed.xml","code":{"version_id":"03a2fe48-da7f-4760-83a0-11e9b8db31a6","script_sha16":"75004d4e7c7f1c4a","state":"attested","deployed":{"at":"2026-10-16T05:34:14.964Z","commit":"4021022fa581","run":"38856409951"},"build_log_entry":419,"score_version":32,"scoring_policy_sha256":"d3c9c95d409dae4e90b8e868ff113e631e445e33c14a0cc008dda144293821b3","decision_rules_sha256":"18dd5636e0896692ab518050cb401744edea2e6f1086dff94fead314959bdc3e","check":"https://crawlcheck.io/api/v1/builds/03a2fe48-da7f-4760-83a0-11e9b8db31a6"},"sha256":"7139d2a59dcc66d72563d5d1198f1d430deb90c1f7cee0b3f137ae1f3e5fd619","signature":{"v":1,"alg":"Ed25519","kid":"1hHmE3fo1rg5Ig70wuFtJ5wBc7G-sUTgUJkxyPWFRps","over":"sha256","message":"crawlcheck-data-v1\n7139d2a59dcc66d72563d5d1198f1d430deb90c1f7cee0b3f137ae1f3e5fd619","sig":"tAHLKGTbBjr1nqh6tFxmgAflssWHVqOzlDuFnxYdbCB2P1dUMPhzCnpiZ9r-kSb1Ju8fnN639Ux54UGe1W09AQ","signed_at":"2026-10-16T09:47:26.324Z","key":{"kid":"1hHmE3fo1rg5Ig70wuFtJ5wBc7G-sUTgUJkxyPWFRps","jwk":{"kty":"OKP","crv":"Ed25519","x":"5TmKCOx0TTHepq7HwyXBHvGQpVCSBd2lV8cLzYVqFxQ"},"published_in":"https://crawlcheck.io/.well-known/http-message-signatures-directory"},"canonical":"sha256 over canonical JSON of every field except sha256 and signature"}}