{"kind":"crawlcheck-census-weekly","v":1,"week":"2026-W45","state":"final","from":"2026-11-02T00:00:00.000Z","to":"2026-11-09T00:00:00.000Z","built_at":"2026-11-09T00:19:14.267Z","frames":[{"frame":"top1k","label":"top 1,000, daily","runs_this_week":7,"run":{"run":38,"finished_at":"2026-11-08T08:15:07.672Z"},"compared_with":{"run":31,"finished_at":"2026-11-01T07:38:23.965Z"},"sites":991,"figures":[{"id":"reachable","group":"Reach","label":"Homepage answers CrawlCheck's census fetch (2xx after redirects)","count":537,"of":991,"pct":54.2,"pct_before":51.4,"points_all":2.8,"points":2.8,"sites_gained":38,"sites_lost":10,"compared":991,"denominator_entered":0,"denominator_left":0},{"id":"refused_us","group":"Reach","label":"Homepage refuses CrawlCheck's census fetch (400, 401, 403 or 429): blocked or rate-limited, not down","count":104,"of":991,"pct":10.5,"pct_before":12.3,"points_all":-1.8,"points":-1.8,"sites_gained":5,"sites_lost":23,"compared":991,"denominator_entered":0,"denominator_left":0},{"id":"no_site","group":"Reach","label":"No website answers at that name (5xx, 530, timeout or no answer)","count":311,"of":991,"pct":31.4,"pct_before":32.6,"points_all":-1.2,"points":-1.2,"sites_gained":3,"sites_lost":15,"compared":991,"denominator_entered":0,"denominator_left":0},{"id":"robots_served","group":"Crawler policy","label":"Serves a robots.txt (200, not HTML)","count":415,"of":537,"pct":77.3,"pct_before":78.4,"points_all":-1.1,"points":1.2,"sites_gained":8,"sites_lost":2,"compared":499,"denominator_entered":38,"denominator_left":10},{"id":"robots_sitemap","group":"Crawler policy","label":"robots.txt names a sitemap","count":296,"of":415,"pct":71.3,"pct_before":72.2,"points_all":-0.9,"points":0.3,"sites_gained":1,"sites_lost":0,"compared":389,"denominator_entered":26,"denominator_left":10},{"id":"content_signal","group":"Crawler policy","label":"robots.txt carries a Content-Signal line","count":19,"of":415,"pct":4.6,"pct_before":4.8,"points_all":-0.2,"points":0,"sites_gained":0,"sites_lost":0,"compared":389,"denominator_entered":26,"denominator_left":10},{"id":"cs_ai_train_no","group":"Crawler policy","label":"Content-Signal says ai-train=no","count":13,"of":415,"pct":3.1,"pct_before":3.3,"points_all":-0.2,"points":0,"sites_gained":0,"sites_lost":0,"compared":389,"denominator_entered":26,"denominator_left":10},{"id":"cs_ai_input_no","group":"Crawler policy","label":"Content-Signal says ai-input=no or search=no","count":4,"of":415,"pct":1,"pct_before":1,"points_all":0,"points":0,"sites_gained":0,"sites_lost":0,"compared":389,"denominator_entered":26,"denominator_left":10},{"id":"aipref_train","group":"Crawler policy","label":"robots.txt names the AI-training policy token (__usage_ai-training, IETF AIPREF draft)","count":4,"of":414,"pct":1,"pct_before":null,"points_all":null,"points":null,"note":"not measured in the earlier run"},{"id":"aipref_train_no","group":"Crawler policy","label":"robots.txt states a preference against AI training on / with that token","count":4,"of":414,"pct":1,"pct_before":null,"points_all":null,"points":null,"note":"not measured in the earlier run"},{"id":"blocks_any_ai","group":"Crawler policy","label":"robots.txt closes / to at least one of the 12 AI agents","count":101,"of":415,"pct":24.3,"pct_before":25.3,"points_all":-1,"points":0.3,"sites_gained":1,"sites_lost":0,"compared":389,"denominator_entered":26,"denominator_left":10},{"id":"blocks_GPTBot","group":"Crawler policy","label":"robots.txt closes / to GPTBot","count":64,"of":415,"pct":15.4,"pct_before":16.8,"points_all":-1.4,"points":0,"sites_gained":0,"sites_lost":0,"compared":389,"denominator_entered":26,"denominator_left":10},{"id":"blocks_OAI-SearchBot","group":"Crawler policy","label":"robots.txt closes / to OAI-SearchBot","count":29,"of":415,"pct":7,"pct_before":7.5,"points_all":-0.5,"points":0.3,"sites_gained":1,"sites_lost":0,"compared":389,"denominator_entered":26,"denominator_left":10},{"id":"blocks_ChatGPT-User","group":"Crawler policy","label":"robots.txt closes / to ChatGPT-User","count":37,"of":415,"pct":8.9,"pct_before":9.8,"points_all":-0.9,"points":0,"sites_gained":0,"sites_lost":0,"compared":389,"denominator_entered":26,"denominator_left":10},{"id":"blocks_ClaudeBot","group":"Crawler policy","label":"robots.txt closes / to ClaudeBot","count":61,"of":415,"pct":14.7,"pct_before":15.5,"points_all":-0.8,"points":0,"sites_gained":0,"sites_lost":0,"compared":389,"denominator_entered":26,"denominator_left":10},{"id":"blocks_Claude-SearchBot","group":"Crawler policy","label":"robots.txt closes / to Claude-SearchBot","count":34,"of":415,"pct":8.2,"pct_before":8.3,"points_all":-0.1,"points":0.3,"sites_gained":1,"sites_lost":0,"compared":389,"denominator_entered":26,"denominator_left":10},{"id":"blocks_PerplexityBot","group":"Crawler policy","label":"robots.txt closes / to PerplexityBot","count":46,"of":415,"pct":11.1,"pct_before":11.5,"points_all":-0.4,"points":0.3,"sites_gained":1,"sites_lost":0,"compared":389,"denominator_entered":26,"denominator_left":10},{"id":"blocks_Google-Extended","group":"Crawler policy","label":"robots.txt closes / to Google-Extended","count":54,"of":415,"pct":13,"pct_before":14,"points_all":-1,"points":0,"sites_gained":0,"sites_lost":0,"compared":389,"denominator_entered":26,"denominator_left":10},{"id":"blocks_Applebot-Extended","group":"Crawler policy","label":"robots.txt closes / to Applebot-Extended","count":49,"of":415,"pct":11.8,"pct_before":12.5,"points_all":-0.7,"points":0,"sites_gained":0,"sites_lost":0,"compared":389,"denominator_entered":26,"denominator_left":10},{"id":"blocks_CCBot","group":"Crawler policy","label":"robots.txt closes / to CCBot","count":78,"of":415,"pct":18.8,"pct_before":19.5,"points_all":-0.7,"points":0,"sites_gained":0,"sites_lost":0,"compared":389,"denominator_entered":26,"denominator_left":10},{"id":"blocks_Bytespider","group":"Crawler policy","label":"robots.txt closes / to Bytespider","count":76,"of":415,"pct":18.3,"pct_before":19.3,"points_all":-1,"points":0,"sites_gained":0,"sites_lost":0,"compared":389,"denominator_entered":26,"denominator_left":10},{"id":"blocks_Meta-ExternalAgent","group":"Crawler policy","label":"robots.txt closes / to Meta-ExternalAgent","count":60,"of":415,"pct":14.5,"pct_before":15,"points_all":-0.5,"points":0.3,"sites_gained":1,"sites_lost":0,"compared":389,"denominator_entered":26,"denominator_left":10},{"id":"blocks_Amazonbot","group":"Crawler policy","label":"robots.txt closes / to Amazonbot","count":55,"of":415,"pct":13.3,"pct_before":14,"points_all":-0.7,"points":0,"sites_gained":0,"sites_lost":0,"compared":389,"denominator_entered":26,"denominator_left":10},{"id":"llms_txt","group":"Machine files","label":"Serves /llms.txt","count":123,"of":537,"pct":22.9,"pct_before":21.6,"points_all":1.3,"points":1.2,"sites_gained":7,"sites_lost":1,"compared":499,"denominator_entered":38,"denominator_left":10},{"id":"agents_md","group":"Machine files","label":"Serves /agents.md","count":16,"of":537,"pct":3,"pct_before":2.4,"points_all":0.6,"points":0.6,"sites_gained":3,"sites_lost":0,"compared":499,"denominator_entered":38,"denominator_left":10},{"id":"agent_skills","group":"Capabilities","label":"Publishes an agent-skills index","count":14,"of":537,"pct":2.6,"pct_before":2.6,"points_all":0,"points":0,"sites_gained":0,"sites_lost":0,"compared":499,"denominator_entered":38,"denominator_left":10},{"id":"mcp_card","group":"Capabilities","label":"Publishes an MCP server card","count":13,"of":537,"pct":2.4,"pct_before":2.6,"points_all":-0.2,"points":0,"sites_gained":0,"sites_lost":0,"compared":499,"denominator_entered":38,"denominator_left":10},{"id":"api_catalog","group":"Capabilities","label":"Publishes an API catalog (RFC 9727)","count":13,"of":537,"pct":2.4,"pct_before":2.4,"points_all":0,"points":0.2,"sites_gained":2,"sites_lost":1,"compared":499,"denominator_entered":38,"denominator_left":10},{"id":"a2a_card","group":"Capabilities","label":"Publishes an A2A agent card","count":3,"of":537,"pct":0.6,"pct_before":0.6,"points_all":0,"points":0,"sites_gained":0,"sites_lost":0,"compared":499,"denominator_entered":38,"denominator_left":10},{"id":"any_capability","group":"Capabilities","label":"Declares any agent capability (any of the four above)","count":17,"of":537,"pct":3.2,"pct_before":3.3,"points_all":-0.1,"points":-0.2,"sites_gained":0,"sites_lost":1,"compared":499,"denominator_entered":38,"denominator_left":10},{"id":"ai_refused_any","group":"Delivery parity","label":"At least one AI identity is refused (non-2xx) where the browser got 2xx","count":112,"of":537,"pct":20.9,"pct_before":18.9,"points_all":2,"points":-0.6,"sites_gained":7,"sites_lost":10,"compared":499,"denominator_entered":38,"denominator_left":10},{"id":"ai_thin_any","group":"Delivery parity","label":"At least one AI identity gets under 70% of the browser's bytes (both 2xx)","count":20,"of":537,"pct":3.7,"pct_before":3.9,"points_all":-0.2,"points":0,"sites_gained":3,"sites_lost":3,"compared":499,"denominator_entered":38,"denominator_left":10},{"id":"mcp_declared","group":"Endpoint rings","label":"Names an MCP endpoint (server card, llms.txt, agents.md, a homepage link, or /mcp answers like one)","count":58,"of":537,"pct":10.8,"pct_before":10.2,"points_all":0.6,"points":0.6,"sites_gained":3,"sites_lost":0,"compared":499,"denominator_entered":38,"denominator_left":10},{"id":"mcp_live","group":"Endpoint rings","label":"An MCP endpoint completes the handshake (initialize and tools/list; no tool called)","count":9,"of":537,"pct":1.7,"pct_before":1.6,"points_all":0.1,"points":0,"sites_gained":1,"sites_lost":1,"compared":499,"denominator_entered":38,"denominator_left":10},{"id":"mcp_live_of_declared","group":"Endpoint rings","label":"Of sites naming an MCP endpoint, one completes the handshake or asks for credentials","count":46,"of":58,"pct":79.3,"pct_before":80.8,"points_all":-1.5,"points":0,"sites_gained":0,"sites_lost":0,"compared":52,"denominator_entered":6,"denominator_left":0},{"id":"a2a_declared","group":"Endpoint rings","label":"The A2A agent card names an endpoint url","count":2,"of":537,"pct":0.4,"pct_before":0.4,"points_all":0,"points":0,"sites_gained":0,"sites_lost":0,"compared":499,"denominator_entered":38,"denominator_left":10},{"id":"a2a_answers","group":"Endpoint rings","label":"Of A2A cards naming a url, the url answers (any non-5xx, not an HTML page)","count":0,"of":2,"pct":0,"pct_before":50,"points_all":-50,"points":-50,"sites_gained":0,"sites_lost":1,"compared":2,"denominator_entered":0,"denominator_left":0},{"id":"webmcp","group":"Endpoint rings","label":"Homepage carries WebMCP (navigator.modelContext, a WebMCP bridge, or an mcp+json block)","count":1,"of":537,"pct":0.2,"pct_before":0.2,"points_all":0,"points":0,"sites_gained":0,"sites_lost":0,"compared":499,"denominator_entered":38,"denominator_left":10},{"id":"sb_listed","group":"Safety","label":"Listed by Google Safe Browsing (all or some pages): browsers show a red warning","count":3,"of":357,"pct":0.8,"pct_before":null,"points_all":null,"points":null,"note":"not measured in the earlier run"},{"id":"cf_listed","group":"Safety","label":"Listed as malware or phishing by a public malware-blocking DNS resolver","count":0,"of":550,"pct":0,"pct_before":null,"points_all":null,"points":null,"note":"not measured in the earlier run"},{"id":"hsts_preloaded","group":"Safety","label":"On the browsers' HSTS preload list (HTTPS-only from the first visit)","count":91,"of":550,"pct":16.5,"pct_before":null,"points_all":null,"points":null,"note":"not measured in the earlier run"},{"id":"outside_scripts","group":"Safety","label":"Homepage loads scripts or styles from another domain","count":348,"of":550,"pct":63.3,"pct_before":null,"points_all":null,"points":null,"note":"not measured in the earlier run"},{"id":"outside_5plus","group":"Safety","label":"Homepage loads scripts or styles from 5 or more other domains","count":31,"of":550,"pct":5.6,"pct_before":null,"points_all":null,"points":null,"note":"not measured in the earlier run"},{"id":"any_integrity","group":"Safety","label":"Pins at least one outside script or style with integrity=","count":34,"of":348,"pct":9.8,"pct_before":null,"points_all":null,"points":null,"note":"not measured in the earlier run"},{"id":"unpinned_cdn","group":"Safety","label":"Loads a library from a public CDN without integrity=","count":47,"of":550,"pct":8.5,"pct_before":null,"points_all":null,"points":null,"note":"not measured in the earlier run"},{"id":"hijacked_cdn","group":"Safety","label":"Still loads scripts from a CDN taken over in 2024 (polyfill.io and its sister CDNs)","count":0,"of":550,"pct":0,"pct_before":null,"points_all":null,"points":null,"note":"not measured in the earlier run"},{"id":"hack_signs","group":"Safety","label":"Homepage shows hidden spam links or cryptominer code","count":0,"of":550,"pct":0,"pct_before":null,"points_all":null,"points":null,"note":"not measured in the earlier run"},{"id":"refused_gptbot","group":"Delivery parity","label":"GPTBot refused where the browser got 2xx","count":97,"of":537,"pct":18.1,"pct_before":16.5,"points_all":1.6,"points":-1.2,"sites_gained":3,"sites_lost":9,"compared":499,"denominator_entered":38,"denominator_left":10},{"id":"refused_claudebot","group":"Delivery parity","label":"ClaudeBot refused where the browser got 2xx","count":95,"of":537,"pct":17.7,"pct_before":14.9,"points_all":2.8,"points":-0.2,"sites_gained":5,"sites_lost":6,"compared":499,"denominator_entered":38,"denominator_left":10},{"id":"refused_oaisearch","group":"Delivery parity","label":"OAI-SearchBot refused where the browser got 2xx","count":74,"of":537,"pct":13.8,"pct_before":10.8,"points_all":3,"points":0,"sites_gained":5,"sites_lost":5,"compared":499,"denominator_entered":38,"denominator_left":10},{"id":"refused_perplexity","group":"Delivery parity","label":"PerplexityBot refused where the browser got 2xx","count":82,"of":537,"pct":15.3,"pct_before":11.8,"points_all":3.5,"points":0.6,"sites_gained":8,"sites_lost":5,"compared":499,"denominator_entered":38,"denominator_left":10}],"movers":[{"id":"a2a_answers","label":"Of A2A cards naming a url, the url answers (any non-5xx, not an HTML page)","points":-50,"sites_gained":0,"sites_lost":1},{"id":"reachable","label":"Homepage answers CrawlCheck's census fetch (2xx after redirects)","points":2.8,"sites_gained":38,"sites_lost":10},{"id":"refused_us","label":"Homepage refuses CrawlCheck's census fetch (400, 401, 403 or 429): blocked or rate-limited, not down","points":-1.8,"sites_gained":5,"sites_lost":23},{"id":"no_site","label":"No website answers at that name (5xx, 530, timeout or no answer)","points":-1.2,"sites_gained":3,"sites_lost":15},{"id":"refused_gptbot","label":"GPTBot refused where the browser got 2xx","points":-1.2,"sites_gained":3,"sites_lost":9}]},{"frame":"top10k","label":"top 10,000, daily","runs_this_week":7,"run":{"run":12,"finished_at":"2026-11-08T00:06:35.271Z"},"compared_with":{"run":5,"finished_at":"2026-10-31T21:02:55.469Z"},"sites":9776,"figures":[{"id":"reachable","group":"Reach","label":"Homepage answers CrawlCheck's census fetch (2xx after redirects)","count":5680,"of":9776,"pct":58.1,"pct_before":57.5,"points_all":0.6,"points":0.6,"sites_gained":191,"sites_lost":132,"compared":9776,"denominator_entered":0,"denominator_left":0},{"id":"refused_us","group":"Reach","label":"Homepage refuses CrawlCheck's census fetch (400, 401, 403 or 429): blocked or rate-limited, not down","count":1324,"of":9776,"pct":13.5,"pct_before":13.7,"points_all":-0.2,"points":-0.1,"sites_gained":77,"sites_lost":91,"compared":9776,"denominator_entered":0,"denominator_left":0},{"id":"no_site","group":"Reach","label":"No website answers at that name (5xx, 530, timeout or no answer)","count":2570,"of":9776,"pct":26.3,"pct_before":26.8,"points_all":-0.5,"points":-0.6,"sites_gained":71,"sites_lost":125,"compared":9776,"denominator_entered":0,"denominator_left":0},{"id":"robots_served","group":"Crawler policy","label":"Serves a robots.txt (200, not HTML)","count":4651,"of":5680,"pct":81.9,"pct_before":81.7,"points_all":0.2,"points":0.1,"sites_gained":36,"sites_lost":31,"compared":5489,"denominator_entered":191,"denominator_left":132},{"id":"robots_sitemap","group":"Crawler policy","label":"robots.txt names a sitemap","count":3263,"of":4651,"pct":70.2,"pct_before":70.1,"points_all":0.1,"points":0.3,"sites_gained":13,"sites_lost":0,"compared":4472,"denominator_entered":179,"denominator_left":122},{"id":"content_signal","group":"Crawler policy","label":"robots.txt carries a Content-Signal line","count":293,"of":4651,"pct":6.3,"pct_before":6.2,"points_all":0.1,"points":0.2,"sites_gained":10,"sites_lost":2,"compared":4472,"denominator_entered":179,"denominator_left":122},{"id":"cs_ai_train_no","group":"Crawler policy","label":"Content-Signal says ai-train=no","count":227,"of":4651,"pct":4.9,"pct_before":4.8,"points_all":0.1,"points":0.1,"sites_gained":6,"sites_lost":3,"compared":4472,"denominator_entered":179,"denominator_left":122},{"id":"cs_ai_input_no","group":"Crawler policy","label":"Content-Signal says ai-input=no or search=no","count":21,"of":4651,"pct":0.5,"pct_before":0.5,"points_all":0,"points":0,"sites_gained":0,"sites_lost":1,"compared":4472,"denominator_entered":179,"denominator_left":122},{"id":"aipref_train","group":"Crawler policy","label":"robots.txt names the AI-training policy token (__usage_ai-training, IETF AIPREF draft)","count":12,"of":4647,"pct":0.3,"pct_before":null,"points_all":null,"points":null,"note":"not measured in the earlier run"},{"id":"aipref_train_no","group":"Crawler policy","label":"robots.txt states a preference against AI training on / with that token","count":12,"of":4647,"pct":0.3,"pct_before":null,"points_all":null,"points":null,"note":"not measured in the earlier run"},{"id":"blocks_any_ai","group":"Crawler policy","label":"robots.txt closes / to at least one of the 12 AI agents","count":1024,"of":4651,"pct":22,"pct_before":21.9,"points_all":0.1,"points":0.1,"sites_gained":10,"sites_lost":5,"compared":4472,"denominator_entered":179,"denominator_left":122},{"id":"blocks_GPTBot","group":"Crawler policy","label":"robots.txt closes / to GPTBot","count":756,"of":4651,"pct":16.3,"pct_before":16.2,"points_all":0.1,"points":0.1,"sites_gained":9,"sites_lost":3,"compared":4472,"denominator_entered":179,"denominator_left":122},{"id":"blocks_OAI-SearchBot","group":"Crawler policy","label":"robots.txt closes / to OAI-SearchBot","count":284,"of":4651,"pct":6.1,"pct_before":6.2,"points_all":-0.1,"points":0.1,"sites_gained":5,"sites_lost":2,"compared":4472,"denominator_entered":179,"denominator_left":122},{"id":"blocks_ChatGPT-User","group":"Crawler policy","label":"robots.txt closes / to ChatGPT-User","count":370,"of":4651,"pct":8,"pct_before":8.1,"points_all":-0.1,"points":0,"sites_gained":2,"sites_lost":3,"compared":4472,"denominator_entered":179,"denominator_left":122},{"id":"blocks_ClaudeBot","group":"Crawler policy","label":"robots.txt closes / to ClaudeBot","count":697,"of":4651,"pct":15,"pct_before":14.9,"points_all":0.1,"points":0.1,"sites_gained":8,"sites_lost":5,"compared":4472,"denominator_entered":179,"denominator_left":122},{"id":"blocks_Claude-SearchBot","group":"Crawler policy","label":"robots.txt closes / to Claude-SearchBot","count":267,"of":4651,"pct":5.7,"pct_before":5.7,"points_all":0,"points":0.1,"sites_gained":5,"sites_lost":2,"compared":4472,"denominator_entered":179,"denominator_left":122},{"id":"blocks_PerplexityBot","group":"Crawler policy","label":"robots.txt closes / to PerplexityBot","count":416,"of":4651,"pct":8.9,"pct_before":9.3,"points_all":-0.4,"points":-0.2,"sites_gained":3,"sites_lost":14,"compared":4472,"denominator_entered":179,"denominator_left":122},{"id":"blocks_Google-Extended","group":"Crawler policy","label":"robots.txt closes / to Google-Extended","count":625,"of":4651,"pct":13.4,"pct_before":13.4,"points_all":0,"points":0.1,"sites_gained":9,"sites_lost":5,"compared":4472,"denominator_entered":179,"denominator_left":122},{"id":"blocks_Applebot-Extended","group":"Crawler policy","label":"robots.txt closes / to Applebot-Extended","count":608,"of":4651,"pct":13.1,"pct_before":12.9,"points_all":0.2,"points":0.2,"sites_gained":12,"sites_lost":4,"compared":4472,"denominator_entered":179,"denominator_left":122},{"id":"blocks_CCBot","group":"Crawler policy","label":"robots.txt closes / to CCBot","count":797,"of":4651,"pct":17.1,"pct_before":17.2,"points_all":-0.1,"points":0,"sites_gained":7,"sites_lost":8,"compared":4472,"denominator_entered":179,"denominator_left":122},{"id":"blocks_Bytespider","group":"Crawler policy","label":"robots.txt closes / to Bytespider","count":791,"of":4651,"pct":17,"pct_before":16.9,"points_all":0.1,"points":0.1,"sites_gained":8,"sites_lost":4,"compared":4472,"denominator_entered":179,"denominator_left":122},{"id":"blocks_Meta-ExternalAgent","group":"Crawler policy","label":"robots.txt closes / to Meta-ExternalAgent","count":645,"of":4651,"pct":13.9,"pct_before":13.7,"points_all":0.2,"points":0.2,"sites_gained":13,"sites_lost":4,"compared":4472,"denominator_entered":179,"denominator_left":122},{"id":"blocks_Amazonbot","group":"Crawler policy","label":"robots.txt closes / to Amazonbot","count":624,"of":4651,"pct":13.4,"pct_before":13.1,"points_all":0.3,"points":0.2,"sites_gained":11,"sites_lost":3,"compared":4472,"denominator_entered":179,"denominator_left":122},{"id":"llms_txt","group":"Machine files","label":"Serves /llms.txt","count":968,"of":5680,"pct":17,"pct_before":16.6,"points_all":0.4,"points":0.5,"sites_gained":32,"sites_lost":7,"compared":5489,"denominator_entered":191,"denominator_left":132},{"id":"agents_md","group":"Machine files","label":"Serves /agents.md","count":84,"of":5680,"pct":1.5,"pct_before":1.3,"points_all":0.2,"points":0.1,"sites_gained":7,"sites_lost":0,"compared":5489,"denominator_entered":191,"denominator_left":132},{"id":"agent_skills","group":"Capabilities","label":"Publishes an agent-skills index","count":58,"of":5680,"pct":1,"pct_before":1,"points_all":0,"points":0,"sites_gained":2,"sites_lost":0,"compared":5489,"denominator_entered":191,"denominator_left":132},{"id":"mcp_card","group":"Capabilities","label":"Publishes an MCP server card","count":61,"of":5680,"pct":1.1,"pct_before":1,"points_all":0.1,"points":0.1,"sites_gained":5,"sites_lost":0,"compared":5489,"denominator_entered":191,"denominator_left":132},{"id":"api_catalog","group":"Capabilities","label":"Publishes an API catalog (RFC 9727)","count":67,"of":5680,"pct":1.2,"pct_before":1.1,"points_all":0.1,"points":0.1,"sites_gained":5,"sites_lost":0,"compared":5489,"denominator_entered":191,"denominator_left":132},{"id":"a2a_card","group":"Capabilities","label":"Publishes an A2A agent card","count":15,"of":5680,"pct":0.3,"pct_before":0.2,"points_all":0.1,"points":0,"sites_gained":2,"sites_lost":0,"compared":5489,"denominator_entered":191,"denominator_left":132},{"id":"any_capability","group":"Capabilities","label":"Declares any agent capability (any of the four above)","count":82,"of":5680,"pct":1.4,"pct_before":1.4,"points_all":0,"points":0.1,"sites_gained":4,"sites_lost":0,"compared":5489,"denominator_entered":191,"denominator_left":132},{"id":"ai_refused_any","group":"Delivery parity","label":"At least one AI identity is refused (non-2xx) where the browser got 2xx","count":1306,"of":5680,"pct":23,"pct_before":23.7,"points_all":-0.7,"points":-0.4,"sites_gained":59,"sites_lost":79,"compared":5489,"denominator_entered":191,"denominator_left":132},{"id":"ai_thin_any","group":"Delivery parity","label":"At least one AI identity gets under 70% of the browser's bytes (both 2xx)","count":133,"of":5680,"pct":2.3,"pct_before":2.5,"points_all":-0.2,"points":-0.8,"sites_gained":7,"sites_lost":51,"compared":5489,"denominator_entered":191,"denominator_left":132},{"id":"mcp_declared","group":"Endpoint rings","label":"Names an MCP endpoint (server card, llms.txt, agents.md, a homepage link, or /mcp answers like one)","count":327,"of":5680,"pct":5.8,"pct_before":5.6,"points_all":0.2,"points":0.1,"sites_gained":10,"sites_lost":2,"compared":5489,"denominator_entered":191,"denominator_left":132},{"id":"mcp_live","group":"Endpoint rings","label":"An MCP endpoint completes the handshake (initialize and tools/list; no tool called)","count":53,"of":5680,"pct":0.9,"pct_before":0.9,"points_all":0,"points":0,"sites_gained":3,"sites_lost":1,"compared":5489,"denominator_entered":191,"denominator_left":132},{"id":"mcp_live_of_declared","group":"Endpoint rings","label":"Of sites naming an MCP endpoint, one completes the handshake or asks for credentials","count":258,"of":327,"pct":78.9,"pct_before":79.4,"points_all":-0.5,"points":0,"sites_gained":1,"sites_lost":1,"compared":309,"denominator_entered":18,"denominator_left":6},{"id":"a2a_declared","group":"Endpoint rings","label":"The A2A agent card names an endpoint url","count":8,"of":5680,"pct":0.1,"pct_before":0.1,"points_all":0,"points":0,"sites_gained":1,"sites_lost":0,"compared":5489,"denominator_entered":191,"denominator_left":132},{"id":"a2a_answers","group":"Endpoint rings","label":"Of A2A cards naming a url, the url answers (any non-5xx, not an HTML page)","count":3,"of":8,"pct":37.5,"pct_before":50,"points_all":-12.5,"points":0,"sites_gained":0,"sites_lost":0,"compared":7,"denominator_entered":1,"denominator_left":1},{"id":"webmcp","group":"Endpoint rings","label":"Homepage carries WebMCP (navigator.modelContext, a WebMCP bridge, or an mcp+json block)","count":5,"of":5680,"pct":0.1,"pct_before":0.1,"points_all":0,"points":0,"sites_gained":0,"sites_lost":0,"compared":5489,"denominator_entered":191,"denominator_left":132},{"id":"sb_listed","group":"Safety","label":"Listed by Google Safe Browsing (all or some pages): browsers show a red warning","count":42,"of":4614,"pct":0.9,"pct_before":null,"points_all":null,"points":null,"note":"not measured in the earlier run"},{"id":"cf_listed","group":"Safety","label":"Listed as malware or phishing by a public malware-blocking DNS resolver","count":15,"of":5698,"pct":0.3,"pct_before":null,"points_all":null,"points":null,"note":"not measured in the earlier run"},{"id":"hsts_preloaded","group":"Safety","label":"On the browsers' HSTS preload list (HTTPS-only from the first visit)","count":456,"of":5698,"pct":8,"pct_before":null,"points_all":null,"points":null,"note":"not measured in the earlier run"},{"id":"outside_scripts","group":"Safety","label":"Homepage loads scripts or styles from another domain","count":3845,"of":5698,"pct":67.5,"pct_before":null,"points_all":null,"points":null,"note":"not measured in the earlier run"},{"id":"outside_5plus","group":"Safety","label":"Homepage loads scripts or styles from 5 or more other domains","count":619,"of":5698,"pct":10.9,"pct_before":null,"points_all":null,"points":null,"note":"not measured in the earlier run"},{"id":"any_integrity","group":"Safety","label":"Pins at least one outside script or style with integrity=","count":387,"of":3845,"pct":10.1,"pct_before":null,"points_all":null,"points":null,"note":"not measured in the earlier run"},{"id":"unpinned_cdn","group":"Safety","label":"Loads a library from a public CDN without integrity=","count":759,"of":5698,"pct":13.3,"pct_before":null,"points_all":null,"points":null,"note":"not measured in the earlier run"},{"id":"hijacked_cdn","group":"Safety","label":"Still loads scripts from a CDN taken over in 2024 (polyfill.io and its sister CDNs)","count":4,"of":5698,"pct":0.1,"pct_before":null,"points_all":null,"points":null,"note":"not measured in the earlier run"},{"id":"hack_signs","group":"Safety","label":"Homepage shows hidden spam links or cryptominer code","count":1,"of":5698,"pct":0,"pct_before":null,"points_all":null,"points":null,"note":"not measured in the earlier run"},{"id":"refused_gptbot","group":"Delivery parity","label":"GPTBot refused where the browser got 2xx","count":1093,"of":5680,"pct":19.2,"pct_before":19.7,"points_all":-0.5,"points":0,"sites_gained":37,"sites_lost":38,"compared":5489,"denominator_entered":191,"denominator_left":132},{"id":"refused_claudebot","group":"Delivery parity","label":"ClaudeBot refused where the browser got 2xx","count":1135,"of":5680,"pct":20,"pct_before":20.3,"points_all":-0.3,"points":0,"sites_gained":52,"sites_lost":54,"compared":5489,"denominator_entered":191,"denominator_left":132},{"id":"refused_oaisearch","group":"Delivery parity","label":"OAI-SearchBot refused where the browser got 2xx","count":707,"of":5680,"pct":12.4,"pct_before":13.3,"points_all":-0.9,"points":-0.4,"sites_gained":25,"sites_lost":46,"compared":5489,"denominator_entered":191,"denominator_left":132},{"id":"refused_perplexity","group":"Delivery parity","label":"PerplexityBot refused where the browser got 2xx","count":824,"of":5680,"pct":14.5,"pct_before":13.4,"points_all":1.1,"points":1.6,"sites_gained":113,"sites_lost":27,"compared":5489,"denominator_entered":191,"denominator_left":132}],"movers":[{"id":"refused_perplexity","label":"PerplexityBot refused where the browser got 2xx","points":1.6,"sites_gained":113,"sites_lost":27},{"id":"ai_thin_any","label":"At least one AI identity gets under 70% of the browser's bytes (both 2xx)","points":-0.8,"sites_gained":7,"sites_lost":51},{"id":"reachable","label":"Homepage answers CrawlCheck's census fetch (2xx after redirects)","points":0.6,"sites_gained":191,"sites_lost":132},{"id":"no_site","label":"No website answers at that name (5xx, 530, timeout or no answer)","points":-0.6,"sites_gained":71,"sites_lost":125},{"id":"llms_txt","label":"Serves /llms.txt","points":0.5,"sites_gained":32,"sites_lost":7}]}],"reading":"Each frame compares the last run that finished in this ISO week (UTC) with the last run that finished before it. Change is like-for-like: points = (sites gained - sites lost) / sites compared, counting only sites measured in both runs and inside the figure's denominator both times, so a site that did not answer in one run cannot move a figure. points_all is the plain difference of the two percentages, and denominator_entered / denominator_left count the sites that moved in or out of a figure's base (often a site that did not answer one run). No site is named; per-site records: https://crawlcheck.io/api/census/records.","method":"https://crawlcheck.io/api/census","page":"https://crawlcheck.io/data/census/weekly?week=2026-W45","sealed_at":"2026-11-09T00:19:14.573Z","editions":[{"week":"2026-W40","url":"https://crawlcheck.io/api/v1/census/weekly?week=2026-W40","archive":"https://crawlcheck.io/api/v1/census/weekly/2026-W40.json"},{"week":"2026-W41","url":"https://crawlcheck.io/api/v1/census/weekly?week=2026-W41","archive":"https://crawlcheck.io/api/v1/census/weekly/2026-W41.json"},{"week":"2026-W42","url":"https://crawlcheck.io/api/v1/census/weekly?week=2026-W42","archive":"https://crawlcheck.io/api/v1/census/weekly/2026-W42.json"},{"week":"2026-W43","url":"https://crawlcheck.io/api/v1/census/weekly?week=2026-W43","archive":"https://crawlcheck.io/api/v1/census/weekly/2026-W43.json"},{"week":"2026-W44","url":"https://crawlcheck.io/api/v1/census/weekly?week=2026-W44","archive":"https://crawlcheck.io/api/v1/census/weekly/2026-W44.json"},{"week":"2026-W45","url":"https://crawlcheck.io/api/v1/census/weekly?week=2026-W45","archive":"https://crawlcheck.io/api/v1/census/weekly/2026-W45.json"},{"week":"2026-W46","url":"https://crawlcheck.io/api/v1/census/weekly?week=2026-W46","archive":"https://crawlcheck.io/api/v1/census/weekly/2026-W46.json"}],"feed":"https://crawlcheck.io/api/v1/census/weekly/feed.xml","code":{"build_id":"9219fb519ff0eaf2","script_sha16":"b00e794a299c5a1b","state":"attested","deployed":{"at":"2026-11-21T18:41:28.011Z"},"build_log_entry":2899,"score_version":43,"scoring_policy_sha256":"3e08fccfbbce356dc82ecd302b32a226a36fc0fb3150a4ab72248fb5795372af","decision_rules_sha256":"18dd5636e0896692ab518050cb401744edea2e6f1086dff94fead314959bdc3e","check":"https://crawlcheck.io/api/v1/builds/9219fb519ff0eaf2"},"sha256":"e53af9088fbe4626005cd1c4d0519a6ca5c1236ffd9604e5e766c658ac3a3888","signature":{"v":1,"alg":"Ed25519","kid":"1hHmE3fo1rg5Ig70wuFtJ5wBc7G-sUTgUJkxyPWFRps","over":"sha256","message":"crawlcheck-data-v1\ne53af9088fbe4626005cd1c4d0519a6ca5c1236ffd9604e5e766c658ac3a3888","sig":"P0X_UmsA9kO6-UJSwfyuBC4j4N3bFuTHCDX6Ki6h85MEtUOm-1IhPl52l0YotlmhwPmdGQjOpZIeptskJDZLAA","signed_at":"2026-11-22T14:11:57.462Z","key":{"kid":"1hHmE3fo1rg5Ig70wuFtJ5wBc7G-sUTgUJkxyPWFRps","jwk":{"kty":"OKP","crv":"Ed25519","x":"5TmKCOx0TTHepq7HwyXBHvGQpVCSBd2lV8cLzYVqFxQ"},"published_in":"https://crawlcheck.io/.well-known/http-message-signatures-directory"},"canonical":"sha256 over canonical JSON of every field except sha256 and signature"}}