{"kind":"crawlcheck-census-weekly","v":1,"week":"2026-W46","state":"final","from":"2026-11-09T00:00:00.000Z","to":"2026-11-16T00:00:00.000Z","built_at":"2026-11-16T00:19:13.398Z","frames":[{"frame":"top1k","label":"top 1,000, daily","runs_this_week":7,"run":{"run":45,"finished_at":"2026-11-15T08:59:28.147Z"},"compared_with":{"run":38,"finished_at":"2026-11-08T08:15:07.672Z"},"sites":991,"figures":[{"id":"reachable","group":"Reach","label":"Homepage answers CrawlCheck's census fetch (2xx after redirects)","count":518,"of":991,"pct":52.3,"pct_before":54.2,"points_all":-1.9,"points":-1.9,"sites_gained":18,"sites_lost":37,"compared":991,"denominator_entered":0,"denominator_left":0},{"id":"refused_us","group":"Reach","label":"Homepage refuses CrawlCheck's census fetch (400, 401, 403 or 429): blocked or rate-limited, not down","count":118,"of":991,"pct":11.9,"pct_before":10.5,"points_all":1.4,"points":1.4,"sites_gained":25,"sites_lost":11,"compared":991,"denominator_entered":0,"denominator_left":0},{"id":"no_site","group":"Reach","label":"No website answers at that name (5xx, 530, timeout or no answer)","count":315,"of":991,"pct":31.8,"pct_before":31.4,"points_all":0.4,"points":0.4,"sites_gained":9,"sites_lost":5,"compared":991,"denominator_entered":0,"denominator_left":0},{"id":"robots_served","group":"Crawler policy","label":"Serves a robots.txt (200, not HTML)","count":410,"of":518,"pct":79.2,"pct_before":77.3,"points_all":1.9,"points":-0.6,"sites_gained":2,"sites_lost":5,"compared":500,"denominator_entered":18,"denominator_left":37},{"id":"robots_sitemap","group":"Crawler policy","label":"robots.txt names a sitemap","count":293,"of":410,"pct":71.5,"pct_before":71.3,"points_all":0.2,"points":0,"sites_gained":1,"sites_lost":1,"compared":392,"denominator_entered":18,"denominator_left":23},{"id":"content_signal","group":"Crawler policy","label":"robots.txt carries a Content-Signal line","count":18,"of":410,"pct":4.4,"pct_before":4.6,"points_all":-0.2,"points":0.3,"sites_gained":1,"sites_lost":0,"compared":392,"denominator_entered":18,"denominator_left":23},{"id":"cs_ai_train_no","group":"Crawler policy","label":"Content-Signal says ai-train=no","count":13,"of":410,"pct":3.2,"pct_before":3.1,"points_all":0.1,"points":0.3,"sites_gained":1,"sites_lost":0,"compared":392,"denominator_entered":18,"denominator_left":23},{"id":"cs_ai_input_no","group":"Crawler policy","label":"Content-Signal says ai-input=no or search=no","count":4,"of":410,"pct":1,"pct_before":1,"points_all":0,"points":0,"sites_gained":0,"sites_lost":0,"compared":392,"denominator_entered":18,"denominator_left":23},{"id":"aipref_train","group":"Crawler policy","label":"robots.txt names the AI-training policy token (__usage_ai-training, IETF AIPREF draft)","count":5,"of":410,"pct":1.2,"pct_before":1,"points_all":0.2,"points":0.3,"sites_gained":1,"sites_lost":0,"compared":392,"denominator_entered":18,"denominator_left":22},{"id":"aipref_train_no","group":"Crawler policy","label":"robots.txt states a preference against AI training on / with that token","count":5,"of":410,"pct":1.2,"pct_before":1,"points_all":0.2,"points":0.3,"sites_gained":1,"sites_lost":0,"compared":392,"denominator_entered":18,"denominator_left":22},{"id":"blocks_any_ai","group":"Crawler policy","label":"robots.txt closes / to at least one of the 12 AI agents","count":104,"of":410,"pct":25.4,"pct_before":24.3,"points_all":1.1,"points":0,"sites_gained":0,"sites_lost":0,"compared":392,"denominator_entered":18,"denominator_left":23},{"id":"blocks_GPTBot","group":"Crawler policy","label":"robots.txt closes / to GPTBot","count":67,"of":410,"pct":16.3,"pct_before":15.4,"points_all":0.9,"points":-0.3,"sites_gained":0,"sites_lost":1,"compared":392,"denominator_entered":18,"denominator_left":23},{"id":"blocks_OAI-SearchBot","group":"Crawler policy","label":"robots.txt closes / to OAI-SearchBot","count":31,"of":410,"pct":7.6,"pct_before":7,"points_all":0.6,"points":-0.3,"sites_gained":0,"sites_lost":1,"compared":392,"denominator_entered":18,"denominator_left":23},{"id":"blocks_ChatGPT-User","group":"Crawler policy","label":"robots.txt closes / to ChatGPT-User","count":38,"of":410,"pct":9.3,"pct_before":8.9,"points_all":0.4,"points":-0.5,"sites_gained":0,"sites_lost":2,"compared":392,"denominator_entered":18,"denominator_left":23},{"id":"blocks_ClaudeBot","group":"Crawler policy","label":"robots.txt closes / to ClaudeBot","count":66,"of":410,"pct":16.1,"pct_before":14.7,"points_all":1.4,"points":0.3,"sites_gained":1,"sites_lost":0,"compared":392,"denominator_entered":18,"denominator_left":23},{"id":"blocks_Claude-SearchBot","group":"Crawler policy","label":"robots.txt closes / to Claude-SearchBot","count":38,"of":410,"pct":9.3,"pct_before":8.2,"points_all":1.1,"points":0,"sites_gained":0,"sites_lost":0,"compared":392,"denominator_entered":18,"denominator_left":23},{"id":"blocks_PerplexityBot","group":"Crawler policy","label":"robots.txt closes / to PerplexityBot","count":51,"of":410,"pct":12.4,"pct_before":11.1,"points_all":1.3,"points":0,"sites_gained":0,"sites_lost":0,"compared":392,"denominator_entered":18,"denominator_left":23},{"id":"blocks_Google-Extended","group":"Crawler policy","label":"robots.txt closes / to Google-Extended","count":57,"of":410,"pct":13.9,"pct_before":13,"points_all":0.9,"points":-0.3,"sites_gained":0,"sites_lost":1,"compared":392,"denominator_entered":18,"denominator_left":23},{"id":"blocks_Applebot-Extended","group":"Crawler policy","label":"robots.txt closes / to Applebot-Extended","count":52,"of":410,"pct":12.7,"pct_before":11.8,"points_all":0.9,"points":0.3,"sites_gained":1,"sites_lost":0,"compared":392,"denominator_entered":18,"denominator_left":23},{"id":"blocks_CCBot","group":"Crawler policy","label":"robots.txt closes / to CCBot","count":81,"of":410,"pct":19.8,"pct_before":18.8,"points_all":1,"points":0,"sites_gained":0,"sites_lost":0,"compared":392,"denominator_entered":18,"denominator_left":23},{"id":"blocks_Bytespider","group":"Crawler policy","label":"robots.txt closes / to Bytespider","count":81,"of":410,"pct":19.8,"pct_before":18.3,"points_all":1.5,"points":0.3,"sites_gained":1,"sites_lost":0,"compared":392,"denominator_entered":18,"denominator_left":23},{"id":"blocks_Meta-ExternalAgent","group":"Crawler policy","label":"robots.txt closes / to Meta-ExternalAgent","count":64,"of":410,"pct":15.6,"pct_before":14.5,"points_all":1.1,"points":0.3,"sites_gained":1,"sites_lost":0,"compared":392,"denominator_entered":18,"denominator_left":23},{"id":"blocks_Amazonbot","group":"Crawler policy","label":"robots.txt closes / to Amazonbot","count":56,"of":410,"pct":13.7,"pct_before":13.3,"points_all":0.4,"points":-0.3,"sites_gained":0,"sites_lost":1,"compared":392,"denominator_entered":18,"denominator_left":23},{"id":"llms_txt","group":"Machine files","label":"Serves /llms.txt","count":114,"of":518,"pct":22,"pct_before":22.9,"points_all":-0.9,"points":-0.2,"sites_gained":1,"sites_lost":2,"compared":500,"denominator_entered":18,"denominator_left":37},{"id":"agents_md","group":"Machine files","label":"Serves /agents.md","count":16,"of":518,"pct":3.1,"pct_before":3,"points_all":0.1,"points":0,"sites_gained":0,"sites_lost":0,"compared":500,"denominator_entered":18,"denominator_left":37},{"id":"agent_skills","group":"Capabilities","label":"Publishes an agent-skills index","count":14,"of":518,"pct":2.7,"pct_before":2.6,"points_all":0.1,"points":0.2,"sites_gained":1,"sites_lost":0,"compared":500,"denominator_entered":18,"denominator_left":37},{"id":"mcp_card","group":"Capabilities","label":"Publishes an MCP server card","count":13,"of":518,"pct":2.5,"pct_before":2.4,"points_all":0.1,"points":0.2,"sites_gained":1,"sites_lost":0,"compared":500,"denominator_entered":18,"denominator_left":37},{"id":"api_catalog","group":"Capabilities","label":"Publishes an API catalog (RFC 9727)","count":12,"of":518,"pct":2.3,"pct_before":2.4,"points_all":-0.1,"points":0,"sites_gained":0,"sites_lost":0,"compared":500,"denominator_entered":18,"denominator_left":37},{"id":"a2a_card","group":"Capabilities","label":"Publishes an A2A agent card","count":3,"of":518,"pct":0.6,"pct_before":0.6,"points_all":0,"points":0.2,"sites_gained":1,"sites_lost":0,"compared":500,"denominator_entered":18,"denominator_left":37},{"id":"any_capability","group":"Capabilities","label":"Declares any agent capability (any of the four above)","count":17,"of":518,"pct":3.3,"pct_before":3.2,"points_all":0.1,"points":0.2,"sites_gained":1,"sites_lost":0,"compared":500,"denominator_entered":18,"denominator_left":37},{"id":"ai_refused_any","group":"Delivery parity","label":"At least one AI identity is refused (non-2xx) where the browser got 2xx","count":99,"of":518,"pct":19.1,"pct_before":20.9,"points_all":-1.8,"points":0.8,"sites_gained":7,"sites_lost":3,"compared":500,"denominator_entered":18,"denominator_left":37},{"id":"ai_thin_any","group":"Delivery parity","label":"At least one AI identity gets under 70% of the browser's bytes (both 2xx)","count":19,"of":518,"pct":3.7,"pct_before":3.7,"points_all":0,"points":-0.8,"sites_gained":0,"sites_lost":4,"compared":500,"denominator_entered":18,"denominator_left":37},{"id":"mcp_declared","group":"Endpoint rings","label":"Names an MCP endpoint (server card, llms.txt, agents.md, a homepage link, or /mcp answers like one)","count":56,"of":518,"pct":10.8,"pct_before":10.8,"points_all":0,"points":0.2,"sites_gained":2,"sites_lost":1,"compared":500,"denominator_entered":18,"denominator_left":37},{"id":"mcp_live","group":"Endpoint rings","label":"An MCP endpoint completes the handshake (initialize and tools/list; no tool called)","count":7,"of":518,"pct":1.4,"pct_before":1.7,"points_all":-0.3,"points":-0.2,"sites_gained":0,"sites_lost":1,"compared":500,"denominator_entered":18,"denominator_left":37},{"id":"mcp_live_of_declared","group":"Endpoint rings","label":"Of sites naming an MCP endpoint, one completes the handshake or asks for credentials","count":43,"of":56,"pct":76.8,"pct_before":79.3,"points_all":-2.5,"points":0,"sites_gained":0,"sites_lost":0,"compared":54,"denominator_entered":2,"denominator_left":4},{"id":"a2a_declared","group":"Endpoint rings","label":"The A2A agent card names an endpoint url","count":2,"of":518,"pct":0.4,"pct_before":0.4,"points_all":0,"points":0.2,"sites_gained":1,"sites_lost":0,"compared":500,"denominator_entered":18,"denominator_left":37},{"id":"a2a_answers","group":"Endpoint rings","label":"Of A2A cards naming a url, the url answers (any non-5xx, not an HTML page)","count":1,"of":2,"pct":50,"pct_before":0,"points_all":50,"points":0,"sites_gained":0,"sites_lost":0,"compared":1,"denominator_entered":1,"denominator_left":1},{"id":"webmcp","group":"Endpoint rings","label":"Homepage carries WebMCP (navigator.modelContext, a WebMCP bridge, or an mcp+json block)","count":1,"of":518,"pct":0.2,"pct_before":0.2,"points_all":0,"points":0,"sites_gained":0,"sites_lost":0,"compared":500,"denominator_entered":18,"denominator_left":37},{"id":"sb_listed","group":"Safety","label":"Listed by Google Safe Browsing (all or some pages): browsers show a red warning","count":3,"of":314,"pct":1,"pct_before":0.8,"points_all":0.2,"points":null,"note":"not measured in the earlier run"},{"id":"cf_listed","group":"Safety","label":"Listed as malware or phishing by a public malware-blocking DNS resolver","count":0,"of":532,"pct":0,"pct_before":0,"points_all":0,"points":null,"note":"not measured in the earlier run"},{"id":"hsts_preloaded","group":"Safety","label":"On the browsers' HSTS preload list (HTTPS-only from the first visit)","count":91,"of":532,"pct":17.1,"pct_before":16.5,"points_all":0.6,"points":null,"note":"not measured in the earlier run"},{"id":"outside_scripts","group":"Safety","label":"Homepage loads scripts or styles from another domain","count":330,"of":532,"pct":62,"pct_before":63.3,"points_all":-1.3,"points":null,"note":"not measured in the earlier run"},{"id":"outside_5plus","group":"Safety","label":"Homepage loads scripts or styles from 5 or more other domains","count":34,"of":532,"pct":6.4,"pct_before":5.6,"points_all":0.8,"points":null,"note":"not measured in the earlier run"},{"id":"any_integrity","group":"Safety","label":"Pins at least one outside script or style with integrity=","count":31,"of":330,"pct":9.4,"pct_before":9.8,"points_all":-0.4,"points":null,"note":"not measured in the earlier run"},{"id":"unpinned_cdn","group":"Safety","label":"Loads a library from a public CDN without integrity=","count":48,"of":532,"pct":9,"pct_before":8.5,"points_all":0.5,"points":null,"note":"not measured in the earlier run"},{"id":"hijacked_cdn","group":"Safety","label":"Still loads scripts from a CDN taken over in 2024 (polyfill.io and its sister CDNs)","count":0,"of":532,"pct":0,"pct_before":0,"points_all":0,"points":null,"note":"not measured in the earlier run"},{"id":"hack_signs","group":"Safety","label":"Homepage shows hidden spam links or cryptominer code","count":0,"of":532,"pct":0,"pct_before":0,"points_all":0,"points":null,"note":"not measured in the earlier run"},{"id":"refused_gptbot","group":"Delivery parity","label":"GPTBot refused where the browser got 2xx","count":82,"of":518,"pct":15.8,"pct_before":18.1,"points_all":-2.3,"points":0.4,"sites_gained":4,"sites_lost":2,"compared":500,"denominator_entered":18,"denominator_left":37},{"id":"refused_claudebot","group":"Delivery parity","label":"ClaudeBot refused where the browser got 2xx","count":81,"of":518,"pct":15.6,"pct_before":17.7,"points_all":-2.1,"points":0.6,"sites_gained":6,"sites_lost":3,"compared":500,"denominator_entered":18,"denominator_left":37},{"id":"refused_oaisearch","group":"Delivery parity","label":"OAI-SearchBot refused where the browser got 2xx","count":58,"of":518,"pct":11.2,"pct_before":13.8,"points_all":-2.6,"points":0.2,"sites_gained":3,"sites_lost":2,"compared":500,"denominator_entered":18,"denominator_left":37},{"id":"refused_perplexity","group":"Delivery parity","label":"PerplexityBot refused where the browser got 2xx","count":69,"of":518,"pct":13.3,"pct_before":15.3,"points_all":-2,"points":1,"sites_gained":6,"sites_lost":1,"compared":500,"denominator_entered":18,"denominator_left":37}],"movers":[{"id":"reachable","label":"Homepage answers CrawlCheck's census fetch (2xx after redirects)","points":-1.9,"sites_gained":18,"sites_lost":37},{"id":"refused_us","label":"Homepage refuses CrawlCheck's census fetch (400, 401, 403 or 429): blocked or rate-limited, not down","points":1.4,"sites_gained":25,"sites_lost":11},{"id":"refused_perplexity","label":"PerplexityBot refused where the browser got 2xx","points":1,"sites_gained":6,"sites_lost":1},{"id":"ai_refused_any","label":"At least one AI identity is refused (non-2xx) where the browser got 2xx","points":0.8,"sites_gained":7,"sites_lost":3},{"id":"ai_thin_any","label":"At least one AI identity gets under 70% of the browser's bytes (both 2xx)","points":-0.8,"sites_gained":0,"sites_lost":4}]},{"frame":"top10k","label":"top 10,000, daily","runs_this_week":7,"run":{"run":19,"finished_at":"2026-11-15T03:08:26.484Z"},"compared_with":{"run":12,"finished_at":"2026-11-08T00:06:35.271Z"},"sites":9776,"figures":[{"id":"reachable","group":"Reach","label":"Homepage answers CrawlCheck's census fetch (2xx after redirects)","count":5671,"of":9776,"pct":58,"pct_before":58.1,"points_all":-0.1,"points":-0.1,"sites_gained":141,"sites_lost":150,"compared":9776,"denominator_entered":0,"denominator_left":0},{"id":"refused_us","group":"Reach","label":"Homepage refuses CrawlCheck's census fetch (400, 401, 403 or 429): blocked or rate-limited, not down","count":1339,"of":9776,"pct":13.7,"pct_before":13.5,"points_all":0.2,"points":0.2,"sites_gained":103,"sites_lost":88,"compared":9776,"denominator_entered":0,"denominator_left":0},{"id":"no_site","group":"Reach","label":"No website answers at that name (5xx, 530, timeout or no answer)","count":2563,"of":9776,"pct":26.2,"pct_before":26.3,"points_all":-0.1,"points":-0.1,"sites_gained":60,"sites_lost":67,"compared":9776,"denominator_entered":0,"denominator_left":0},{"id":"robots_served","group":"Crawler policy","label":"Serves a robots.txt (200, not HTML)","count":4636,"of":5671,"pct":81.7,"pct_before":81.9,"points_all":-0.2,"points":0,"sites_gained":31,"sites_lost":30,"compared":5530,"denominator_entered":141,"denominator_left":150},{"id":"robots_sitemap","group":"Crawler policy","label":"robots.txt names a sitemap","count":3264,"of":4636,"pct":70.4,"pct_before":70.2,"points_all":0.2,"points":0.1,"sites_gained":9,"sites_lost":5,"compared":4498,"denominator_entered":138,"denominator_left":153},{"id":"content_signal","group":"Crawler policy","label":"robots.txt carries a Content-Signal line","count":301,"of":4636,"pct":6.5,"pct_before":6.3,"points_all":0.2,"points":0.1,"sites_gained":7,"sites_lost":2,"compared":4498,"denominator_entered":138,"denominator_left":153},{"id":"cs_ai_train_no","group":"Crawler policy","label":"Content-Signal says ai-train=no","count":231,"of":4636,"pct":5,"pct_before":4.9,"points_all":0.1,"points":0.1,"sites_gained":5,"sites_lost":2,"compared":4498,"denominator_entered":138,"denominator_left":153},{"id":"cs_ai_input_no","group":"Crawler policy","label":"Content-Signal says ai-input=no or search=no","count":22,"of":4636,"pct":0.5,"pct_before":0.5,"points_all":0,"points":0,"sites_gained":1,"sites_lost":0,"compared":4498,"denominator_entered":138,"denominator_left":153},{"id":"aipref_train","group":"Crawler policy","label":"robots.txt names the AI-training policy token (__usage_ai-training, IETF AIPREF draft)","count":17,"of":4632,"pct":0.4,"pct_before":0.3,"points_all":0.1,"points":0.1,"sites_gained":4,"sites_lost":0,"compared":4494,"denominator_entered":138,"denominator_left":153},{"id":"aipref_train_no","group":"Crawler policy","label":"robots.txt states a preference against AI training on / with that token","count":17,"of":4632,"pct":0.4,"pct_before":0.3,"points_all":0.1,"points":0.1,"sites_gained":4,"sites_lost":0,"compared":4494,"denominator_entered":138,"denominator_left":153},{"id":"blocks_any_ai","group":"Crawler policy","label":"robots.txt closes / to at least one of the 12 AI agents","count":1016,"of":4636,"pct":21.9,"pct_before":22,"points_all":-0.1,"points":0,"sites_gained":5,"sites_lost":3,"compared":4498,"denominator_entered":138,"denominator_left":153},{"id":"blocks_GPTBot","group":"Crawler policy","label":"robots.txt closes / to GPTBot","count":746,"of":4636,"pct":16.1,"pct_before":16.3,"points_all":-0.2,"points":-0.1,"sites_gained":3,"sites_lost":6,"compared":4498,"denominator_entered":138,"denominator_left":153},{"id":"blocks_OAI-SearchBot","group":"Crawler policy","label":"robots.txt closes / to OAI-SearchBot","count":280,"of":4636,"pct":6,"pct_before":6.1,"points_all":-0.1,"points":0,"sites_gained":2,"sites_lost":3,"compared":4498,"denominator_entered":138,"denominator_left":153},{"id":"blocks_ChatGPT-User","group":"Crawler policy","label":"robots.txt closes / to ChatGPT-User","count":363,"of":4636,"pct":7.8,"pct_before":8,"points_all":-0.2,"points":0,"sites_gained":3,"sites_lost":4,"compared":4498,"denominator_entered":138,"denominator_left":153},{"id":"blocks_ClaudeBot","group":"Crawler policy","label":"robots.txt closes / to ClaudeBot","count":690,"of":4636,"pct":14.9,"pct_before":15,"points_all":-0.1,"points":0,"sites_gained":4,"sites_lost":4,"compared":4498,"denominator_entered":138,"denominator_left":153},{"id":"blocks_Claude-SearchBot","group":"Crawler policy","label":"robots.txt closes / to Claude-SearchBot","count":268,"of":4636,"pct":5.8,"pct_before":5.7,"points_all":0.1,"points":0,"sites_gained":3,"sites_lost":2,"compared":4498,"denominator_entered":138,"denominator_left":153},{"id":"blocks_PerplexityBot","group":"Crawler policy","label":"robots.txt closes / to PerplexityBot","count":410,"of":4636,"pct":8.8,"pct_before":8.9,"points_all":-0.1,"points":0,"sites_gained":1,"sites_lost":3,"compared":4498,"denominator_entered":138,"denominator_left":153},{"id":"blocks_Google-Extended","group":"Crawler policy","label":"robots.txt closes / to Google-Extended","count":615,"of":4636,"pct":13.3,"pct_before":13.4,"points_all":-0.1,"points":-0.2,"sites_gained":3,"sites_lost":11,"compared":4498,"denominator_entered":138,"denominator_left":153},{"id":"blocks_Applebot-Extended","group":"Crawler policy","label":"robots.txt closes / to Applebot-Extended","count":608,"of":4636,"pct":13.1,"pct_before":13.1,"points_all":0,"points":0,"sites_gained":6,"sites_lost":4,"compared":4498,"denominator_entered":138,"denominator_left":153},{"id":"blocks_CCBot","group":"Crawler policy","label":"robots.txt closes / to CCBot","count":792,"of":4636,"pct":17.1,"pct_before":17.1,"points_all":0,"points":0,"sites_gained":4,"sites_lost":4,"compared":4498,"denominator_entered":138,"denominator_left":153},{"id":"blocks_Bytespider","group":"Crawler policy","label":"robots.txt closes / to Bytespider","count":786,"of":4636,"pct":17,"pct_before":17,"points_all":0,"points":0.1,"sites_gained":6,"sites_lost":3,"compared":4498,"denominator_entered":138,"denominator_left":153},{"id":"blocks_Meta-ExternalAgent","group":"Crawler policy","label":"robots.txt closes / to Meta-ExternalAgent","count":644,"of":4636,"pct":13.9,"pct_before":13.9,"points_all":0,"points":0.1,"sites_gained":7,"sites_lost":4,"compared":4498,"denominator_entered":138,"denominator_left":153},{"id":"blocks_Amazonbot","group":"Crawler policy","label":"robots.txt closes / to Amazonbot","count":619,"of":4636,"pct":13.4,"pct_before":13.4,"points_all":0,"points":0,"sites_gained":5,"sites_lost":5,"compared":4498,"denominator_entered":138,"denominator_left":153},{"id":"llms_txt","group":"Machine files","label":"Serves /llms.txt","count":972,"of":5671,"pct":17.1,"pct_before":17,"points_all":0.1,"points":0.2,"sites_gained":14,"sites_lost":4,"compared":5530,"denominator_entered":141,"denominator_left":150},{"id":"agents_md","group":"Machine files","label":"Serves /agents.md","count":87,"of":5671,"pct":1.5,"pct_before":1.5,"points_all":0,"points":0,"sites_gained":3,"sites_lost":2,"compared":5530,"denominator_entered":141,"denominator_left":150},{"id":"agent_skills","group":"Capabilities","label":"Publishes an agent-skills index","count":61,"of":5671,"pct":1.1,"pct_before":1,"points_all":0.1,"points":0,"sites_gained":2,"sites_lost":0,"compared":5530,"denominator_entered":141,"denominator_left":150},{"id":"mcp_card","group":"Capabilities","label":"Publishes an MCP server card","count":68,"of":5671,"pct":1.2,"pct_before":1.1,"points_all":0.1,"points":0.1,"sites_gained":6,"sites_lost":0,"compared":5530,"denominator_entered":141,"denominator_left":150},{"id":"api_catalog","group":"Capabilities","label":"Publishes an API catalog (RFC 9727)","count":75,"of":5671,"pct":1.3,"pct_before":1.2,"points_all":0.1,"points":0.1,"sites_gained":6,"sites_lost":0,"compared":5530,"denominator_entered":141,"denominator_left":150},{"id":"a2a_card","group":"Capabilities","label":"Publishes an A2A agent card","count":18,"of":5671,"pct":0.3,"pct_before":0.3,"points_all":0,"points":0,"sites_gained":2,"sites_lost":0,"compared":5530,"denominator_entered":141,"denominator_left":150},{"id":"any_capability","group":"Capabilities","label":"Declares any agent capability (any of the four above)","count":90,"of":5671,"pct":1.6,"pct_before":1.4,"points_all":0.2,"points":0.1,"sites_gained":6,"sites_lost":0,"compared":5530,"denominator_entered":141,"denominator_left":150},{"id":"ai_refused_any","group":"Delivery parity","label":"At least one AI identity is refused (non-2xx) where the browser got 2xx","count":1291,"of":5671,"pct":22.8,"pct_before":23,"points_all":-0.2,"points":-0.1,"sites_gained":53,"sites_lost":61,"compared":5530,"denominator_entered":141,"denominator_left":150},{"id":"ai_thin_any","group":"Delivery parity","label":"At least one AI identity gets under 70% of the browser's bytes (both 2xx)","count":132,"of":5671,"pct":2.3,"pct_before":2.3,"points_all":0,"points":0,"sites_gained":13,"sites_lost":12,"compared":5530,"denominator_entered":141,"denominator_left":150},{"id":"mcp_declared","group":"Endpoint rings","label":"Names an MCP endpoint (server card, llms.txt, agents.md, a homepage link, or /mcp answers like one)","count":333,"of":5671,"pct":5.9,"pct_before":5.8,"points_all":0.1,"points":0.1,"sites_gained":7,"sites_lost":2,"compared":5530,"denominator_entered":141,"denominator_left":150},{"id":"mcp_live","group":"Endpoint rings","label":"An MCP endpoint completes the handshake (initialize and tools/list; no tool called)","count":54,"of":5671,"pct":1,"pct_before":0.9,"points_all":0.1,"points":0,"sites_gained":2,"sites_lost":2,"compared":5530,"denominator_entered":141,"denominator_left":150},{"id":"mcp_live_of_declared","group":"Endpoint rings","label":"Of sites naming an MCP endpoint, one completes the handshake or asks for credentials","count":261,"of":333,"pct":78.4,"pct_before":78.9,"points_all":-0.5,"points":-0.6,"sites_gained":0,"sites_lost":2,"compared":322,"denominator_entered":11,"denominator_left":5},{"id":"a2a_declared","group":"Endpoint rings","label":"The A2A agent card names an endpoint url","count":9,"of":5671,"pct":0.2,"pct_before":0.1,"points_all":0.1,"points":0,"sites_gained":1,"sites_lost":0,"compared":5530,"denominator_entered":141,"denominator_left":150},{"id":"a2a_answers","group":"Endpoint rings","label":"Of A2A cards naming a url, the url answers (any non-5xx, not an HTML page)","count":4,"of":9,"pct":44.4,"pct_before":37.5,"points_all":6.9,"points":0,"sites_gained":0,"sites_lost":0,"compared":8,"denominator_entered":1,"denominator_left":0},{"id":"webmcp","group":"Endpoint rings","label":"Homepage carries WebMCP (navigator.modelContext, a WebMCP bridge, or an mcp+json block)","count":5,"of":5671,"pct":0.1,"pct_before":0.1,"points_all":0,"points":0,"sites_gained":0,"sites_lost":0,"compared":5530,"denominator_entered":141,"denominator_left":150},{"id":"sb_listed","group":"Safety","label":"Listed by Google Safe Browsing (all or some pages): browsers show a red warning","count":38,"of":4393,"pct":0.9,"pct_before":0.9,"points_all":0,"points":null,"note":"not measured in the earlier run"},{"id":"cf_listed","group":"Safety","label":"Listed as malware or phishing by a public malware-blocking DNS resolver","count":16,"of":5687,"pct":0.3,"pct_before":0.3,"points_all":0,"points":null,"note":"not measured in the earlier run"},{"id":"hsts_preloaded","group":"Safety","label":"On the browsers' HSTS preload list (HTTPS-only from the first visit)","count":458,"of":5690,"pct":8,"pct_before":8,"points_all":0,"points":null,"note":"not measured in the earlier run"},{"id":"outside_scripts","group":"Safety","label":"Homepage loads scripts or styles from another domain","count":3808,"of":5690,"pct":66.9,"pct_before":67.5,"points_all":-0.6,"points":null,"note":"not measured in the earlier run"},{"id":"outside_5plus","group":"Safety","label":"Homepage loads scripts or styles from 5 or more other domains","count":612,"of":5690,"pct":10.8,"pct_before":10.9,"points_all":-0.1,"points":null,"note":"not measured in the earlier run"},{"id":"any_integrity","group":"Safety","label":"Pins at least one outside script or style with integrity=","count":375,"of":3808,"pct":9.8,"pct_before":10.1,"points_all":-0.3,"points":null,"note":"not measured in the earlier run"},{"id":"unpinned_cdn","group":"Safety","label":"Loads a library from a public CDN without integrity=","count":752,"of":5690,"pct":13.2,"pct_before":13.3,"points_all":-0.1,"points":null,"note":"not measured in the earlier run"},{"id":"hijacked_cdn","group":"Safety","label":"Still loads scripts from a CDN taken over in 2024 (polyfill.io and its sister CDNs)","count":4,"of":5690,"pct":0.1,"pct_before":0.1,"points_all":0,"points":null,"note":"not measured in the earlier run"},{"id":"hack_signs","group":"Safety","label":"Homepage shows hidden spam links or cryptominer code","count":1,"of":5690,"pct":0,"pct_before":0,"points_all":0,"points":null,"note":"not measured in the earlier run"},{"id":"refused_gptbot","group":"Delivery parity","label":"GPTBot refused where the browser got 2xx","count":1082,"of":5671,"pct":19.1,"pct_before":19.2,"points_all":-0.1,"points":-0.2,"sites_gained":32,"sites_lost":42,"compared":5530,"denominator_entered":141,"denominator_left":150},{"id":"refused_claudebot","group":"Delivery parity","label":"ClaudeBot refused where the browser got 2xx","count":1119,"of":5671,"pct":19.7,"pct_before":20,"points_all":-0.3,"points":-0.1,"sites_gained":49,"sites_lost":57,"compared":5530,"denominator_entered":141,"denominator_left":150},{"id":"refused_oaisearch","group":"Delivery parity","label":"OAI-SearchBot refused where the browser got 2xx","count":716,"of":5671,"pct":12.6,"pct_before":12.4,"points_all":0.2,"points":0.2,"sites_gained":34,"sites_lost":25,"compared":5530,"denominator_entered":141,"denominator_left":150},{"id":"refused_perplexity","group":"Delivery parity","label":"PerplexityBot refused where the browser got 2xx","count":887,"of":5671,"pct":15.6,"pct_before":14.5,"points_all":1.1,"points":1.2,"sites_gained":101,"sites_lost":36,"compared":5530,"denominator_entered":141,"denominator_left":150}],"movers":[{"id":"refused_perplexity","label":"PerplexityBot refused where the browser got 2xx","points":1.2,"sites_gained":101,"sites_lost":36},{"id":"mcp_live_of_declared","label":"Of sites naming an MCP endpoint, one completes the handshake or asks for credentials","points":-0.6,"sites_gained":0,"sites_lost":2},{"id":"refused_us","label":"Homepage refuses CrawlCheck's census fetch (400, 401, 403 or 429): blocked or rate-limited, not down","points":0.2,"sites_gained":103,"sites_lost":88},{"id":"refused_gptbot","label":"GPTBot refused where the browser got 2xx","points":-0.2,"sites_gained":32,"sites_lost":42},{"id":"refused_oaisearch","label":"OAI-SearchBot refused where the browser got 2xx","points":0.2,"sites_gained":34,"sites_lost":25}]}],"reading":"Each frame compares the last run that finished in this ISO week (UTC) with the last run that finished before it. Change is like-for-like: points = (sites gained - sites lost) / sites compared, counting only sites measured in both runs and inside the figure's denominator both times, so a site that did not answer in one run cannot move a figure. points_all is the plain difference of the two percentages, and denominator_entered / denominator_left count the sites that moved in or out of a figure's base (often a site that did not answer one run). No site is named; per-site records: https://crawlcheck.io/api/census/records.","method":"https://crawlcheck.io/api/census","page":"https://crawlcheck.io/data/census/weekly?week=2026-W46","sealed_at":"2026-11-16T00:19:14.606Z","editions":[{"week":"2026-W40","url":"https://crawlcheck.io/api/v1/census/weekly?week=2026-W40","archive":"https://crawlcheck.io/api/v1/census/weekly/2026-W40.json"},{"week":"2026-W41","url":"https://crawlcheck.io/api/v1/census/weekly?week=2026-W41","archive":"https://crawlcheck.io/api/v1/census/weekly/2026-W41.json"},{"week":"2026-W42","url":"https://crawlcheck.io/api/v1/census/weekly?week=2026-W42","archive":"https://crawlcheck.io/api/v1/census/weekly/2026-W42.json"},{"week":"2026-W43","url":"https://crawlcheck.io/api/v1/census/weekly?week=2026-W43","archive":"https://crawlcheck.io/api/v1/census/weekly/2026-W43.json"},{"week":"2026-W44","url":"https://crawlcheck.io/api/v1/census/weekly?week=2026-W44","archive":"https://crawlcheck.io/api/v1/census/weekly/2026-W44.json"},{"week":"2026-W45","url":"https://crawlcheck.io/api/v1/census/weekly?week=2026-W45","archive":"https://crawlcheck.io/api/v1/census/weekly/2026-W45.json"},{"week":"2026-W46","url":"https://crawlcheck.io/api/v1/census/weekly?week=2026-W46","archive":"https://crawlcheck.io/api/v1/census/weekly/2026-W46.json"}],"feed":"https://crawlcheck.io/api/v1/census/weekly/feed.xml","code":{"build_id":"9219fb519ff0eaf2","script_sha16":"b00e794a299c5a1b","state":"attested","deployed":{"at":"2026-11-21T18:41:28.011Z"},"build_log_entry":2899,"score_version":43,"scoring_policy_sha256":"3e08fccfbbce356dc82ecd302b32a226a36fc0fb3150a4ab72248fb5795372af","decision_rules_sha256":"18dd5636e0896692ab518050cb401744edea2e6f1086dff94fead314959bdc3e","check":"https://crawlcheck.io/api/v1/builds/9219fb519ff0eaf2"},"sha256":"20d22b01dbde7ad0e965f637259309b10430686dbde63a34d1396b3417f72872","signature":{"v":1,"alg":"Ed25519","kid":"1hHmE3fo1rg5Ig70wuFtJ5wBc7G-sUTgUJkxyPWFRps","over":"sha256","message":"crawlcheck-data-v1\n20d22b01dbde7ad0e965f637259309b10430686dbde63a34d1396b3417f72872","sig":"QQyLt8jC9XNhqd8ezHhg2gMaBvT9UsU3ZI7bTLqpZjhwaegTmuhj5nWOxfuGz5ytbuULhX6qciBBpaZjRvszBg","signed_at":"2026-11-22T14:11:57.457Z","key":{"kid":"1hHmE3fo1rg5Ig70wuFtJ5wBc7G-sUTgUJkxyPWFRps","jwk":{"kty":"OKP","crv":"Ed25519","x":"5TmKCOx0TTHepq7HwyXBHvGQpVCSBd2lV8cLzYVqFxQ"},"published_in":"https://crawlcheck.io/.well-known/http-message-signatures-directory"},"canonical":"sha256 over canonical JSON of every field except sha256 and signature"}}