{"kind":"crawlcheck-fix-recommendations","v":1,"domain":"arize.com","stack":{"platform":"WordPress","cdn":"Cloudflare","host":"Kinsta","bot_protection":"Cloudflare Bot Management"},"record_read_at":"2026-11-02T12:37:40.641Z","findings":[{"id":"agents_md","problem":"serves no /agents.md","fix_label":"Published agents.md","optional":true,"similar_sites_fixed":9,"held_now":9,"all_sites_fixed":15,"examples":[{"site":"similar site A","stack":"WordPress + Cloudflare + Cloudflare Bot Management","shares":["platform","bot protection","CDN"],"fixed_at":"2026-10-31","held_now":true,"ai_crawlers_served_now":"4 of 4"},{"site":"similar site B","stack":"Cloudflare + Cloudflare Bot Management","shares":["bot protection","CDN"],"fixed_at":"2026-10-22","held_now":true,"ai_crawlers_served_now":"not measured"},{"site":"similar site C","stack":"WordPress","shares":["platform"],"fixed_at":"2026-10-30","held_now":true,"ai_crawlers_served_now":"10 of 10"},{"site":"similar site D","stack":"Cloudflare","shares":["CDN"],"fixed_at":"2026-11-01","held_now":true,"ai_crawlers_served_now":"10 of 10"},{"site":"similar site E","stack":"Cloudflare","shares":["CDN"],"fixed_at":"2026-10-24","held_now":true,"ai_crawlers_served_now":"4 of 4"}],"reading":"9 sites on a stack like yours fixed this; 9 still hold the fix today"},{"id":"entity_map","problem":"serves no /entitymap.json","fix_label":"Published an entity map","optional":true,"similar_sites_fixed":2,"held_now":2,"all_sites_fixed":2,"examples":[{"site":"similar site A","stack":"WordPress + Cloudflare","shares":["platform","CDN"],"fixed_at":"2026-10-29","held_now":true,"ai_crawlers_served_now":"10 of 10"},{"site":"similar site B","stack":"Cloudflare","shares":["CDN"],"fixed_at":"2026-10-15","held_now":true,"ai_crawlers_served_now":"10 of 10"}],"reading":"only 2 site(s) on a stack like yours have fixed this so far (2 on any stack)"}],"index":{"built_at":"2026-11-03T13:23:32.415Z","events_read":1765,"fix_events":172,"from_census":18,"sites":123,"fixes_by_problem":[{"problem":"agents_md","sites_fixed":15,"held":15},{"problem":"edge_refusal","sites_fixed":14,"held":11},{"problem":"entity_map","sites_fixed":2,"held":2},{"problem":"llms_txt","sites_fixed":78,"held":75},{"problem":"robots:Amazonbot","sites_fixed":6,"held":6},{"problem":"robots:Applebot-Extended","sites_fixed":5,"held":5},{"problem":"robots:Bytespider","sites_fixed":2,"held":2},{"problem":"robots:CCBot","sites_fixed":3,"held":3},{"problem":"robots:ChatGPT-User","sites_fixed":4,"held":4},{"problem":"robots:Claude-SearchBot","sites_fixed":8,"held":8},{"problem":"robots:ClaudeBot","sites_fixed":2,"held":2},{"problem":"robots:GPTBot","sites_fixed":3,"held":3},{"problem":"robots:Google-Extended","sites_fixed":4,"held":4},{"problem":"robots:OAI-SearchBot","sites_fixed":8,"held":8},{"problem":"robots:PerplexityBot","sites_fixed":7,"held":7},{"problem":"robots:meta-externalagent","sites_fixed":6,"held":6},{"problem":"served_less","sites_fixed":4,"held":3}],"change_events_by_kind":[["signal-on | Blocked <agent> in robots.txt",472],["added | Added Akamai",76],["signal-on | Published llms.txt",75],["added | Added Google Tag Manager",65],["signal-off | Unblocked <agent> in robots.txt",57],["removed | Removed Akamai",56],["added | Verified its domain with OpenAI",52],["signal-on | Started enforcing its GPTBot block at the server",41],["signal-on | Started enforcing its ClaudeBot block at the server",41],["added | Added Cloudflare",40],["removed | Removed Google Tag Manager",31],["signal-off | Stopped refusing ClaudeBot at the server",28],["signal-off | Stopped refusing GPTBot at the server",25],["added | Added Cloudflare Bot Management",25],["signal-on | Started refusing ClaudeBot at the server while robots.txt allows it",24]]},"reading":"Problems are read from CrawlCheck's technology record of the domain. For each, the sites whose stack shares the most with yours (platform 3, bot protection 2, CDN 2, host 1; at least 2) that had the same problem and fixed it, newest first among equals. A fix counts once two readings confirm it (two scans in the technology record, or two census runs with the problem followed by two without); held_now says whether the latest scan still shows it fixed, and ai_crawlers_served_now how many AI crawlers that site serves today. Sites are not named.","code":{"version_id":"13ebd64d-8ec8-461a-8c3f-bf37428f7cee","script_sha16":"1b0983be13c5ebd0","state":"attested","deployed":{"at":"2026-11-03T05:25:06.329Z","commit":"58aa604b1abd","run":"41583702043"},"build_log_entry":1393,"score_version":34,"scoring_policy_sha256":"76ee7526c402d711f2c976ad88ae13d4d3411aaa9d5ca3dc87ab3102396b1370","decision_rules_sha256":"18dd5636e0896692ab518050cb401744edea2e6f1086dff94fead314959bdc3e","check":"https://crawlcheck.io/api/v1/builds/13ebd64d-8ec8-461a-8c3f-bf37428f7cee"},"sha256":"59b18ff8c8590769d6e6c44984e16e25de4438f17a0c8a461365ab29b478daf0","signature":{"v":1,"alg":"Ed25519","kid":"1hHmE3fo1rg5Ig70wuFtJ5wBc7G-sUTgUJkxyPWFRps","over":"sha256","message":"crawlcheck-data-v1\n59b18ff8c8590769d6e6c44984e16e25de4438f17a0c8a461365ab29b478daf0","sig":"fTU89s_KwOyBTBoeLlhUmxFAbuu-qoTiapUwzzJXU-EGWT8Olbt5Wda2dSoU4VyAj5TBXZayUNBIc3ODpxonAQ","signed_at":"2026-11-03T17:41:00.626Z","key":{"kid":"1hHmE3fo1rg5Ig70wuFtJ5wBc7G-sUTgUJkxyPWFRps","jwk":{"kty":"OKP","crv":"Ed25519","x":"5TmKCOx0TTHepq7HwyXBHvGQpVCSBd2lV8cLzYVqFxQ"},"published_in":"https://crawlcheck.io/.well-known/http-message-signatures-directory"},"canonical":"sha256 over canonical JSON of every field except sha256 and signature"}}