{"kind":"crawlcheck-fix-recommendations","v":1,"domain":"datarobot.com","stack":{"platform":"WordPress","cdn":"Fastly","host":"Pantheon","bot_protection":null},"record_read_at":"2026-11-08T15:41:49.645Z","findings":[{"id":"agents_md","problem":"serves no /agents.md","fix_label":"Published agents.md","optional":true,"similar_sites_fixed":6,"held_now":5,"all_sites_fixed":22,"examples":[{"site":"similar site A","stack":"WordPress + Cloudflare","shares":["platform"],"fixed_at":"2026-11-09","held_now":true,"ai_crawlers_served_now":"10 of 10"},{"site":"similar site B","stack":"WordPress + Cloudflare + Cloudflare Bot Management","shares":["platform"],"fixed_at":"2026-11-04","held_now":true,"ai_crawlers_served_now":"13 of 13"},{"site":"similar site C","stack":"WordPress + Cloudflare + Cloudflare Bot Management","shares":["platform"],"fixed_at":"2026-10-31","held_now":true,"ai_crawlers_served_now":"4 of 4"},{"site":"similar site D","stack":"WordPress","shares":["platform"],"fixed_at":"2026-10-30","held_now":true,"ai_crawlers_served_now":"13 of 13"},{"site":"similar site E","stack":"WordPress + Cloudflare","shares":["platform"],"fixed_at":"2026-11-10","held_now":false,"ai_crawlers_served_now":"13 of 13"}],"reading":"6 sites on a stack like yours fixed this; 5 still hold the fix today"},{"id":"entity_map","problem":"serves no /entitymap.json","fix_label":"Published an entity map","optional":true,"similar_sites_fixed":1,"held_now":1,"all_sites_fixed":2,"examples":[{"site":"similar site A","stack":"WordPress + Cloudflare","shares":["platform"],"fixed_at":"2026-10-29","held_now":true,"ai_crawlers_served_now":"10 of 10"}],"reading":"only 1 site(s) on a stack like yours have fixed this so far (2 on any stack)"}],"index":{"built_at":"2026-11-12T02:21:36.665Z","events_read":6061,"fix_events":221,"from_census":15,"sites":156,"fixes_by_problem":[{"problem":"agents_md","sites_fixed":22,"held":15},{"problem":"edge_refusal","sites_fixed":8,"held":6},{"problem":"entity_map","sites_fixed":2,"held":2},{"problem":"llms_txt","sites_fixed":103,"held":102},{"problem":"robots:Amazonbot","sites_fixed":8,"held":6},{"problem":"robots:Applebot-Extended","sites_fixed":5,"held":3},{"problem":"robots:Bytespider","sites_fixed":1,"held":1},{"problem":"robots:CCBot","sites_fixed":3,"held":3},{"problem":"robots:ChatGPT-User","sites_fixed":8,"held":8},{"problem":"robots:Claude-SearchBot","sites_fixed":10,"held":8},{"problem":"robots:ClaudeBot","sites_fixed":3,"held":3},{"problem":"robots:GPTBot","sites_fixed":4,"held":4},{"problem":"robots:Google-Extended","sites_fixed":8,"held":8},{"problem":"robots:OAI-SearchBot","sites_fixed":11,"held":9},{"problem":"robots:PerplexityBot","sites_fixed":11,"held":11},{"problem":"robots:meta-externalagent","sites_fixed":6,"held":4},{"problem":"served_less","sites_fixed":6,"held":5}],"change_events_by_kind":[["signal-on | Blocked <agent> in robots.txt",1350],["added | Added Cloudflare",430],["added | Added Google Tag Manager",378],["added | Added Google Analytics 4",245],["added | Added Akamai",223],["added | Verified its domain with OpenAI",217],["added | Added Cloudflare Bot Management",187],["added | Added Amazon CloudFront",124],["added | Added WordPress",111],["signal-on | Started enforcing its ClaudeBot block at the server",111],["added | Added Google Workspace",109],["signal-on | Started enforcing its GPTBot block at the server",109],["signal-on | Published llms.txt",102],["added | Added Microsoft Clarity",88],["signal-off | Stopped refusing GPTBot at the server",85]]},"reading":"Problems are read from CrawlCheck's technology record of the domain. For each, the sites whose stack shares the most with yours (platform 3, bot protection 2, CDN 2, host 1; at least 2) that had the same problem and fixed it, newest first among equals. A fix counts once two readings confirm it (two scans in the technology record, or two census runs with the problem followed by two without); held_now says whether the latest scan still shows it fixed, and ai_crawlers_served_now how many AI crawlers that site serves today. Sites are not named.","code":{"version_id":"d37a5446-b7f8-40d7-98a1-bc657596f162","script_sha16":"345b75392bb2daf9","state":"attested","deployed":{"at":"2026-11-12T18:15:40.913Z","commit":"ed32d2b13974","run":"43277732900"},"build_log_entry":1963,"score_version":34,"scoring_policy_sha256":"21af5c858fca26199e01525fec7b19b353df30151b1d29fdeba63a28b90d44a2","decision_rules_sha256":"18dd5636e0896692ab518050cb401744edea2e6f1086dff94fead314959bdc3e","check":"https://crawlcheck.io/api/v1/builds/d37a5446-b7f8-40d7-98a1-bc657596f162"},"sha256":"245028fe76092923eab0d17665a40fe9200c356c05008789bc3ac3ee5464adb7","signature":{"v":1,"alg":"Ed25519","kid":"1hHmE3fo1rg5Ig70wuFtJ5wBc7G-sUTgUJkxyPWFRps","over":"sha256","message":"crawlcheck-data-v1\n245028fe76092923eab0d17665a40fe9200c356c05008789bc3ac3ee5464adb7","sig":"rLfvN5wjsvDu5BnMKMCdYiXSJgJ60Vm3T777aDT9AknjY_34KeYTVFcWQDU2Etp_r0t6dkvr26kL8cI5QxH9Cg","signed_at":"2026-11-13T00:13:52.525Z","key":{"kid":"1hHmE3fo1rg5Ig70wuFtJ5wBc7G-sUTgUJkxyPWFRps","jwk":{"kty":"OKP","crv":"Ed25519","x":"5TmKCOx0TTHepq7HwyXBHvGQpVCSBd2lV8cLzYVqFxQ"},"published_in":"https://crawlcheck.io/.well-known/http-message-signatures-directory"},"canonical":"sha256 over canonical JSON of every field except sha256 and signature"}}