{"kind":"crawlcheck-fix-recommendations","v":1,"domain":"comet.com","stack":{"platform":"WordPress","cdn":"Fastly","host":null,"bot_protection":null},"record_read_at":"2026-11-05T22:32:45.999Z","findings":[{"id":"llms_txt","problem":"serves no /llms.txt","fix_label":"Published llms.txt","optional":false,"similar_sites_fixed":19,"held_now":19,"all_sites_fixed":103,"examples":[{"site":"similar site A","stack":"WordPress + Fastly","shares":["platform","CDN"],"fixed_at":"2026-10-31","held_now":true,"ai_crawlers_served_now":"4 of 4"},{"site":"similar site B","stack":"WordPress","shares":["platform"],"fixed_at":"2026-11-11","held_now":true,"ai_crawlers_served_now":"5 of 13"},{"site":"similar site C","stack":"WordPress + Amazon CloudFront + WP Engine","shares":["platform"],"fixed_at":"2026-11-10","held_now":true,"ai_crawlers_served_now":"4 of 4"},{"site":"similar site D","stack":"WordPress + Cloudflare + Cloudflare Bot Management","shares":["platform"],"fixed_at":"2026-11-04","held_now":true,"ai_crawlers_served_now":"13 of 13"},{"site":"similar site E","stack":"WordPress","shares":["platform"],"fixed_at":"2026-10-30","held_now":true,"ai_crawlers_served_now":"13 of 13"}],"reading":"19 sites on a stack like yours fixed this; 19 still hold the fix today"},{"id":"entity_map","problem":"serves no /entitymap.json","fix_label":"Published an entity map","optional":true,"similar_sites_fixed":1,"held_now":1,"all_sites_fixed":2,"examples":[{"site":"similar site A","stack":"WordPress + Cloudflare","shares":["platform"],"fixed_at":"2026-10-29","held_now":true,"ai_crawlers_served_now":"10 of 10"}],"reading":"only 1 site(s) on a stack like yours have fixed this so far (2 on any stack)"}],"index":{"built_at":"2026-11-12T02:21:36.665Z","events_read":6061,"fix_events":221,"from_census":15,"sites":156,"fixes_by_problem":[{"problem":"agents_md","sites_fixed":22,"held":15},{"problem":"edge_refusal","sites_fixed":8,"held":6},{"problem":"entity_map","sites_fixed":2,"held":2},{"problem":"llms_txt","sites_fixed":103,"held":102},{"problem":"robots:Amazonbot","sites_fixed":8,"held":6},{"problem":"robots:Applebot-Extended","sites_fixed":5,"held":3},{"problem":"robots:Bytespider","sites_fixed":1,"held":1},{"problem":"robots:CCBot","sites_fixed":3,"held":3},{"problem":"robots:ChatGPT-User","sites_fixed":8,"held":8},{"problem":"robots:Claude-SearchBot","sites_fixed":10,"held":8},{"problem":"robots:ClaudeBot","sites_fixed":3,"held":3},{"problem":"robots:GPTBot","sites_fixed":4,"held":4},{"problem":"robots:Google-Extended","sites_fixed":8,"held":8},{"problem":"robots:OAI-SearchBot","sites_fixed":11,"held":9},{"problem":"robots:PerplexityBot","sites_fixed":11,"held":11},{"problem":"robots:meta-externalagent","sites_fixed":6,"held":4},{"problem":"served_less","sites_fixed":6,"held":5}],"change_events_by_kind":[["signal-on | Blocked <agent> in robots.txt",1350],["added | Added Cloudflare",430],["added | Added Google Tag Manager",378],["added | Added Google Analytics 4",245],["added | Added Akamai",223],["added | Verified its domain with OpenAI",217],["added | Added Cloudflare Bot Management",187],["added | Added Amazon CloudFront",124],["added | Added WordPress",111],["signal-on | Started enforcing its ClaudeBot block at the server",111],["added | Added Google Workspace",109],["signal-on | Started enforcing its GPTBot block at the server",109],["signal-on | Published llms.txt",102],["added | Added Microsoft Clarity",88],["signal-off | Stopped refusing GPTBot at the server",85]]},"reading":"Problems are read from CrawlCheck's technology record of the domain. For each, the sites whose stack shares the most with yours (platform 3, bot protection 2, CDN 2, host 1; at least 2) that had the same problem and fixed it, newest first among equals. A fix counts once two readings confirm it (two scans in the technology record, or two census runs with the problem followed by two without); held_now says whether the latest scan still shows it fixed, and ai_crawlers_served_now how many AI crawlers that site serves today. Sites are not named.","code":{"version_id":"d37a5446-b7f8-40d7-98a1-bc657596f162","script_sha16":"345b75392bb2daf9","state":"attested","deployed":{"at":"2026-11-12T18:15:40.913Z","commit":"ed32d2b13974","run":"43277732900"},"build_log_entry":1963,"score_version":34,"scoring_policy_sha256":"21af5c858fca26199e01525fec7b19b353df30151b1d29fdeba63a28b90d44a2","decision_rules_sha256":"18dd5636e0896692ab518050cb401744edea2e6f1086dff94fead314959bdc3e","check":"https://crawlcheck.io/api/v1/builds/d37a5446-b7f8-40d7-98a1-bc657596f162"},"sha256":"0de4c42c92dbb58651f3e58a3443ec78a6049f9c103c78b514a9f6a0d5c1e874","signature":{"v":1,"alg":"Ed25519","kid":"1hHmE3fo1rg5Ig70wuFtJ5wBc7G-sUTgUJkxyPWFRps","over":"sha256","message":"crawlcheck-data-v1\n0de4c42c92dbb58651f3e58a3443ec78a6049f9c103c78b514a9f6a0d5c1e874","sig":"6rqcCfo6VpvnWPxJ8x8kOYL0svf1-LLurrXYxLWbjpcFascQeRtGdZ8KNqIBYWjhODaKOQCbkJTfjp1WWyBDAA","signed_at":"2026-11-13T00:13:50.000Z","key":{"kid":"1hHmE3fo1rg5Ig70wuFtJ5wBc7G-sUTgUJkxyPWFRps","jwk":{"kty":"OKP","crv":"Ed25519","x":"5TmKCOx0TTHepq7HwyXBHvGQpVCSBd2lV8cLzYVqFxQ"},"published_in":"https://crawlcheck.io/.well-known/http-message-signatures-directory"},"canonical":"sha256 over canonical JSON of every field except sha256 and signature"}}