{"kind":"crawlcheck-fix-recommendations","v":1,"domain":"denverpost.com","stack":{"platform":"WordPress","cdn":null,"host":null,"bot_protection":null},"record_read_at":"2026-11-12T00:18:39.809Z","findings":[{"id":"llms_txt","problem":"serves no /llms.txt","fix_label":"Published llms.txt","optional":false,"similar_sites_fixed":13,"held_now":13,"all_sites_fixed":103,"examples":[{"site":"similar site A","stack":"WordPress","shares":["platform"],"fixed_at":"2026-11-11","held_now":true,"ai_crawlers_served_now":"5 of 13"},{"site":"similar site B","stack":"WordPress + Amazon CloudFront + WP Engine","shares":["platform"],"fixed_at":"2026-11-10","held_now":true,"ai_crawlers_served_now":"4 of 4"},{"site":"similar site C","stack":"WordPress + Cloudflare + Cloudflare Bot Management","shares":["platform"],"fixed_at":"2026-11-04","held_now":true,"ai_crawlers_served_now":"13 of 13"},{"site":"similar site D","stack":"WordPress + Fastly","shares":["platform"],"fixed_at":"2026-10-31","held_now":true,"ai_crawlers_served_now":"4 of 4"},{"site":"similar site E","stack":"WordPress","shares":["platform"],"fixed_at":"2026-10-30","held_now":true,"ai_crawlers_served_now":"13 of 13"}],"reading":"13 sites on a stack like yours fixed this; 13 still hold the fix today"},{"id":"agents_md","problem":"serves no /agents.md","fix_label":"Published agents.md","optional":true,"similar_sites_fixed":5,"held_now":4,"all_sites_fixed":22,"examples":[{"site":"similar site A","stack":"WordPress + Cloudflare","shares":["platform"],"fixed_at":"2026-11-09","held_now":true,"ai_crawlers_served_now":"10 of 10"},{"site":"similar site B","stack":"WordPress + Cloudflare + Cloudflare Bot Management","shares":["platform"],"fixed_at":"2026-11-04","held_now":true,"ai_crawlers_served_now":"13 of 13"},{"site":"similar site C","stack":"WordPress + Cloudflare + Cloudflare Bot Management","shares":["platform"],"fixed_at":"2026-10-31","held_now":true,"ai_crawlers_served_now":"4 of 4"},{"site":"similar site D","stack":"WordPress","shares":["platform"],"fixed_at":"2026-10-30","held_now":true,"ai_crawlers_served_now":"13 of 13"},{"site":"similar site E","stack":"WordPress + Cloudflare","shares":["platform"],"fixed_at":"2026-11-10","held_now":false,"ai_crawlers_served_now":"13 of 13"}],"reading":"5 sites on a stack like yours fixed this; 4 still hold the fix today"},{"id":"entity_map","problem":"serves no /entitymap.json","fix_label":"Published an entity map","optional":true,"similar_sites_fixed":1,"held_now":1,"all_sites_fixed":2,"examples":[{"site":"similar site A","stack":"WordPress + Cloudflare","shares":["platform"],"fixed_at":"2026-10-29","held_now":true,"ai_crawlers_served_now":"10 of 10"}],"reading":"only 1 site(s) on a stack like yours have fixed this so far (2 on any stack)"},{"id":"robots:OAI-SearchBot","problem":"blocks OAI-SearchBot in robots.txt","fix_label":"Unblocked OAI-SearchBot in robots.txt","optional":true,"note":"only a problem if the block is not deliberate","similar_sites_fixed":1,"held_now":1,"all_sites_fixed":11,"examples":[{"site":"similar site A","stack":"WordPress + Fastly + Pantheon","shares":["platform"],"fixed_at":"2026-10-31","held_now":true,"ai_crawlers_served_now":"4 of 4"}],"reading":"only 1 site(s) on a stack like yours have fixed this so far (11 on any stack)"},{"id":"robots:ChatGPT-User","problem":"blocks ChatGPT-User in robots.txt","fix_label":"Unblocked ChatGPT-User in robots.txt","optional":true,"note":"only a problem if the block is not deliberate","similar_sites_fixed":2,"held_now":2,"all_sites_fixed":8,"examples":[{"site":"similar site A","stack":"WordPress + Amazon CloudFront + Netlify","shares":["platform"],"fixed_at":"2026-11-06","held_now":true,"ai_crawlers_served_now":"3 of 4"},{"site":"similar site B","stack":"WordPress + Fastly + Pantheon","shares":["platform"],"fixed_at":"2026-10-31","held_now":true,"ai_crawlers_served_now":"4 of 4"}],"reading":"only 2 site(s) on a stack like yours have fixed this so far (8 on any stack)"},{"id":"robots:Claude-SearchBot","problem":"blocks Claude-SearchBot in robots.txt","fix_label":"Unblocked Claude-SearchBot in robots.txt","optional":true,"note":"only a problem if the block is not deliberate","similar_sites_fixed":1,"held_now":1,"all_sites_fixed":10,"examples":[{"site":"similar site A","stack":"WordPress + Fastly + Pantheon","shares":["platform"],"fixed_at":"2026-10-31","held_now":true,"ai_crawlers_served_now":"4 of 4"}],"reading":"only 1 site(s) on a stack like yours have fixed this so far (10 on any stack)"},{"id":"robots:Claude-User","problem":"blocks Claude-User in robots.txt","fix_label":"Unblocked Claude-User in robots.txt","optional":true,"note":"only a problem if the block is not deliberate","similar_sites_fixed":0,"held_now":0,"all_sites_fixed":0,"examples":[],"reading":"no site in the record has fixed this yet"},{"id":"robots:PerplexityBot","problem":"blocks PerplexityBot in robots.txt","fix_label":"Unblocked PerplexityBot in robots.txt","optional":true,"note":"only a problem if the block is not deliberate","similar_sites_fixed":4,"held_now":4,"all_sites_fixed":11,"examples":[{"site":"similar site A","stack":"WordPress + Amazon CloudFront + Netlify","shares":["platform"],"fixed_at":"2026-11-06","held_now":true,"ai_crawlers_served_now":"3 of 4"},{"site":"similar site B","stack":"WordPress + Fastly + Pantheon","shares":["platform"],"fixed_at":"2026-10-31","held_now":true,"ai_crawlers_served_now":"4 of 4"},{"site":"similar site C","stack":"WordPress + Cloudflare + SiteGround + Cloudflare Bot Management","shares":["platform"],"fixed_at":"2026-10-31","held_now":true,"ai_crawlers_served_now":"3 of 4"},{"site":"similar site D","stack":"WordPress + SiteGround","shares":["platform"],"fixed_at":"2026-10-31","held_now":true,"ai_crawlers_served_now":"4 of 4"}],"reading":"4 sites on a stack like yours fixed this; 4 still hold the fix today"},{"id":"robots:Perplexity-User","problem":"blocks Perplexity-User in robots.txt","fix_label":"Unblocked Perplexity-User in robots.txt","optional":true,"note":"only a problem if the block is not deliberate","similar_sites_fixed":0,"held_now":0,"all_sites_fixed":0,"examples":[],"reading":"no site in the record has fixed this yet"},{"id":"robots:Amazonbot","problem":"blocks Amazonbot in robots.txt","fix_label":"Unblocked Amazonbot in robots.txt","optional":true,"note":"only a problem if the block is not deliberate","similar_sites_fixed":3,"held_now":3,"all_sites_fixed":8,"examples":[{"site":"similar site A","stack":"WordPress + Fastly","shares":["platform"],"fixed_at":"2026-11-10","held_now":true,"ai_crawlers_served_now":"1 of 13"},{"site":"similar site B","stack":"WordPress + Fastly + Google Cloud","shares":["platform"],"fixed_at":"2026-11-06","held_now":true,"ai_crawlers_served_now":"3 of 4"},{"site":"similar site C","stack":"WordPress + Fastly + Pantheon","shares":["platform"],"fixed_at":"2026-10-31","held_now":true,"ai_crawlers_served_now":"4 of 4"}],"reading":"3 sites on a stack like yours fixed this; 3 still hold the fix today"},{"id":"robots:GPTBot","problem":"blocks GPTBot in robots.txt","fix_label":"Unblocked GPTBot in robots.txt","optional":true,"note":"only a problem if the block is not deliberate","similar_sites_fixed":1,"held_now":1,"all_sites_fixed":4,"examples":[{"site":"similar site A","stack":"WordPress + Fastly + Pantheon","shares":["platform"],"fixed_at":"2026-10-31","held_now":true,"ai_crawlers_served_now":"4 of 4"}],"reading":"only 1 site(s) on a stack like yours have fixed this so far (4 on any stack)"},{"id":"robots:ClaudeBot","problem":"blocks ClaudeBot in robots.txt","fix_label":"Unblocked ClaudeBot in robots.txt","optional":true,"note":"only a problem if the block is not deliberate","similar_sites_fixed":1,"held_now":1,"all_sites_fixed":3,"examples":[{"site":"similar site A","stack":"WordPress + Fastly + Pantheon","shares":["platform"],"fixed_at":"2026-10-31","held_now":true,"ai_crawlers_served_now":"4 of 4"}],"reading":"only 1 site(s) on a stack like yours have fixed this so far (3 on any stack)"},{"id":"robots:Google-Extended","problem":"blocks Google-Extended in robots.txt","fix_label":"Unblocked Google-Extended in robots.txt","optional":true,"note":"only a problem if the block is not deliberate","similar_sites_fixed":2,"held_now":2,"all_sites_fixed":8,"examples":[{"site":"similar site A","stack":"WordPress + Amazon CloudFront + Netlify","shares":["platform"],"fixed_at":"2026-11-06","held_now":true,"ai_crawlers_served_now":"3 of 4"},{"site":"similar site B","stack":"WordPress + Fastly + Pantheon","shares":["platform"],"fixed_at":"2026-10-31","held_now":true,"ai_crawlers_served_now":"4 of 4"}],"reading":"only 2 site(s) on a stack like yours have fixed this so far (8 on any stack)"},{"id":"robots:Applebot-Extended","problem":"blocks Applebot-Extended in robots.txt","fix_label":"Unblocked Applebot-Extended in robots.txt","optional":true,"note":"only a problem if the block is not deliberate","similar_sites_fixed":1,"held_now":1,"all_sites_fixed":5,"examples":[{"site":"similar site A","stack":"WordPress + Fastly + Pantheon","shares":["platform"],"fixed_at":"2026-10-31","held_now":true,"ai_crawlers_served_now":"4 of 4"}],"reading":"only 1 site(s) on a stack like yours have fixed this so far (5 on any stack)"},{"id":"robots:CCBot","problem":"blocks CCBot in robots.txt","fix_label":"Unblocked CCBot in robots.txt","optional":true,"note":"only a problem if the block is not deliberate","similar_sites_fixed":1,"held_now":1,"all_sites_fixed":3,"examples":[{"site":"similar site A","stack":"WordPress + Fastly + Pantheon","shares":["platform"],"fixed_at":"2026-10-31","held_now":true,"ai_crawlers_served_now":"4 of 4"}],"reading":"only 1 site(s) on a stack like yours have fixed this so far (3 on any stack)"},{"id":"robots:Bytespider","problem":"blocks Bytespider in robots.txt","fix_label":"Unblocked Bytespider in robots.txt","optional":true,"note":"only a problem if the block is not deliberate","similar_sites_fixed":1,"held_now":1,"all_sites_fixed":1,"examples":[{"site":"similar site A","stack":"WordPress + Fastly + Pantheon","shares":["platform"],"fixed_at":"2026-10-31","held_now":true,"ai_crawlers_served_now":"4 of 4"}],"reading":"only 1 site(s) on a stack like yours have fixed this so far (1 on any stack)"},{"id":"robots:meta-externalagent","problem":"blocks meta-externalagent in robots.txt","fix_label":"Unblocked meta-externalagent in robots.txt","optional":true,"note":"only a problem if the block is not deliberate","similar_sites_fixed":1,"held_now":1,"all_sites_fixed":6,"examples":[{"site":"similar site A","stack":"WordPress + Fastly + Pantheon","shares":["platform"],"fixed_at":"2026-10-31","held_now":true,"ai_crawlers_served_now":"4 of 4"}],"reading":"only 1 site(s) on a stack like yours have fixed this so far (6 on any stack)"}],"index":{"built_at":"2026-11-12T02:21:36.665Z","events_read":6061,"fix_events":221,"from_census":15,"sites":156,"fixes_by_problem":[{"problem":"agents_md","sites_fixed":22,"held":15},{"problem":"edge_refusal","sites_fixed":8,"held":6},{"problem":"entity_map","sites_fixed":2,"held":2},{"problem":"llms_txt","sites_fixed":103,"held":102},{"problem":"robots:Amazonbot","sites_fixed":8,"held":6},{"problem":"robots:Applebot-Extended","sites_fixed":5,"held":3},{"problem":"robots:Bytespider","sites_fixed":1,"held":1},{"problem":"robots:CCBot","sites_fixed":3,"held":3},{"problem":"robots:ChatGPT-User","sites_fixed":8,"held":8},{"problem":"robots:Claude-SearchBot","sites_fixed":10,"held":8},{"problem":"robots:ClaudeBot","sites_fixed":3,"held":3},{"problem":"robots:GPTBot","sites_fixed":4,"held":4},{"problem":"robots:Google-Extended","sites_fixed":8,"held":8},{"problem":"robots:OAI-SearchBot","sites_fixed":11,"held":9},{"problem":"robots:PerplexityBot","sites_fixed":11,"held":11},{"problem":"robots:meta-externalagent","sites_fixed":6,"held":4},{"problem":"served_less","sites_fixed":6,"held":5}],"change_events_by_kind":[["signal-on | Blocked <agent> in robots.txt",1350],["added | Added Cloudflare",430],["added | Added Google Tag Manager",378],["added | Added Google Analytics 4",245],["added | Added Akamai",223],["added | Verified its domain with OpenAI",217],["added | Added Cloudflare Bot Management",187],["added | Added Amazon CloudFront",124],["added | Added WordPress",111],["signal-on | Started enforcing its ClaudeBot block at the server",111],["added | Added Google Workspace",109],["signal-on | Started enforcing its GPTBot block at the server",109],["signal-on | Published llms.txt",102],["added | Added Microsoft Clarity",88],["signal-off | Stopped refusing GPTBot at the server",85]]},"reading":"Problems are read from CrawlCheck's technology record of the domain. For each, the sites whose stack shares the most with yours (platform 3, bot protection 2, CDN 2, host 1; at least 2) that had the same problem and fixed it, newest first among equals. A fix counts once two readings confirm it (two scans in the technology record, or two census runs with the problem followed by two without); held_now says whether the latest scan still shows it fixed, and ai_crawlers_served_now how many AI crawlers that site serves today. Sites are not named.","code":{"version_id":"1d1d52e5-dc11-4eb5-b9c5-998383eb8746","script_sha16":"6f23bc3f68806651","state":"attested","deployed":{"at":"2026-11-12T16:26:36.442Z","commit":"6c016a5ea940","run":"43258445498"},"build_log_entry":1961,"score_version":34,"scoring_policy_sha256":"21af5c858fca26199e01525fec7b19b353df30151b1d29fdeba63a28b90d44a2","decision_rules_sha256":"18dd5636e0896692ab518050cb401744edea2e6f1086dff94fead314959bdc3e","check":"https://crawlcheck.io/api/v1/builds/1d1d52e5-dc11-4eb5-b9c5-998383eb8746"},"sha256":"943b093ae63b533030cf6a4bfa6c4b02abfef1fc5b1a777d869eb9e97ffe592b","signature":{"v":1,"alg":"Ed25519","kid":"1hHmE3fo1rg5Ig70wuFtJ5wBc7G-sUTgUJkxyPWFRps","over":"sha256","message":"crawlcheck-data-v1\n943b093ae63b533030cf6a4bfa6c4b02abfef1fc5b1a777d869eb9e97ffe592b","sig":"mkzHNjhVNW45X_u6HRxFylSm5fYTbT0XnjidP5Tbdh8n9utzw5PPiK03L2iEN9P902p4jMpr046evlkABEFrAA","signed_at":"2026-11-12T16:58:15.941Z","key":{"kid":"1hHmE3fo1rg5Ig70wuFtJ5wBc7G-sUTgUJkxyPWFRps","jwk":{"kty":"OKP","crv":"Ed25519","x":"5TmKCOx0TTHepq7HwyXBHvGQpVCSBd2lV8cLzYVqFxQ"},"published_in":"https://crawlcheck.io/.well-known/http-message-signatures-directory"},"canonical":"sha256 over canonical JSON of every field except sha256 and signature"}}