{"ok":true,"kind":"crawlcheck-dataset-problems","v":1,"updated":"2026-11-22T00:49:58.527Z","days":["2026-11-21","2026-11-20","2026-11-19","2026-11-18","2026-11-17","2026-11-16","2026-11-15"],"tiers":[{"up_to_sites":1000,"price":99},{"up_to_sites":10000,"price":199},{"up_to_sites":null,"price":299}],"problems":[{"code":"CSP_REPORTING_BROKEN","title":"CSP violation reporting is misconfigured","source":"daily dataset","sites":92,"sites_label":"92","for_sale":false,"price":null,"page":"https://crawlcheck.io/datasets/problems/csp-reporting-broken"},{"code":"HSTS_MISSING","title":"HTTPS is served without a Strict-Transport-Security header","source":"daily dataset","sites":65059,"sites_label":"65059","for_sale":true,"price":299,"page":"https://crawlcheck.io/datasets/problems/hsts-missing"},{"code":"HSTS_PRELOAD_INELIGIBLE","title":"The HSTS header asks for browser preloading but does not qualify for it","source":"daily dataset","sites":3618,"sites_label":"3618","for_sale":true,"price":199,"page":"https://crawlcheck.io/datasets/problems/hsts-preload-ineligible"},{"code":"HSTS_PRELOAD_REMOVED","title":"This domain was removed from the browser HSTS preload list","source":"full scans","sites":10,"sites_label":"10","for_sale":false,"price":null,"page":"https://crawlcheck.io/datasets/problems/hsts-preload-removed"},{"code":"CONTENT_NEEDS_JAVASCRIPT","title":"The page’s content only exists after JavaScript runs","source":"full scans","sites":59,"sites_label":"59","for_sale":false,"price":null,"page":"https://crawlcheck.io/datasets/problems/content-needs-javascript"},{"code":"SITEMAP_NO_LASTMOD","title":"The sitemap does not say when anything changed","source":"daily dataset","sites":9473,"sites_label":"9473","for_sale":true,"price":199,"page":"https://crawlcheck.io/datasets/problems/sitemap-no-lastmod"},{"code":"BROKEN_INTERNAL_LINK","title":"Links on this site point at pages that do not answer","source":"daily dataset","sites":10992,"sites_label":"10992","for_sale":true,"price":299,"page":"https://crawlcheck.io/datasets/problems/broken-internal-link"},{"code":"SCRIPT_ONLY_LINKS","title":"Some pages can only be reached by a click handler, not a link","source":"daily dataset","sites":1318,"sites_label":"1318","for_sale":true,"price":199,"page":"https://crawlcheck.io/datasets/problems/script-only-links"},{"code":"ARD_SOURCE_BROKEN","title":"The site advertises an ARD entry source that does not answer","source":"daily dataset","sites":5,"sites_label":"5","for_sale":false,"price":null,"page":"https://crawlcheck.io/datasets/problems/ard-source-broken"},{"code":"ARD_MANIFEST_INVALID","title":"The ARD manifest breaks the specification","source":"daily dataset","sites":276,"sites_label":"276","for_sale":true,"price":99,"page":"https://crawlcheck.io/datasets/problems/ard-manifest-invalid"},{"code":"ARD_MANIFEST_WARNINGS","title":"The ARD manifest is valid but its entries will not be found by search","source":"daily dataset","sites":208,"sites_label":"208","for_sale":true,"price":99,"page":"https://crawlcheck.io/datasets/problems/ard-manifest-warnings"},{"code":"ARD_LEGACY_PATH","title":"The ARD manifest is served only under its old name","source":"daily dataset","sites":216,"sites_label":"216","for_sale":true,"price":99,"page":"https://crawlcheck.io/datasets/problems/ard-legacy-path"},{"code":"WEBMCP_SCHEMA_INVALID","title":"A WebMCP form tool is missing what Chrome needs to register it","source":"daily dataset","sites":0,"sites_label":"0","for_sale":false,"price":null,"page":"https://crawlcheck.io/datasets/problems/webmcp-schema-invalid"},{"code":"WEBMCP_SCHEMA_WARNINGS","title":"WebMCP form tools have fields an agent cannot identify","source":"daily dataset","sites":null,"sites_label":"fewer than 5","for_sale":false,"price":null,"page":"https://crawlcheck.io/datasets/problems/webmcp-schema-warnings"},{"code":"TRUST_PATH_LOOKALIKE","title":"The site sends agents' data to a domain that imitates its own","source":"daily dataset","sites":45,"sites_label":"45","for_sale":false,"price":null,"page":"https://crawlcheck.io/datasets/problems/trust-path-lookalike"},{"code":"TRUST_PATH_CLEARTEXT","title":"The site sends agents' data over plain http","source":"daily dataset","sites":62,"sites_label":"62","for_sale":false,"price":null,"page":"https://crawlcheck.io/datasets/problems/trust-path-cleartext"},{"code":"FORM_ONLY_CONTENT","title":"Some listings can only be reached by filling in a form","source":"daily dataset","sites":147,"sites_label":"147","for_sale":true,"price":99,"page":"https://crawlcheck.io/datasets/problems/form-only-content"},{"code":"FRONTIER_BLOAT","title":"Links send crawlers to the same pages under many different addresses","source":"daily dataset","sites":2723,"sites_label":"2723","for_sale":true,"price":199,"page":"https://crawlcheck.io/datasets/problems/frontier-bloat"},{"code":"SCANNER_REFUSED_BROWSER_READ","title":"The site refuses plain automated requests; it was measured through a real browser","source":"daily dataset","sites":298,"sites_label":"298","for_sale":true,"price":99,"page":"https://crawlcheck.io/datasets/problems/scanner-refused-browser-read"},{"code":"BROKEN_EMBED","title":"An embedded frame on the homepage cannot load","source":"daily dataset","sites":794,"sites_label":"794","for_sale":true,"price":99,"page":"https://crawlcheck.io/datasets/problems/broken-embed"},{"code":"IMAGES_BLOCKED_GOOGLEBOT_IMAGE","title":"robots.txt keeps Google's image crawler off the site's images","source":"daily dataset","sites":2371,"sites_label":"2371","for_sale":true,"price":199,"page":"https://crawlcheck.io/datasets/problems/images-blocked-googlebot-image"},{"code":"SHOPPING_BLOCKED_STOREBOT","title":"robots.txt keeps Google's Shopping crawler off the store","source":"daily dataset","sites":103,"sites_label":"103","for_sale":true,"price":99,"page":"https://crawlcheck.io/datasets/problems/shopping-blocked-storebot"},{"code":"IMAGE_FORMAT_MISMATCH","title":"Images are not the format their file names say","source":"daily dataset","sites":7284,"sites_label":"7284","for_sale":true,"price":199,"page":"https://crawlcheck.io/datasets/problems/image-format-mismatch"},{"code":"DATE_MOVED_NO_CHANGE","title":"The page's date changed but its words did not","source":"full scans","sites":7,"sites_label":"7","for_sale":false,"price":null,"page":"https://crawlcheck.io/datasets/problems/date-moved-no-change"},{"code":"CHANGED_DATE_STAYED","title":"The page's words changed but its date did not","source":"full scans","sites":8,"sites_label":"8","for_sale":false,"price":null,"page":"https://crawlcheck.io/datasets/problems/changed-date-stayed"},{"code":"PHONE_SHARED_BY_OTHER_BUSINESSES","title":"Directories list this phone number for other businesses","source":"full scans","sites":9,"sites_label":"9","for_sale":false,"price":null,"page":"https://crawlcheck.io/datasets/problems/phone-shared-by-other-businesses"},{"code":"PAGES_SNIPPET_RESTRICTED","title":"Pages meant to be found hide their own search snippet","source":"full scans","sites":null,"sites_label":"fewer than 5","for_sale":false,"price":null,"page":"https://crawlcheck.io/datasets/problems/pages-snippet-restricted"},{"code":"REDIRECT_CHAIN","title":"The homepage is reached through a chain of redirects","source":"daily dataset","sites":63573,"sites_label":"63573","for_sale":true,"price":299,"page":"https://crawlcheck.io/datasets/problems/redirect-chain"},{"code":"HOST_DUPLICATE_200","title":"www and the apex both serve the site instead of one redirecting","source":"daily dataset","sites":15237,"sites_label":"15237","for_sale":true,"price":299,"page":"https://crawlcheck.io/datasets/problems/host-duplicate-200"},{"code":"ROBOTS_RULES_SHADOWED","title":"robots.txt rules do not apply to named agents","source":"daily dataset","sites":16901,"sites_label":"16901","for_sale":true,"price":299,"page":"https://crawlcheck.io/datasets/problems/robots-rules-shadowed"},{"code":"INSTRUCTION_CONFLICT","title":"The instruction files contradict each other","source":"daily dataset","sites":1153,"sites_label":"1153","for_sale":true,"price":199,"page":"https://crawlcheck.io/datasets/problems/instruction-conflict"},{"code":"MACHINE_CHAIN_HEAVY","title":"The machine files cost more than the page","source":"full scans","sites":371,"sites_label":"371","for_sale":true,"price":99,"page":"https://crawlcheck.io/datasets/problems/machine-chain-heavy"},{"code":"MACHINE_FILE_ACCEPT_DEPENDENT","title":"A machine file is served to browsers but not to crawlers","source":"daily dataset","sites":0,"sites_label":"0","for_sale":false,"price":null,"page":"https://crawlcheck.io/datasets/problems/machine-file-accept-dependent"},{"code":"UNIFORM_REFUSAL","title":"This origin refused CrawlCheck, so the site was never measured","source":"daily dataset","sites":3577,"sites_label":"3577","for_sale":true,"price":199,"page":"https://crawlcheck.io/datasets/problems/uniform-refusal"},{"code":"ORIGIN_REFUSED_SCANNER","title":"The homepage and robots.txt both refused this client, so the site was never measured","source":"daily dataset","sites":0,"sites_label":"0","for_sale":false,"price":null,"page":"https://crawlcheck.io/datasets/problems/origin-refused-scanner"},{"code":"MACHINE_FILE_CACHE_SPLIT","title":"This file answers differently depending on whether the cache is hit","source":"daily dataset","sites":60,"sites_label":"60","for_sale":false,"price":null,"page":"https://crawlcheck.io/datasets/problems/machine-file-cache-split"},{"code":"MACHINE_FILE_CACHE_STALE","title":"The cache is still serving a file the origin no longer has","source":"daily dataset","sites":404,"sites_label":"404","for_sale":true,"price":99,"page":"https://crawlcheck.io/datasets/problems/machine-file-cache-stale"},{"code":"PLACEHOLDER_CONTENT","title":"Unfinished placeholder text is live on this page","source":"full scans","sites":14,"sites_label":"14","for_sale":false,"price":null,"page":"https://crawlcheck.io/datasets/problems/placeholder-content"},{"code":"CHALLENGE_SERVED_200","title":"A verification page is being served instead of the file","source":"daily dataset","sites":70,"sites_label":"70","for_sale":false,"price":null,"page":"https://crawlcheck.io/datasets/problems/challenge-served-200"},{"code":"ANSWER_ENGINE_REFUSED","title":"An answer engine's crawler was refused while a browser was served","source":"full scans","sites":78,"sites_label":"78","for_sale":false,"price":null,"page":"https://crawlcheck.io/datasets/problems/answer-engine-refused"},{"code":"PAGE_IS_MOSTLY_CODE","title":"Almost nothing a machine receives from this page is readable text","source":"full scans","sites":573,"sites_label":"573","for_sale":true,"price":99,"page":"https://crawlcheck.io/datasets/problems/page-is-mostly-code"},{"code":"STALE_CACHE_SERVED","title":"A cached copy of this page older than an hour was served","source":"full scans","sites":139,"sites_label":"139","for_sale":true,"price":99,"page":"https://crawlcheck.io/datasets/problems/stale-cache-served"},{"code":"CRAWLER_SERVED_LESS","title":"A named crawler was served materially less text than an ordinary client","source":"full scans","sites":40,"sites_label":"40","for_sale":false,"price":null,"page":"https://crawlcheck.io/datasets/problems/crawler-served-less"},{"code":"CRAWLER_REDIRECTED_AWAY","title":"A named crawler was sent somewhere the ordinary client was not","source":"full scans","sites":6,"sites_label":"6","for_sale":false,"price":null,"page":"https://crawlcheck.io/datasets/problems/crawler-redirected-away"},{"code":"CRAWLER_SERVED_EXTRA","title":"Crawlers are given scripts, links or instructions that visitors never see","source":"daily dataset","sites":0,"sites_label":"0","for_sale":false,"price":null,"page":"https://crawlcheck.io/datasets/problems/crawler-served-extra"},{"code":"CHALLENGE_PINNED_AT_EDGE","title":"The verification page is cached at the CDN edge","source":"daily dataset","sites":6,"sites_label":"6","for_sale":false,"price":null,"page":"https://crawlcheck.io/datasets/problems/challenge-pinned-at-edge"},{"code":"ROBOTS_IS_HTML","title":"robots.txt is HTML, not text","source":"daily dataset","sites":449,"sites_label":"449","for_sale":true,"price":99,"page":"https://crawlcheck.io/datasets/problems/robots-is-html"},{"code":"ROBOTS_IS_CATCHALL","title":"robots.txt answers with the catch-all page - there is no robots.txt","source":"daily dataset","sites":6118,"sites_label":"6118","for_sale":true,"price":199,"page":"https://crawlcheck.io/datasets/problems/robots-is-catchall"},{"code":"ROBOTS_LEGACY_AGENT_ONLY","title":"robots.txt rules name a retired crawler, not the one that visits","source":"daily dataset","sites":309,"sites_label":"309","for_sale":true,"price":99,"page":"https://crawlcheck.io/datasets/problems/robots-legacy-agent-only"},{"code":"ROBOTS_SITEMAP_RELATIVE","title":"robots.txt declares a sitemap with a relative path","source":"daily dataset","sites":1155,"sites_label":"1155","for_sale":true,"price":199,"page":"https://crawlcheck.io/datasets/problems/robots-sitemap-relative"},{"code":"ROBOTS_NOINDEX_LINE","title":"robots.txt uses a Noindex line no crawler obeys","source":"daily dataset","sites":118,"sites_label":"118","for_sale":true,"price":99,"page":"https://crawlcheck.io/datasets/problems/robots-noindex-line"},{"code":"ROBOTS_GROUP_NAME_TYPO","title":"robots.txt has a group for a misspelt crawler name","source":"daily dataset","sites":1796,"sites_label":"1796","for_sale":true,"price":199,"page":"https://crawlcheck.io/datasets/problems/robots-group-name-typo"},{"code":"DOMAIN_IS_ALIAS","title":"This domain is an alias: every path redirects to another site","source":"full scans","sites":null,"sites_label":"fewer than 5","for_sale":false,"price":null,"page":"https://crawlcheck.io/datasets/problems/domain-is-alias"},{"code":"ROBOTS_BLOCKED","title":"robots.txt is unreadable","source":"daily dataset","sites":628,"sites_label":"628","for_sale":true,"price":99,"page":"https://crawlcheck.io/datasets/problems/robots-blocked"},{"code":"ROBOTS_DISALLOW_ALL","title":"robots.txt blocks every crawler it does not name","source":"daily dataset","sites":3055,"sites_label":"3055","for_sale":true,"price":199,"page":"https://crawlcheck.io/datasets/problems/robots-disallow-all"},{"code":"SITEMAP_IS_HTML","title":"The sitemap returns HTML, not XML","source":"daily dataset","sites":363,"sites_label":"363","for_sale":true,"price":99,"page":"https://crawlcheck.io/datasets/problems/sitemap-is-html"},{"code":"SITEMAP_BLOCKED","title":"The sitemap is refused","source":"daily dataset","sites":4670,"sites_label":"4670","for_sale":true,"price":199,"page":"https://crawlcheck.io/datasets/problems/sitemap-blocked"},{"code":"SITEMAP_ORIGIN_ERROR","title":"The origin returned a server error on the sitemap path","source":"daily dataset","sites":605,"sites_label":"605","for_sale":true,"price":99,"page":"https://crawlcheck.io/datasets/problems/sitemap-origin-error"},{"code":"HOMEPAGE_REFUSED","title":"The homepage refused this client, so every homepage section is unmeasured","source":"daily dataset","sites":450,"sites_label":"450","for_sale":true,"price":99,"page":"https://crawlcheck.io/datasets/problems/homepage-refused"},{"code":"HOMEPAGE_IS_INTERSTITIAL","title":"The homepage answered with an interstitial, not a page","source":"daily dataset","sites":146,"sites_label":"146","for_sale":true,"price":99,"page":"https://crawlcheck.io/datasets/problems/homepage-is-interstitial"},{"code":"HOMEPAGE_REDIRECTS_OFF_HOST","title":"The homepage sends crawlers to a different hostname","source":"full scans","sites":46,"sites_label":"46","for_sale":false,"price":null,"page":"https://crawlcheck.io/datasets/problems/homepage-redirects-off-host"},{"code":"SITEMAP_UNREADABLE","title":"Whether a sitemap exists could not be determined — every path was refused","source":"daily dataset","sites":1735,"sites_label":"1735","for_sale":true,"price":199,"page":"https://crawlcheck.io/datasets/problems/sitemap-unreadable"},{"code":"DECLARED_SITEMAP_REFUSED","title":"The sitemap robots.txt declares refused this client","source":"daily dataset","sites":169,"sites_label":"169","for_sale":true,"price":99,"page":"https://crawlcheck.io/datasets/problems/declared-sitemap-refused"},{"code":"SITEMAP_DECLARES_NOTHING","title":"A sitemap answers 200 and lists nothing","source":"full scans","sites":25,"sites_label":"25","for_sale":false,"price":null,"page":"https://crawlcheck.io/datasets/problems/sitemap-declares-nothing"},{"code":"ORIGIN_BLOCKED_BEHIND_CACHE","title":"A security layer refused this file — source unconfirmed","source":"daily dataset","sites":339,"sites_label":"339","for_sale":true,"price":99,"page":"https://crawlcheck.io/datasets/problems/origin-blocked-behind-cache"},{"code":"DECLARED_SITEMAP_BROKEN","title":"robots.txt points at a sitemap that fails","source":"daily dataset","sites":1541,"sites_label":"1541","for_sale":true,"price":199,"page":"https://crawlcheck.io/datasets/problems/declared-sitemap-broken"},{"code":"SITEMAP_LASTMOD_INVALID","title":"A sitemap lastmod is not a valid date","source":"daily dataset","sites":336,"sites_label":"336","for_sale":true,"price":99,"page":"https://crawlcheck.io/datasets/problems/sitemap-lastmod-invalid"},{"code":"SITEMAP_URL_OFF_HOST","title":"The sitemap lists URLs on another host","source":"daily dataset","sites":5872,"sites_label":"5872","for_sale":true,"price":199,"page":"https://crawlcheck.io/datasets/problems/sitemap-url-off-host"},{"code":"SITEMAP_CHILD_BROKEN","title":"A child sitemap named in the sitemap index does not load","source":"daily dataset","sites":767,"sites_label":"767","for_sale":true,"price":99,"page":"https://crawlcheck.io/datasets/problems/sitemap-child-broken"},{"code":"NO_SITEMAP_FOUND","title":"No XML sitemap found","source":"daily dataset","sites":34245,"sites_label":"34245","for_sale":true,"price":299,"page":"https://crawlcheck.io/datasets/problems/no-sitemap-found"},{"code":"ROBOTS_NOT_200","title":"robots.txt does not return 200","source":"daily dataset","sites":14296,"sites_label":"14296","for_sale":true,"price":299,"page":"https://crawlcheck.io/datasets/problems/robots-not-200"},{"code":"ROBOTS_NO_SITEMAP","title":"robots.txt declares no sitemap","source":"daily dataset","sites":37607,"sites_label":"37607","for_sale":true,"price":299,"page":"https://crawlcheck.io/datasets/problems/robots-no-sitemap"},{"code":"STAGING_LEAK","title":"A staging hostname is exposed in the page source","source":"daily dataset","sites":461,"sites_label":"461","for_sale":true,"price":99,"page":"https://crawlcheck.io/datasets/problems/staging-leak"},{"code":"ROBOTS_CTYPE","title":"robots.txt has an unexpected content type","source":"daily dataset","sites":459,"sites_label":"459","for_sale":true,"price":99,"page":"https://crawlcheck.io/datasets/problems/robots-ctype"},{"code":"ENTITY_COLLISION","title":"Two different entities share the same identifying data","source":"full scans","sites":35,"sites_label":"35","for_sale":false,"price":null,"page":"https://crawlcheck.io/datasets/problems/entity-collision"},{"code":"ENTITY_NO_COORDINATES","title":"An entity is missing geographic coordinates","source":"full scans","sites":119,"sites_label":"119","for_sale":true,"price":99,"page":"https://crawlcheck.io/datasets/problems/entity-no-coordinates"},{"code":"GEO_COUNTRY_MISMATCH","title":"Published coordinates are not in the country the address names","source":"full scans","sites":null,"sites_label":"fewer than 5","for_sale":false,"price":null,"page":"https://crawlcheck.io/datasets/problems/geo-country-mismatch"},{"code":"LLMS_LINK_BROKEN","title":"llms.txt links to pages that do not load","source":"daily dataset","sites":9250,"sites_label":"9250","for_sale":true,"price":199,"page":"https://crawlcheck.io/datasets/problems/llms-link-broken"},{"code":"NO_LLMS_TXT","title":"No llms.txt","source":"daily dataset","sites":106691,"sites_label":"106691","for_sale":true,"price":299,"page":"https://crawlcheck.io/datasets/problems/no-llms-txt"}],"signals":[{"code":"AI_OPTOUT_SET","title":"An AI opt-out signal is set","sites":98,"sites_label":"98","for_sale":false,"price":null,"why":"the site's own choice to refuse AI answers or AI search in robots.txt Content-Signal; a decision, not a defect","page":"https://crawlcheck.io/datasets/problems/ai-optout-set"}],"cohort_rule":"a list of 1 to 4 sites shows \"fewer than 5\" and is not sold: that few would identify the sites","full_scan_only":[{"code":"AD_DENSITY_HIGH","title":"Ads take too much of the page on a phone","sites":null,"for_sale":false,"why":"found only by a full scan of a site; no full scan in the last 7 days has found it yet, so there is no list"},{"code":"MOBILE_INTERSTITIAL","title":"A pop-up covers the page on a phone when it loads","sites":null,"for_sale":false,"why":"found only by a full scan of a site; no full scan in the last 7 days has found it yet, so there is no list"}]}