{"ok":true,"kind":"crawlcheck-datasets","v":1,"updated":"2026-10-07T19:18:02.479Z","timezone":"America/Denver","totals":{"rows":267516,"measured":179599},"lanes":[{"id":"saas","label":"SaaS and software, worldwide","rows":34006,"measured":25047,"days":5},{"id":"gov","label":"Government and education","rows":18361,"measured":12554,"days":5},{"id":"local","label":"US local businesses, one state per day","rows":34058,"measured":25040,"days":5},{"id":"filler","label":"Everything else the corpus reached (publishers, shops, directories, docs)","rows":181091,"measured":116958,"days":5}],"days":[{"day":"2026-10-07","lanes":{"saas":{"rows":6775,"measured":4996,"bytes":5168637},"gov":{"rows":3550,"measured":2501,"bytes":2493741},"local":{"rows":6560,"measured":4988,"bytes":5961433},"filler":{"rows":6085,"measured":4510,"bytes":4411171}}},{"day":"2026-10-06","lanes":{"saas":{"rows":7085,"measured":5001,"bytes":5207380},"gov":{"rows":3560,"measured":2505,"bytes":2515410},"local":{"rows":7105,"measured":5011,"bytes":6295227},"filler":{"rows":50674,"measured":32135,"bytes":32847701}}},{"day":"2026-10-05","lanes":{"saas":{"rows":7135,"measured":5004,"bytes":5213851},"gov":{"rows":3685,"measured":2512,"bytes":2507613},"local":{"rows":6940,"measured":4995,"bytes":6250619},"filler":{"rows":51840,"measured":33564,"bytes":34125498}}},{"day":"2026-10-04","lanes":{"saas":{"rows":7130,"measured":5036,"bytes":5256054},"gov":{"rows":3610,"measured":2526,"bytes":2531418},"local":{"rows":6770,"measured":5035,"bytes":6284766},"filler":{"rows":54964,"measured":34556,"bytes":35440688}}},{"day":"2026-10-03","lanes":{"saas":{"rows":5881,"measured":5010,"bytes":5179183},"gov":{"rows":3956,"measured":2510,"bytes":2459128},"local":{"rows":6683,"measured":5011,"bytes":5841927},"filler":{"rows":17528,"measured":12193,"bytes":12079015}}},{"day":"2026-10-02","lanes":{}}],"latest":{"day":"2026-10-07","lanes":{"saas":{"rows":6775,"measured":4996,"kind":{"other":2666,"saas":1437,"docs":20,"local-service":118,"publisher":511,"food":12,"ecommerce":153,"directory":1,"nonprofit":9,"health":7,"gov-edu":8,"travel":21,"finance":22,"legal":4,"real-estate":6,"automotive":1},"state":{},"platform":{"unknown":3121,"wordpress":729,"webflow":198,"wix":18,"nextjs":783,"squarespace":39,"shopify":87,"drupal":21},"grade":{"C":1266,"A":1157,"D":1669,"F":387,"B":319},"llms_txt":958,"ai_blocking":585,"top_tech":[["Google Fonts",1621],["jQuery",1589],["Google Analytics",1360],["Google Tag Manager",1212],["Font Awesome",957],["Next.js",790],["Bootstrap",694],["Yoast SEO",401],["Elementor",303],["reCAPTCHA",264],["Meta Pixel",250],["WooCommerce",229],["Google Ads",226],["YouTube embed",213],["Webflow",211],["React",190],["Rank Math",182],["HubSpot",171],["Microsoft Clarity",166],["Vue",127],["Adobe Fonts",111],["LinkedIn Insight",106],["Vimeo",101],["Google Maps",100],["Nuxt",97]]},"gov":{"rows":3550,"measured":2501,"kind":{"gov-edu":2501},"state":{},"platform":{"drupal":241,"wordpress":661,"unknown":1496,"nextjs":36,"wix":37,"webflow":10,"squarespace":20},"grade":{"C":907,"D":899,"A":288,"B":157,"F":143},"llms_txt":126,"ai_blocking":426,"top_tech":[["jQuery",1635],["Google Fonts",1108],["Font Awesome",1048],["Google Analytics",982],["Google Tag Manager",794],["Bootstrap",572],["Yoast SEO",262],["Elementor",258],["YouTube embed",213],["Adobe Fonts",181],["reCAPTCHA",174],["WooCommerce",162],["Google Maps",114],["Meta Pixel",68],["Vimeo",55],["Divi",52],["New Relic",49],["React",47],["Microsoft Clarity",45],["Vue",45],["Rank Math",42],["Wix",38],["Next.js",37],["Google Ads",30],["GoDaddy Builder",28]]},"local":{"rows":6560,"measured":4988,"kind":{"food":64,"local-service":2226,"publisher":640,"saas":74,"ecommerce":163,"health":1017,"gov-edu":31,"travel":12,"nonprofit":71,"automotive":254,"finance":170,"real-estate":92,"legal":174},"state":{"new-york":4988},"platform":{"wordpress":1871,"unknown":2016,"squarespace":459,"wix":317,"shopify":49,"webflow":82,"nextjs":115,"drupal":79},"grade":{"C":1660,"D":978,"A":1404,"F":138,"B":469},"llms_txt":920,"ai_blocking":657,"top_tech":[["jQuery",2716],["Google Fonts",1921],["Google Analytics",1904],["Font Awesome",1571],["Google Tag Manager",1276],["Yoast SEO",1009],["Elementor",757],["reCAPTCHA",654],["Adobe Fonts",580],["Bootstrap",579],["WooCommerce",578],["Google Maps",541],["Squarespace",459],["Meta Pixel",391],["YouTube embed",337],["Wix",322],["Google Ads",283],["Rank Math",232],["Divi",195],["WP Rocket",192],["GoDaddy Builder",167],["Vimeo",150],["React",124],["Next.js",118],["Sentry",111]]},"filler":{"rows":6085,"measured":4510,"kind":{"other":2773,"publisher":718,"saas":488,"finance":9,"ecommerce":117,"gov-edu":173,"local-service":88,"legal":3,"nonprofit":24,"automotive":5,"health":10,"food":9,"travel":79,"real-estate":6,"docs":8},"state":{},"platform":{"unknown":2980,"wordpress":869,"nextjs":329,"wix":44,"drupal":139,"shopify":48,"webflow":73,"squarespace":28},"grade":{"C":1280,"F":374,"D":1592,"B":409,"A":713},"llms_txt":558,"ai_blocking":743,"top_tech":[["jQuery",1992],["Google Tag Manager",1486],["Google Fonts",1356],["Google Analytics",1269],["Font Awesome",1064],["Bootstrap",613],["Yoast SEO",490],["Next.js",333],["reCAPTCHA",316],["YouTube embed",295],["Elementor",276],["Adobe Fonts",229],["WooCommerce",220],["Meta Pixel",184],["OneTrust",175],["React",150],["Google Ads",148],["Matomo",129],["Vue",127],["Microsoft Clarity",107],["Nuxt",98],["Rank Math",94],["Google Maps",93],["New Relic",86],["Webflow",84]]}}},"fields":[{"name":"domain","meaning":"Registrable domain scanned"},{"name":"at","meaning":"When the homepage was read (UTC, ISO 8601)"},{"name":"lane","meaning":"Lane the site was scanned in: saas, gov, local, filler"},{"name":"kind","meaning":"Site kind: the page's own declaration (JSON-LD), else its patterns; for local rows with no page signal, the OpenStreetMap category"},{"name":"kind_detected","meaning":"Kind read from the page alone"},{"name":"state","meaning":"US state (local lane)"},{"name":"city","meaning":"City from OpenStreetMap (local lane)"},{"name":"name","meaning":"Business name from OpenStreetMap (local lane)"},{"name":"osm_category","meaning":"OpenStreetMap category, e.g. trades:plumber (local lane)"},{"name":"src","meaning":"Where the domain came from: dotgov:<type>, tranco:<tld>, wikidata, discovered, OSM category"},{"name":"title","meaning":"Homepage <title>"},{"name":"description","meaning":"Homepage meta description"},{"name":"language","meaning":"en or non-en, from the page"},{"name":"grade","meaning":"CrawlCheck grade, A to F"},{"name":"score","meaning":"CrawlCheck overall score 0-100, before any cap"},{"name":"grade_cap_reason","meaning":"When a finding caps the letter below what the score would give, which finding and why"},{"name":"crawl_clarity","meaning":"Crawl clarity score 0-100"},{"name":"user_clarity","meaning":"User clarity score 0-100"},{"name":"findings","meaning":"Number of findings"},{"name":"finding_codes","meaning":"Finding codes, ;-separated"},{"name":"platform","meaning":"CMS band: wordpress, shopify, webflow, squarespace, wix, nextjs, drupal, unknown"},{"name":"cms","meaning":"CMS and builders detected"},{"name":"tech","meaning":"Technologies detected in the homepage: analytics, tag managers, chat, payments, booking, consent, frameworks"},{"name":"edge","meaning":"Edge or CDN detected from headers"},{"name":"server","meaning":"Server header"},{"name":"host","meaning":"First hosting signal"},{"name":"rendering","meaning":"server or client (under 1% of the payload is content)"},{"name":"content_ratio","meaning":"Share of the homepage payload that is visible content, %"},{"name":"blocking_resources","meaning":"Render-blocking resources in <head>"},{"name":"id_ratio","meaning":"Share of JSON-LD nodes with an @id"},{"name":"ld_types","meaning":"schema.org types declared in JSON-LD"},{"name":"llms_txt","meaning":"Serves /llms.txt"},{"name":"surfaces","meaning":"Agent surfaces served: agents.md, agent skills, MCP card, API catalog, media kit"},{"name":"robots_measured","meaning":"robots.txt was read"},{"name":"ai_blocked","meaning":"AI crawlers robots.txt closes / to"},{"name":"https","meaning":"Homepage on HTTPS"},{"name":"hsts","meaning":"HSTS header present"},{"name":"sitemap_urls","meaning":"URLs declared in sitemaps"},{"name":"size","meaning":"small (<50), medium (<2000), large"},{"name":"home_bytes","meaning":"Homepage bytes"},{"name":"ttfb_ms","meaning":"Time to first byte from our observer, ms"},{"name":"load_ms","meaning":"Time to the full homepage from our observer, ms"},{"name":"unmeasured","meaning":"true when the homepage could not be measured"},{"name":"reason","meaning":"Why it could not be measured"}],"tech_detected":["Google Analytics","Google Tag Manager","Google Ads","Meta Pixel","LinkedIn Insight","TikTok Pixel","Microsoft Clarity","Bing UET","Hotjar","Segment","Mixpanel","Amplitude","Heap","Plausible","Matomo","HubSpot","Salesforce Pardot","Marketo","Klaviyo","Mailchimp","Intercom","Drift","Zendesk","LiveChat","Tawk.to","Crisp","Podium","Birdeye","CallRail","Calendly","ServiceTitan","Housecall Pro","Jobber","Stripe","PayPal","Square","Shopify","WooCommerce","BigCommerce","Yotpo","Judge.me","Trustpilot","OneTrust","Cookiebot","Termly","reCAPTCHA","hCaptcha","Cloudflare Turnstile","Cloudflare Web Analytics","Sentry","New Relic","Datadog RUM","Optimizely","VWO","Algolia","jQuery","React","Next.js","Nuxt","Vue","Angular","Svelte","Gatsby","Bootstrap","Tailwind CSS","Elementor","Divi","Yoast SEO","Rank Math","WP Rocket","Webflow","Framer","Ghost","Wix","Squarespace","Duda","GoDaddy Builder","Google Fonts","Adobe Fonts","Font Awesome","YouTube embed","Vimeo","Google Maps","Typeform","Jotform","POWR"],"sample":{"csv":"https://crawlcheck.io/api/datasets/sample.csv","ndjson":"https://crawlcheck.io/api/datasets/sample.ndjson","rows":25},"files":{"url":"https://crawlcheck.io/api/datasets/file?day=YYYY-MM-DD&lane=saas|gov|local|filler&format=csv|ndjson","filters":["kind","state","tech","platform","measured=1"],"access":"a CrawlCheck licence key with dataset access (header x-crawlcheck-key)"},"access":{"page":"https://crawlcheck.io/datasets","request":"POST https://crawlcheck.io/api/datasets/request {email, company, lanes, note}"}}