{"ok":true,"kind":"crawlcheck-datasets","v":1,"catalogue_release":"ds-20261122-0120-e3b71a24","generated_at":"2026-11-22T01:33:58.175Z","updated":"2026-11-22T01:20:08.248Z","distinct_updated":"2026-11-22T00:52:51.680Z","timezone":"America/Denver","segments":[{"id":"stores","name":"Online stores","for":"Agencies, app makers and payment companies that sell to shops","what":"Sites that sell online: the page declares a store, or it runs Shopify, WooCommerce or BigCommerce.","sites":18152,"lanes":{"saas":1456,"gov":812,"local":11510,"filler":4374},"platform":[["wordpress",11085],["unknown",3740],["shopify",1686],["wix",1407],["nextjs",155]],"with_llms_txt":4582,"blocking_ai_crawlers":1088,"unit":"distinct sites, measured only","freshness":{"basis":"each site's newest measured read","newest_age_days":0.1,"median_age_days":3.8,"oldest_age_days":6.7,"within_7d_pct":100},"export_mode":"latest-per-domain","download":"https://crawlcheck.io/api/datasets/segment?id=stores&format=csv"},{"id":"publishers","name":"Publishers and news sites","for":"AI companies licensing content, ad tech and PR teams","what":"Sites that publish articles, each with how it treats AI crawlers.","sites":13193,"lanes":{"saas":2095,"local":6642,"filler":4456},"platform":[["wordpress",9259],["unknown",3200],["drupal",361],["nextjs",295],["squarespace",37]],"with_llms_txt":1199,"blocking_ai_crawlers":1458,"unit":"distinct sites, measured only","freshness":{"basis":"each site's newest measured read","newest_age_days":0.1,"median_age_days":3.6,"oldest_age_days":6.7,"within_7d_pct":100},"export_mode":"latest-per-domain","download":"https://crawlcheck.io/api/datasets/segment?id=publishers&format=csv"},{"id":"ai-blocking","name":"Sites that block AI crawlers","for":"AI companies, content-licensing teams and researchers","what":"Sites whose robots.txt shuts out at least one named AI agent, such as GPTBot, ClaudeBot or CCBot.","sites":9668,"lanes":{"saas":2182,"gov":928,"local":3607,"filler":2951},"platform":[["unknown",6494],["wordpress",1987],["nextjs",589],["drupal",252],["webflow",125]],"with_llms_txt":1413,"blocking_ai_crawlers":9668,"unit":"distinct sites, measured only","freshness":{"basis":"each site's newest measured read","newest_age_days":0.1,"median_age_days":3.7,"oldest_age_days":6.7,"within_7d_pct":100},"export_mode":"latest-per-domain","download":"https://crawlcheck.io/api/datasets/segment?id=ai-blocking&format=csv"},{"id":"llms-txt","name":"Sites with an llms.txt","for":"AI search and tool makers, and researchers","what":"Sites that publish an llms.txt file for AI agents.","sites":25103,"lanes":{"saas":5530,"gov":629,"local":15739,"filler":3205},"platform":[["unknown",9135],["wix",6115],["wordpress",5348],["nextjs",2345],["shopify",1432]],"with_llms_txt":25103,"blocking_ai_crawlers":1413,"unit":"distinct sites, measured only","freshness":{"basis":"each site's newest measured read","newest_age_days":0.1,"median_age_days":3.7,"oldest_age_days":6.7,"within_7d_pct":100},"export_mode":"latest-per-domain","download":"https://crawlcheck.io/api/datasets/segment?id=llms-txt&format=csv"},{"id":"home-services","name":"Trades and home services","for":"Agencies and software companies that sell to contractors","what":"Plumbers, electricians, roofers, HVAC, landscapers and other trades: tagged as a trade in OpenStreetMap, saying so on their own homepage, or running field-service software (ServiceTitan, Housecall Pro, Jobber).","sites":2674,"lanes":{"local":2553,"filler":116,"saas":5},"platform":[["wordpress",1325],["unknown",961],["wix",146],["squarespace",104],["nextjs",90]],"with_llms_txt":840,"blocking_ai_crawlers":141,"unit":"distinct sites, measured only","freshness":{"basis":"each site's newest measured read","newest_age_days":0.1,"median_age_days":4.7,"oldest_age_days":6.7,"within_7d_pct":100},"export_mode":"latest-per-domain","download":"https://crawlcheck.io/api/datasets/segment?id=home-services&format=csv"},{"id":"email-spoofable","name":"Domains open to email spoofing","for":"Email security vendors, MSPs and IT providers","what":"Domains that receive email but publish no DMARC policy, or one set to none, so anyone can send mail in their name.","sites":69079,"lanes":{"saas":10999,"gov":4592,"local":43610,"filler":9878},"platform":[["unknown",31230],["wordpress",21949],["squarespace",5192],["wix",4517],["nextjs",3368]],"with_llms_txt":14754,"blocking_ai_crawlers":3911,"unit":"distinct sites, measured only","freshness":{"basis":"each site's newest measured read","newest_age_days":0.1,"median_age_days":3.6,"oldest_age_days":6.7,"within_7d_pct":100},"export_mode":"latest-per-domain","download":"https://crawlcheck.io/api/datasets/segment?id=email-spoofable&format=csv"},{"id":"security-gaps","name":"Sites missing basic security","for":"Web security vendors, MSPs and agencies","what":"Sites sending two or fewer of the six standard security headers, exposing a server version, running jQuery older than 3.5, or loading insecure content on HTTPS.","sites":96940,"lanes":{"saas":18440,"gov":7898,"local":51056,"filler":19546},"platform":[["unknown",49059],["wordpress",30071],["wix",6537],["nextjs",4781],["squarespace",3789]],"with_llms_txt":16484,"blocking_ai_crawlers":6754,"unit":"distinct sites, measured only","freshness":{"basis":"each site's newest measured read","newest_age_days":0.1,"median_age_days":3.6,"oldest_age_days":6.7,"within_7d_pct":100},"export_mode":"latest-per-domain","download":"https://crawlcheck.io/api/datasets/segment?id=security-gaps&format=csv"},{"id":"expiring-soon","name":"Domains expiring in the next 45 days","for":"Registrars, domain investors, brand-protection and IT providers","what":"Live sites whose domain registration runs out within 45 days, with registrar, DNS provider and the rest of the site's record.","sites":6281,"lanes":{"saas":1068,"gov":208,"local":3554,"filler":1451},"platform":[["unknown",3413],["wordpress",1624],["squarespace",396],["wix",317],["nextjs",284]],"with_llms_txt":1115,"blocking_ai_crawlers":613,"unit":"distinct sites, measured only","freshness":{"basis":"each site's newest measured read","newest_age_days":0.1,"median_age_days":3.5,"oldest_age_days":6.7,"within_7d_pct":100},"export_mode":"latest-per-domain","download":"https://crawlcheck.io/api/datasets/segment?id=expiring-soon&format=csv"},{"id":"new-domains","name":"Domains registered in the last 6 months","for":"Agencies, SaaS sales teams and fraud and brand-protection analysts","what":"Live sites on a domain registered within 180 days: new businesses, rebrands and look-alikes, each with what the site already runs.","sites":2433,"lanes":{"saas":879,"gov":205,"local":575,"filler":774},"platform":[["unknown",1759],["wordpress",342],["nextjs",240],["wix",32],["squarespace",31]],"with_llms_txt":415,"blocking_ai_crawlers":499,"unit":"distinct sites, measured only","freshness":{"basis":"each site's newest measured read","newest_age_days":0.2,"median_age_days":3.5,"oldest_age_days":6.7,"within_7d_pct":100},"export_mode":"latest-per-domain","download":"https://crawlcheck.io/api/datasets/segment?id=new-domains&format=csv"},{"id":"martech","name":"Sites running a marketing or support platform","for":"Competing software vendors and agencies","what":"Sites running HubSpot, Pardot, Marketo, Klaviyo, Mailchimp, Intercom, Drift or Zendesk.","sites":6293,"lanes":{"saas":1863,"gov":252,"local":2888,"filler":1290},"platform":[["wordpress",2278],["unknown",2238],["shopify",623],["webflow",414],["nextjs",296]],"with_llms_txt":1718,"blocking_ai_crawlers":430,"unit":"distinct sites, measured only","freshness":{"basis":"each site's newest measured read","newest_age_days":0.1,"median_age_days":3.9,"oldest_age_days":6.7,"within_7d_pct":100},"export_mode":"latest-per-domain","download":"https://crawlcheck.io/api/datasets/segment?id=martech&format=csv"}],"segments_raw":{"updated":"2026-11-22T00:40:56.733Z","days":["2026-11-21","2026-11-20","2026-11-19","2026-11-18","2026-11-17","2026-11-16","2026-11-15"],"files":28,"missing":0,"segments":{"stores":{"sites":18152,"lanes":{"saas":1456,"gov":812,"local":11510,"filler":4374},"platform":{"wordpress":11085,"unknown":3740,"wix":1407,"nextjs":155,"shopify":1686,"webflow":25,"drupal":17,"squarespace":37},"llms_txt":4582,"ai_blocking":1088,"freshness":{"basis":"each site's newest measured read","newest_age_days":0.1,"median_age_days":3.8,"oldest_age_days":6.7,"within_7d_pct":100}},"publishers":{"sites":13193,"lanes":{"saas":2095,"local":6642,"filler":4456},"platform":{"wordpress":9259,"unknown":3200,"nextjs":295,"drupal":361,"squarespace":37,"webflow":33,"wix":4,"shopify":4},"llms_txt":1199,"ai_blocking":1458,"freshness":{"basis":"each site's newest measured read","newest_age_days":0.1,"median_age_days":3.6,"oldest_age_days":6.7,"within_7d_pct":100}},"ai-blocking":{"sites":9668,"lanes":{"saas":2182,"gov":928,"local":3607,"filler":2951},"platform":{"nextjs":589,"wordpress":1987,"unknown":6494,"webflow":125,"drupal":252,"shopify":69,"squarespace":114,"wix":38},"llms_txt":1413,"ai_blocking":9668,"freshness":{"basis":"each site's newest measured read","newest_age_days":0.1,"median_age_days":3.7,"oldest_age_days":6.7,"within_7d_pct":100}},"llms-txt":{"sites":25103,"lanes":{"saas":5530,"gov":629,"local":15739,"filler":3205},"platform":{"wordpress":5348,"unknown":9135,"nextjs":2345,"webflow":489,"wix":6115,"shopify":1432,"drupal":112,"squarespace":127},"llms_txt":25103,"ai_blocking":1413,"freshness":{"basis":"each site's newest measured read","newest_age_days":0.1,"median_age_days":3.7,"oldest_age_days":6.7,"within_7d_pct":100}},"home-services":{"sites":2674,"lanes":{"local":2553,"filler":116,"saas":5},"platform":{"wordpress":1325,"wix":146,"unknown":961,"nextjs":90,"webflow":43,"squarespace":104,"drupal":4,"shopify":1},"llms_txt":840,"ai_blocking":141,"freshness":{"basis":"each site's newest measured read","newest_age_days":0.1,"median_age_days":4.7,"oldest_age_days":6.7,"within_7d_pct":100}},"email-spoofable":{"sites":69079,"lanes":{"saas":10999,"gov":4592,"local":43610,"filler":9878},"platform":{"wordpress":21949,"nextjs":3368,"squarespace":5192,"unknown":31230,"webflow":1100,"wix":4517,"drupal":705,"shopify":1018},"llms_txt":14754,"ai_blocking":3911,"freshness":{"basis":"each site's newest measured read","newest_age_days":0.1,"median_age_days":3.6,"oldest_age_days":6.7,"within_7d_pct":100}},"security-gaps":{"sites":96940,"lanes":{"saas":18440,"gov":7898,"local":51056,"filler":19546},"platform":{"wordpress":30071,"unknown":49059,"nextjs":4781,"squarespace":3789,"webflow":1368,"wix":6537,"drupal":1087,"shopify":248},"llms_txt":16484,"ai_blocking":6754,"freshness":{"basis":"each site's newest measured read","newest_age_days":0.1,"median_age_days":3.6,"oldest_age_days":6.7,"within_7d_pct":100}},"expiring-soon":{"sites":6281,"lanes":{"saas":1068,"gov":208,"local":3554,"filler":1451},"platform":{"nextjs":284,"unknown":3413,"wix":317,"wordpress":1624,"drupal":54,"squarespace":396,"shopify":96,"webflow":97},"llms_txt":1115,"ai_blocking":613,"freshness":{"basis":"each site's newest measured read","newest_age_days":0.1,"median_age_days":3.5,"oldest_age_days":6.7,"within_7d_pct":100}},"new-domains":{"sites":2433,"lanes":{"saas":879,"gov":205,"local":575,"filler":774},"platform":{"unknown":1759,"wordpress":342,"nextjs":240,"squarespace":31,"shopify":21,"wix":32,"webflow":5,"drupal":3},"llms_txt":415,"ai_blocking":499,"freshness":{"basis":"each site's newest measured read","newest_age_days":0.2,"median_age_days":3.5,"oldest_age_days":6.7,"within_7d_pct":100}},"martech":{"sites":6293,"lanes":{"saas":1863,"gov":252,"local":2888,"filler":1290},"platform":{"wordpress":2278,"nextjs":296,"webflow":414,"unknown":2238,"drupal":127,"shopify":623,"squarespace":266,"wix":51},"llms_txt":1718,"ai_blocking":430,"freshness":{"basis":"each site's newest measured read","newest_age_days":0.1,"median_age_days":3.9,"oldest_age_days":6.7,"within_7d_pct":100}}}},"totals":{"rows":1021639,"measured":724183,"unmeasured":297456,"unique_domains":782981,"latest_measured":557520,"freshness":{"basis":"each site's newest measured read, in days before this count","newest_age_days":0,"median_age_days":23.6,"oldest_age_days":49.3,"within_7d_pct":24,"within_30d_pct":60.7},"distinct_complete":true,"units":{"rows":"reads, one per site per day it was read (measured or not)","measured":"reads that got a page back","unmeasured":"reads that got no answer; never counted in a headline figure","unique_domains":"distinct sites across all days and lanes","latest_measured":"distinct sites whose newest read was measured"}},"lanes":[{"id":"saas","label":"SaaS and software, worldwide","rows":167688,"measured":119583,"days":50,"unique_domains":104042,"latest_measured":74454,"freshness":{"basis":"each site's newest measured read, in days before this count","newest_age_days":0.5,"median_age_days":11.5,"oldest_age_days":49.3,"within_7d_pct":35.6,"within_30d_pct":83.1}},{"id":"gov","label":"Government and education","rows":103923,"measured":67927,"days":48,"unique_domains":63419,"latest_measured":41875,"freshness":{"basis":"each site's newest measured read, in days before this count","newest_age_days":0.3,"median_age_days":10.7,"oldest_age_days":49.3,"within_7d_pct":29.2,"within_30d_pct":81.8}},{"id":"local","label":"US local businesses, one state per day","rows":316059,"measured":221657,"days":48,"unique_domains":260991,"latest_measured":181080,"freshness":{"basis":"each site's newest measured read, in days before this count","newest_age_days":0.3,"median_age_days":18.7,"oldest_age_days":49.3,"within_7d_pct":36.7,"within_30d_pct":71.6}},{"id":"filler","label":"Everything else the corpus reached (publishers, shops, directories, docs)","rows":433969,"measured":315016,"days":48,"unique_domains":371261,"latest_measured":271671,"freshness":{"basis":"each site's newest measured read, in days before this count","newest_age_days":0,"median_age_days":36.5,"oldest_age_days":49.3,"within_7d_pct":10.3,"within_30d_pct":41.1}}],"exports":[{"endpoint":"/api/datasets/file","mode":"day","rows":"every read in that lane on that day (Mountain Time), measured or not; measured=1 keeps the measured ones; a site appears once","filename":"crawlcheck-<lane>-<day>-day.csv"},{"endpoint":"/api/datasets/state","mode":"history","rows":"every read of every business in the state, all days; a site appears once per day it was read","filename":"crawlcheck-local-<state>-history.csv"},{"endpoint":"/api/datasets/segment","mode":"latest-per-domain","rows":"each site once: its newest measured read in the newest 7 days","filename":"crawlcheck-<list>-<newest day>-latest.csv"},{"endpoint":"/api/datasets/tech","mode":"latest-per-domain","rows":"each site once: its newest measured read in the newest 7 days","filename":"crawlcheck-tech-<technology>-<newest day>-latest.csv"},{"endpoint":"/api/datasets/problem","mode":"latest-per-domain","rows":"each site once: its newest measured read in the newest 7 days","filename":"crawlcheck-problem-<code>-<newest day>-latest.csv"}],"removals":{"url":"https://crawlcheck.io/api/datasets/removals","since":"?since=YYYY-MM-DD","rule":"A domain whose owner opts out leaves every file, list and sample from the next read. Rows you already downloaded are not recalled by us: delete every row whose domain is in /api/datasets/removals (match on sha256 of the lower-cased domain without www.), and re-check at least every 30 days and before each re-use. Aggregate counts already published (totals, list sizes, rates) are history and are not recomputed; named records are removed."},"schema":{"version":"2026-10-29.1","columns":79,"order":"new columns are added at the end; a column is never renamed in place (a renamed field keeps its old name as an alias)","nulls":{"unmeasured row":"every measurement field is empty; reason says why the read failed","empty in a measured row":"not determined: the check was not run for this row (the field is newer than the row, see changelog) or its request failed","false / 0 / missing / []":"checked and not found: false and 0 mean the check ran and the answer was no; missing (dns_spf, dns_dmarc) means the record was looked up and absent; an empty list means the detector ran and matched nothing","robots-dependent fields":"ai_blocked and ai_closed are [] both when robots.txt allows everyone and when it was not read; robots_measured says which"},"changelog":[["2026-11-14","dsc1","phone, email, contact_src added on local-lane rows: what the business publishes on its own homepage. Rows read before this date do not have them"],["2026-10-29","cat3","rendering_heuristic, browser_render_verified, fetch_ms (load_ms kept as an alias), tech_evidence, tech_first_seen, tech_last_seen added"],["2026-10-29","sitetype2","kind: a page that declares NewsMediaOrganization is a publisher, even beside a LocalBusiness node"],["2026-10-23","hs1/hs2","Trades and home services list: OSM trades or the page's own words, not every local row; our own network removed from every file"],["2026-10-17","dssec1/dsdns1/rdap1","sec_*, dns_*, reg_* and net_* columns added"],["2026-10-16","ac1","ai_closed and content_signal added"],["2026-10-15","dsg1","ready-made lists"],["2026-10-03","ds1-ds5","dataset row extras (tech, cms, ld_types, llms_txt, surfaces, robots); grade filled on corpus rows; opted-out domains removed at read time"],["2026-10-01","dsp2","blocking AI crawlers counts AI agents only"]]},"days":[{"day":"2026-11-21","lanes":{"saas":{"rows":2339,"measured":1744,"bytes":4382910},"gov":{"rows":3165,"measured":1982,"bytes":4595053},"local":{"rows":10480,"measured":7297,"bytes":20055226},"filler":{"rows":3110,"measured":2475,"bytes":6085219}}},{"day":"2026-11-20","lanes":{"saas":{"rows":5909,"measured":3695,"bytes":8911166},"gov":{"rows":2078,"measured":1381,"bytes":3142768},"local":{"rows":14264,"measured":10033,"bytes":27906753},"filler":{"rows":4015,"measured":3299,"bytes":8027910}}},{"day":"2026-11-19","lanes":{"saas":{"rows":7015,"measured":4957,"bytes":11774702},"gov":{"rows":2284,"measured":1552,"bytes":3654171},"local":{"rows":14105,"measured":10134,"bytes":28066411},"filler":{"rows":4167,"measured":3480,"bytes":8531526}}},{"day":"2026-11-18","lanes":{"saas":{"rows":7077,"measured":5114,"bytes":12837036},"gov":{"rows":2536,"measured":1739,"bytes":4138930},"local":{"rows":14514,"measured":10192,"bytes":28186232},"filler":{"rows":5358,"measured":4547,"bytes":11030897}}},{"day":"2026-11-17","lanes":{"saas":{"rows":7311,"measured":5062,"bytes":12833340},"gov":{"rows":2544,"measured":1722,"bytes":4131975},"local":{"rows":14891,"measured":10049,"bytes":28498678},"filler":{"rows":5553,"measured":4717,"bytes":11536088}}},{"day":"2026-11-16","lanes":{"saas":{"rows":6398,"measured":4253,"bytes":10972140},"gov":{"rows":3015,"measured":1935,"bytes":4628932},"local":{"rows":14125,"measured":10171,"bytes":28754139},"filler":{"rows":5874,"measured":4963,"bytes":12116751}}},{"day":"2026-11-15","lanes":{"saas":{"rows":3941,"measured":2457,"bytes":6330660},"gov":{"rows":4007,"measured":2502,"bytes":5871553},"local":{"rows":14635,"measured":10109,"bytes":28604288},"filler":{"rows":4807,"measured":4085,"bytes":10265151}}},{"day":"2026-11-14","lanes":{"saas":{"rows":803,"measured":665,"bytes":1676339},"gov":{"rows":3899,"measured":2372,"bytes":5602601},"local":{"rows":11587,"measured":7715,"bytes":21897382},"filler":{"rows":4697,"measured":3909,"bytes":9239037}}},{"day":"2026-11-13","lanes":{"saas":{"rows":1412,"measured":1238,"bytes":3071670},"gov":{"rows":3363,"measured":2145,"bytes":5087876},"local":{"rows":2565,"measured":1835,"bytes":4973864},"filler":{"rows":3240,"measured":2555,"bytes":6282418}}},{"day":"2026-11-12","lanes":{"saas":{"rows":3181,"measured":2433,"bytes":6158757},"gov":{"rows":3788,"measured":2475,"bytes":5973031},"local":{"rows":4065,"measured":2591,"bytes":7202579},"filler":{"rows":1754,"measured":1422,"bytes":3591519}}},{"day":"2026-11-11","lanes":{"saas":{"rows":7046,"measured":5050,"bytes":12853061},"gov":{"rows":3709,"measured":2426,"bytes":5852357},"local":{"rows":2163,"measured":1586,"bytes":4257905},"filler":{"rows":2177,"measured":1707,"bytes":4223474}}},{"day":"2026-11-10","lanes":{"saas":{"rows":7194,"measured":5023,"bytes":12781605},"gov":{"rows":3849,"measured":2473,"bytes":5988291},"local":{"rows":1035,"measured":677,"bytes":1867355},"filler":{"rows":2588,"measured":2071,"bytes":5155128}}},{"day":"2026-11-09","lanes":{"saas":{"rows":7436,"measured":5018,"bytes":12885680},"gov":{"rows":3836,"measured":2500,"bytes":6054411},"local":{"rows":2095,"measured":1527,"bytes":4173388},"filler":{"rows":5011,"measured":4204,"bytes":9921135}}},{"day":"2026-11-08","lanes":{"saas":{"rows":6929,"measured":5014,"bytes":12965482},"gov":{"rows":1603,"measured":1139,"bytes":2645411},"local":{"rows":2459,"measured":1629,"bytes":4517410},"filler":{"rows":2811,"measured":2448,"bytes":5915254}}},{"day":"2026-11-07","lanes":{"saas":{"rows":4251,"measured":3507,"bytes":9005017},"gov":{"rows":1003,"measured":788,"bytes":1724498},"local":{"rows":2295,"measured":1486,"bytes":4011622},"filler":{"rows":8605,"measured":7166,"bytes":17319724}}},{"day":"2026-11-06","lanes":{"saas":{"rows":501,"measured":461,"bytes":1151094},"gov":{"rows":1003,"measured":749,"bytes":1672582},"local":{"rows":1142,"measured":687,"bytes":1856888},"filler":{"rows":3359,"measured":2770,"bytes":6534763}}},{"day":"2026-11-05","lanes":{"saas":{"rows":222,"measured":189,"bytes":462234},"gov":{"rows":1814,"measured":1245,"bytes":3003998},"local":{"rows":3678,"measured":2440,"bytes":6606144},"filler":{"rows":7073,"measured":5922,"bytes":13489049}}},{"day":"2026-11-04","lanes":{"saas":{"rows":161,"measured":142,"bytes":345984},"gov":{"rows":1449,"measured":880,"bytes":2148070},"local":{"rows":2316,"measured":1564,"bytes":4165012},"filler":{"rows":2936,"measured":2380,"bytes":5810881}}},{"day":"2026-11-03","lanes":{"saas":{"rows":125,"measured":106,"bytes":257209},"gov":{"rows":759,"measured":445,"bytes":1104642},"local":{"rows":2649,"measured":1756,"bytes":4742250},"filler":{"rows":4099,"measured":3256,"bytes":7609713}}},{"day":"2026-11-02","lanes":{"saas":{"rows":178,"measured":148,"bytes":381612},"gov":{"rows":79,"measured":55,"bytes":122833},"local":{"rows":4317,"measured":2829,"bytes":7546937},"filler":{"rows":4771,"measured":3926,"bytes":9044477}}},{"day":"2026-11-01","lanes":{"saas":{"rows":157,"measured":129,"bytes":337792},"gov":{"rows":70,"measured":58,"bytes":125294},"local":{"rows":4475,"measured":3160,"bytes":8466917},"filler":{"rows":5820,"measured":4828,"bytes":11214971}}},{"day":"2026-10-31","lanes":{"saas":{"rows":98,"measured":77,"bytes":200195},"gov":{"rows":107,"measured":86,"bytes":190580},"local":{"rows":4130,"measured":2892,"bytes":7708076},"filler":{"rows":8659,"measured":6793,"bytes":15869481}}},{"day":"2026-10-30","lanes":{"saas":{"rows":173,"measured":150,"bytes":383150},"gov":{"rows":150,"measured":132,"bytes":268347},"local":{"rows":2659,"measured":1977,"bytes":5246503},"filler":{"rows":8873,"measured":6854,"bytes":16099295}}},{"day":"2026-10-29","lanes":{"saas":{"rows":49,"measured":44,"bytes":102298},"gov":{"rows":139,"measured":117,"bytes":213754},"local":{"rows":12706,"measured":9129,"bytes":22335203},"filler":{"rows":9551,"measured":7387,"bytes":16110410}}},{"day":"2026-10-28","lanes":{"saas":{"rows":102,"measured":79,"bytes":184243},"gov":{"rows":140,"measured":121,"bytes":224807},"local":{"rows":2765,"measured":2024,"bytes":4940162},"filler":{"rows":9541,"measured":7157,"bytes":15275831}}},{"day":"2026-10-27","lanes":{"saas":{"rows":168,"measured":144,"bytes":324236},"gov":{"rows":187,"measured":146,"bytes":283219},"local":{"rows":3530,"measured":2506,"bytes":6144091},"filler":{"rows":6070,"measured":4556,"bytes":9857169}}},{"day":"2026-10-26","lanes":{"saas":{"rows":624,"measured":539,"bytes":1200929},"gov":{"rows":270,"measured":196,"bytes":366928},"local":{"rows":3081,"measured":2088,"bytes":5132317},"filler":{"rows":3653,"measured":2874,"bytes":6075388}}},{"day":"2026-10-25","lanes":{"saas":{"rows":3939,"measured":2264,"bytes":4983901},"gov":{"rows":555,"measured":412,"bytes":791295},"filler":{"rows":2921,"measured":2335,"bytes":4746313},"local":{"rows":1440,"measured":1017,"bytes":2593373}}},{"day":"2026-10-24","lanes":{"saas":{"rows":4784,"measured":3147,"bytes":6717404},"gov":{"rows":1578,"measured":941,"bytes":1948784},"local":{"rows":7931,"measured":5588,"bytes":13725146},"filler":{"rows":2529,"measured":1840,"bytes":3909268}}},{"day":"2026-10-23","lanes":{"saas":{"rows":1272,"measured":866,"bytes":2055542},"gov":{"rows":2002,"measured":1214,"bytes":2522184},"local":{"rows":8906,"measured":6095,"bytes":14965141},"filler":{"rows":6501,"measured":4735,"bytes":10181303}}},{"day":"2026-10-22","lanes":{"saas":{"rows":663,"measured":456,"bytes":1046681},"gov":{"rows":377,"measured":305,"bytes":543024},"local":{"rows":5931,"measured":4029,"bytes":9241027},"filler":{"rows":7109,"measured":5464,"bytes":10933997}}},{"day":"2026-10-21","lanes":{"saas":{"rows":308,"measured":215,"bytes":456055},"gov":{"rows":350,"measured":283,"bytes":500746},"local":{"rows":8494,"measured":5763,"bytes":13208812},"filler":{"rows":5643,"measured":4517,"bytes":8971575}}},{"day":"2026-10-20","lanes":{"saas":{"rows":415,"measured":354,"bytes":733865},"gov":{"rows":374,"measured":320,"bytes":561096},"local":{"rows":4731,"measured":3206,"bytes":7339828},"filler":{"rows":5948,"measured":4621,"bytes":9149013}}},{"day":"2026-10-19","lanes":{"saas":{"rows":1449,"measured":1261,"bytes":2539454}}},{"day":"2026-10-18","lanes":{"saas":{"rows":7447,"measured":4553,"bytes":8997831}}},{"day":"2026-10-17","lanes":{"saas":{"rows":6000,"measured":5074,"bytes":7400588},"gov":{"rows":905,"measured":673,"bytes":1025865},"local":{"rows":5872,"measured":3929,"bytes":6847644},"filler":{"rows":9013,"measured":6415,"bytes":11920728}}},{"day":"2026-10-16","lanes":{"saas":{"rows":806,"measured":591,"bytes":601714},"gov":{"rows":3127,"measured":1893,"bytes":2030301},"local":{"rows":10223,"measured":7219,"bytes":8732549},"filler":{"rows":8971,"measured":6576,"bytes":6632978}}},{"day":"2026-10-15","lanes":{"saas":{"rows":206,"measured":157,"bytes":164899},"gov":{"rows":3292,"measured":2054,"bytes":2165177},"local":{"rows":7075,"measured":5012,"bytes":6187961},"filler":{"rows":9509,"measured":6824,"bytes":6761632}}},{"day":"2026-10-14","lanes":{"saas":{"rows":201,"measured":169,"bytes":172168},"gov":{"rows":3013,"measured":1766,"bytes":1851538},"local":{"rows":6909,"measured":4639,"bytes":5812510},"filler":{"rows":9324,"measured":6564,"bytes":6605698}}},{"day":"2026-10-13","lanes":{"saas":{"rows":610,"measured":507,"bytes":519482},"gov":{"rows":2684,"measured":1694,"bytes":1719829},"local":{"rows":7211,"measured":5016,"bytes":6077239},"filler":{"rows":7754,"measured":5865,"bytes":5797411}}},{"day":"2026-10-12","lanes":{"saas":{"rows":3009,"measured":2254,"bytes":2344667},"gov":{"rows":2812,"measured":1684,"bytes":1732625},"local":{"rows":7044,"measured":5004,"bytes":6093389},"filler":{"rows":8456,"measured":6443,"bytes":6254511}}},{"day":"2026-10-11","lanes":{"saas":{"rows":6925,"measured":5005,"bytes":5204534},"gov":{"rows":1988,"measured":1263,"bytes":1286832},"local":{"rows":7173,"measured":5008,"bytes":6062922},"filler":{"rows":9539,"measured":7520,"bytes":7279230}}},{"day":"2026-10-10","lanes":{"saas":{"rows":7285,"measured":4991,"bytes":5230941},"gov":{"rows":3879,"measured":2500,"bytes":2568149},"local":{"rows":6860,"measured":5000,"bytes":5944020},"filler":{"rows":2727,"measured":2215,"bytes":2147918}}},{"day":"2026-10-09","lanes":{"saas":{"rows":5904,"measured":3782,"bytes":4018669},"gov":{"rows":3924,"measured":2510,"bytes":2600225},"local":{"rows":6889,"measured":5009,"bytes":6058825},"filler":{"rows":9420,"measured":7423,"bytes":7073921}}},{"day":"2026-10-08","lanes":{"saas":{"rows":1659,"measured":1452,"bytes":1444973},"gov":{"rows":3856,"measured":2500,"bytes":2598295},"local":{"rows":6586,"measured":5000,"bytes":6043564},"filler":{"rows":8780,"measured":6583,"bytes":6379412}}},{"day":"2026-10-07","lanes":{"saas":{"rows":6775,"measured":4996,"bytes":5168637},"gov":{"rows":3550,"measured":2501,"bytes":2493741},"local":{"rows":6560,"measured":4988,"bytes":5961433},"filler":{"rows":6647,"measured":4877,"bytes":4769522}}},{"day":"2026-10-06","lanes":{"saas":{"rows":7085,"measured":5001,"bytes":5207380},"gov":{"rows":3560,"measured":2505,"bytes":2515410},"local":{"rows":7105,"measured":5011,"bytes":6295227},"filler":{"rows":50674,"measured":32135,"bytes":32847701}}},{"day":"2026-10-05","lanes":{"saas":{"rows":7135,"measured":5004,"bytes":5213851},"gov":{"rows":3685,"measured":2512,"bytes":2507613},"local":{"rows":6940,"measured":4995,"bytes":6250619},"filler":{"rows":51840,"measured":33564,"bytes":34125498}}},{"day":"2026-10-04","lanes":{"saas":{"rows":7130,"measured":5036,"bytes":5256054},"gov":{"rows":3610,"measured":2526,"bytes":2531418},"local":{"rows":6770,"measured":5035,"bytes":6284766},"filler":{"rows":54964,"measured":34556,"bytes":35440688}}},{"day":"2026-10-03","lanes":{"saas":{"rows":5881,"measured":5010,"bytes":5179183},"gov":{"rows":3956,"measured":2510,"bytes":2459128},"local":{"rows":6683,"measured":5011,"bytes":5841927},"filler":{"rows":17528,"measured":12193,"bytes":12079015}}}],"latest":{"day":"2026-11-21","lanes":{"saas":{"rows":2339,"measured":1744,"kind":{"local-service":40,"publisher":124,"other":883,"saas":617,"gov-edu":7,"automotive":8,"ecommerce":35,"docs":16,"health":4,"legal":3,"nonprofit":1,"food":1,"travel":1,"finance":4},"state":{},"platform":{"wordpress":226,"unknown":1103,"nextjs":332,"squarespace":5,"webflow":54,"wix":9,"drupal":11,"shopify":4},"grade":{"A":484,"B":145,"D":569,"C":423,"F":81},"llms_txt":547,"ai_blocking":247,"top_tech":[["jQuery",487],["Google Tag Manager",475],["Google Analytics",468],["Google Fonts",467],["Next.js",334],["Font Awesome",287],["Yoast SEO",154],["Bootstrap",140],["React",97],["Elementor",93],["YouTube embed",89],["Google Ads",75],["WooCommerce",73],["Meta Pixel",73],["reCAPTCHA",70],["HubSpot",65],["Webflow",60],["Plausible",48],["Microsoft Clarity",48],["Rank Math",41],["Adobe Fonts",39],["Vue",38],["OneTrust",37],["Cloudflare Web Analytics",36],["LinkedIn Insight",32]]},"gov":{"rows":3165,"measured":1982,"kind":{"gov-edu":1982},"state":{},"platform":{"unknown":1107,"wordpress":476,"drupal":353,"squarespace":4,"nextjs":38,"webflow":3,"wix":1},"grade":{"D":802,"B":136,"C":732,"F":155,"A":121},"llms_txt":97,"ai_blocking":289,"top_tech":[["jQuery",1249],["Font Awesome",918],["Google Fonts",807],["Google Analytics",760],["Google Tag Manager",716],["Bootstrap",536],["YouTube embed",254],["Yoast SEO",235],["Elementor",151],["Adobe Fonts",138],["Meta Pixel",118],["WooCommerce",99],["Google Maps",96],["reCAPTCHA",91],["Google Ads",58],["Microsoft Clarity",57],["Vimeo",55],["React",43],["Next.js",38],["Matomo",37],["Divi",33],["Hotjar",31],["New Relic",31],["Mailchimp",27],["Rank Math",27]]},"local":{"rows":10480,"measured":7297,"kind":{"publisher":829,"local-service":3361,"food":1173,"health":478,"automotive":235,"ecommerce":580,"finance":210,"saas":97,"travel":117,"legal":78,"gov-edu":21,"nonprofit":72,"real-estate":46},"state":{"ohio":7297},"platform":{"wordpress":2233,"wix":687,"unknown":3275,"nextjs":178,"squarespace":685,"shopify":83,"webflow":92,"drupal":64},"grade":{"A":1934,"C":2696,"D":1503,"B":791,"F":200},"llms_txt":1630,"ai_blocking":1369,"top_tech":[["jQuery",3555],["Google Fonts",2758],["Google Analytics",2387],["Font Awesome",2338],["Google Tag Manager",1678],["Elementor",1304],["Yoast SEO",1155],["Bootstrap",956],["Adobe Fonts",880],["WooCommerce",873],["reCAPTCHA",814],["Google Maps",730],["Wix",692],["Squarespace",676],["Meta Pixel",555],["YouTube embed",458],["Google Ads",314],["Divi",310],["Rank Math",213],["GoDaddy Builder",209],["Vimeo",201],["Duda",181],["Next.js",178],["WP Rocket",177],["React",159]]},"filler":{"rows":3110,"measured":2475,"kind":{"publisher":380,"other":1503,"local-service":137,"ecommerce":98,"saas":182,"food":22,"gov-edu":65,"nonprofit":13,"finance":27,"automotive":4,"travel":23,"docs":2,"health":14,"real-estate":4,"legal":1},"state":{},"platform":{"wordpress":672,"unknown":1450,"nextjs":119,"wix":56,"squarespace":70,"drupal":52,"webflow":32,"shopify":24},"grade":{"C":657,"A":453,"B":214,"D":899,"F":187},"llms_txt":307,"ai_blocking":463,"top_tech":[["jQuery",1237],["Google Fonts",844],["Google Analytics",810],["Font Awesome",706],["Google Tag Manager",583],["Bootstrap",339],["Yoast SEO",335],["Elementor",285],["WooCommerce",213],["reCAPTCHA",207],["Adobe Fonts",154],["Meta Pixel",140],["YouTube embed",139],["Next.js",122],["Google Ads",98],["Google Maps",93],["Vimeo",93],["React",80],["Rank Math",71],["Divi",71],["OneTrust",69],["Squarespace",65],["Microsoft Clarity",57],["Wix",55],["Mailchimp",54]]}}},"fields":[{"name":"domain","meaning":"Registrable domain scanned"},{"name":"at","meaning":"When the homepage was read (UTC, ISO 8601)"},{"name":"lane","meaning":"Lane the site was scanned in: saas, gov, local, filler"},{"name":"kind","meaning":"Site kind: the page's own declaration (JSON-LD), else its patterns; for local rows with no page signal, the OpenStreetMap category"},{"name":"kind_detected","meaning":"Kind read from the page alone"},{"name":"state","meaning":"US state (local lane)"},{"name":"city","meaning":"City from OpenStreetMap (local lane)"},{"name":"name","meaning":"Business name from OpenStreetMap (local lane)"},{"name":"phone","meaning":"Phone number the business publishes on its own homepage (local lane, rows read from 2026-11-14)"},{"name":"email","meaning":"Email address the business publishes on its own homepage (local lane, rows read from 2026-11-14); an address the site hides from scripts is left out"},{"name":"contact_src","meaning":"Where each contact was read: schema (the page's JSON-LD) or link (a tel: or mailto: link)"},{"name":"osm_category","meaning":"OpenStreetMap category, e.g. trades:plumber (local lane)"},{"name":"src","meaning":"Where the domain came from: dotgov:<type>, tranco:<tld>, wikidata, discovered, OSM category"},{"name":"title","meaning":"Homepage <title>"},{"name":"description","meaning":"Homepage meta description"},{"name":"language","meaning":"en or non-en, from the page"},{"name":"grade","meaning":"CrawlCheck grade, A to F"},{"name":"score","meaning":"CrawlCheck overall score 0-100, before any cap"},{"name":"grade_cap_reason","meaning":"When a finding caps the letter below what the score would give, which finding and why"},{"name":"crawl_clarity","meaning":"Crawl clarity score 0-100"},{"name":"user_clarity","meaning":"User clarity score 0-100"},{"name":"findings","meaning":"Number of findings"},{"name":"finding_codes","meaning":"Finding codes, ;-separated"},{"name":"platform","meaning":"CMS band: wordpress, shopify, webflow, squarespace, wix, nextjs, drupal, unknown"},{"name":"cms","meaning":"CMS and builders detected"},{"name":"tech","meaning":"Technologies detected in the homepage: analytics, tag managers, chat, payments, booking, consent, frameworks"},{"name":"edge","meaning":"Edge or CDN detected from headers"},{"name":"server","meaning":"Server header"},{"name":"host","meaning":"First hosting signal"},{"name":"rendering","meaning":"server or client, by a rule on the HTML we received, not by rendering it: client when under 1% of the payload is visible text (rule content_ratio_lt_1pct/v1, see rendering_heuristic). A heavy server-rendered page can read client"},{"name":"content_ratio","meaning":"Share of the homepage payload that is visible content, %"},{"name":"blocking_resources","meaning":"Render-blocking resources in <head>"},{"name":"id_ratio","meaning":"Share of JSON-LD nodes with an @id"},{"name":"ld_types","meaning":"schema.org types declared in JSON-LD"},{"name":"llms_txt","meaning":"Serves /llms.txt"},{"name":"surfaces","meaning":"Agent surfaces served: agents.md, agent skills, MCP card, API catalog, media kit"},{"name":"robots_measured","meaning":"robots.txt was read"},{"name":"ai_blocked","meaning":"AI crawlers robots.txt closes / to"},{"name":"ai_closed","meaning":"Every AI agent (training and answer) robots.txt closes / to, of ~60 tested; reads from 16 Oct 2026"},{"name":"content_signal","meaning":"The Content-Signal line, verbatim, when one was read"},{"name":"https","meaning":"Homepage on HTTPS"},{"name":"hsts","meaning":"HSTS header present"},{"name":"sitemap_urls","meaning":"URLs declared in sitemaps"},{"name":"size","meaning":"small (<50), medium (<2000), large"},{"name":"home_bytes","meaning":"Homepage bytes"},{"name":"ttfb_ms","meaning":"Time to first byte from our observer, ms"},{"name":"load_ms","meaning":"Deprecated alias of fetch_ms, same value; kept so existing imports do not break"},{"name":"sec_headers","meaning":"Standard security headers sent, of 6: HSTS, a CSP that restricts scripts, X-Frame-Options or frame-ancestors, nosniff, Referrer-Policy, Permissions-Policy"},{"name":"sec_hsts_max_age","meaning":"HSTS max-age in seconds (blank when no HSTS)"},{"name":"sec_csp","meaning":"Content Security Policy: none, report-only, no-script-rule, unsafe-inline, strict"},{"name":"sec_version_leak","meaning":"Server or framework version exposed in headers, e.g. Apache/2.4.41 or PHP/7.4"},{"name":"sec_jquery","meaning":"jQuery version loaded by the homepage"},{"name":"sec_mixed_content","meaning":"http:// resources loaded on the https homepage"},{"name":"sec_cookies_insecure","meaning":"Cookies set without the Secure flag"},{"name":"sec_scripts_no_sri","meaning":"Scripts from other domains without an integrity hash"},{"name":"dns_spf","meaning":"SPF policy: fail, softfail, neutral, pass, none (no all), missing"},{"name":"dns_dmarc","meaning":"DMARC policy: reject, quarantine, none, missing (inherited from the parent domain when the subdomain has none)"},{"name":"dns_caa","meaning":"CAA record limits which certificate authorities may issue"},{"name":"dns_mx","meaning":"Mail servers (a null MX counts as 0)"},{"name":"dns_dnssec","meaning":"DNS answers validated with DNSSEC"},{"name":"reg_registrar","meaning":"Registrar of the registrable domain (RDAP)"},{"name":"reg_created","meaning":"Domain registered on (YYYY-MM-DD)"},{"name":"reg_expires","meaning":"Registration expires on (YYYY-MM-DD)"},{"name":"reg_age_days","meaning":"Days since the domain was registered"},{"name":"reg_expires_in_days","meaning":"Days until the registration expires"},{"name":"reg_dns_provider","meaning":"Who runs the domain's name servers, e.g. Cloudflare, GoDaddy, Amazon Route 53"},{"name":"reg_dnssec","meaning":"DNSSEC delegation signed at the registry"},{"name":"net_ip","meaning":"Homepage IPv4 address"},{"name":"net_asn","meaning":"Network (autonomous system) the IP belongs to"},{"name":"net_as_name","meaning":"Network owner, e.g. Cloudflare, Amazon, Automattic"},{"name":"net_country","meaning":"Country the network registered the IP block in"},{"name":"unmeasured","meaning":"true when the homepage could not be measured"},{"name":"reason","meaning":"Why it could not be measured"},{"name":"rendering_heuristic","meaning":"Id of the rule that set rendering (content_ratio_lt_1pct/v1); a new rule gets a new id"},{"name":"browser_render_verified","meaning":"true only when a real browser rendered the page and confirmed rendering; false for every corpus row"},{"name":"fetch_ms","meaning":"Time from our request to the last byte of the homepage HTML document (read up to 4 MB), from one datacentre. No images, scripts or styles are fetched and nothing is rendered: not a page-load time"},{"name":"tech_evidence","meaning":"For each technology, where in the HTML it was matched: script-src, inline-script, script-attribute, link-href, iframe-src, meta-tag, img-src, attribute, text, markup. Rows from 29 Oct 2026"},{"name":"tech_first_seen","meaning":"For each technology, the first day (UTC) any read of this site in this lane saw it, since the lane's files began"},{"name":"tech_last_seen","meaning":"For each technology, the latest day a read saw it"}],"tech_detected":["Google Analytics","Google Tag Manager","Google Ads","Meta Pixel","LinkedIn Insight","TikTok Pixel","Microsoft Clarity","Bing UET","Hotjar","Segment","Mixpanel","Amplitude","Heap","Plausible","Matomo","HubSpot","Salesforce Pardot","Marketo","Klaviyo","Mailchimp","Intercom","Drift","Zendesk","LiveChat","Tawk.to","Crisp","Podium","Birdeye","CallRail","Calendly","ServiceTitan","Housecall Pro","Jobber","Stripe","PayPal","Square","Shopify","WooCommerce","BigCommerce","Yotpo","Judge.me","Trustpilot","OneTrust","Cookiebot","Termly","reCAPTCHA","hCaptcha","Cloudflare Turnstile","Cloudflare Web Analytics","Sentry","New Relic","Datadog RUM","Optimizely","VWO","Algolia","jQuery","React","Next.js","Nuxt","Vue","Angular","Svelte","Gatsby","Bootstrap","Tailwind CSS","Elementor","Divi","Yoast SEO","Rank Math","WP Rocket","Webflow","Framer","Ghost","Wix","Squarespace","Duda","GoDaddy Builder","Google Fonts","Adobe Fonts","Font Awesome","YouTube embed","Vimeo","Google Maps","Typeform","Jotform","POWR"],"sample":{"csv":"https://crawlcheck.io/api/datasets/sample.csv","ndjson":"https://crawlcheck.io/api/datasets/sample.ndjson","rows":25},"files":{"url":"https://crawlcheck.io/api/datasets/file?day=YYYY-MM-DD&lane=saas|gov|local|filler&format=csv|ndjson","filters":["kind","state","tech","platform","measured=1"],"access":"a CrawlCheck licence key with dataset access (header x-crawlcheck-key)"},"access":{"page":"https://crawlcheck.io/datasets","request":"POST https://crawlcheck.io/api/datasets/request {email, company, lanes, note}"},"catalogue":{"license":{"open":"CC BY 4.0","url":"https://creativecommons.org/licenses/by/4.0/","credit":"CrawlCheck, with the URL of the file"},"open":[{"id":"snapshot","name":"The Resolve snapshot","what":"Every public signed answer about a measured site, one file set a day; each line verifies on its own with no call to us","cadence":"daily, newest 7 public","count":99784,"count_of":"answers in the newest (2026-11-21)","links":[["/api/v1/snapshot","Index"],["/docs/api#snapshot","File layout"]]},{"id":"census","name":"Agentic-web census","what":"The most-visited domains on a fixed frame: crawler policy, machine files, declared capabilities, and whether AI crawlers get what a browser gets; every site named","cadence":"daily and weekly","count":null,"links":[["/api/census","Daily records"],["/api/v1/census/weekly","Weekly"],["/api/v1/census/weekly/feed.xml","Atom"]]},{"id":"instruments","name":"The instruments","what":"Decisions to outcomes, Metrics, Agent safety benchmark, Agent Traffic Index, Assurance, Status, Agent conduct, Proof: the Resolve log, Agentic web census, weekly, Agentic web map, Share of preflight, Vendor website benchmark, Registry: what sites declare to agents: each a signed JSON twin with one sealed edition per ISO week","cadence":"signed hourly, sealed weekly","count":13,"count_of":"instruments","links":[["/api/v1/instruments","Index"],["/data","The pages"]]},{"id":"findings","name":"Finding frequencies","what":"Every scan run here, counted by finding: how common, how serious; names no domain","cadence":"continuous","count":null,"links":[["/api/public/counts","JSON"],["/data#findings","Table"]]},{"id":"log","name":"The Resolve log","what":"Every signed answer’s leaf, merged into chained, witnessed batches with RFC 9162 proofs","cadence":"every batch","count":622007,"count_of":"leaves","links":[["/api/v1/log","Log"],["/keys","Signing keys"]]},{"id":"crawlers","name":"AI crawler catalogue","what":"Every named AI, search and training agent: operator, purpose, and how its traffic can be verified","cadence":"on change","count":116,"count_of":"agents","links":[["/ai-crawlers.json","JSON"],["/ai-crawlers","Page"]]},{"id":"directory","name":"Vendor website benchmark","what":"SEO and AI-visibility vendors measured by the same rules as everyone else, with immutable daily snapshots","cadence":"daily","count":null,"links":[["/directory.json","JSON"],["/directory.csv","CSV"],["/api/v2/directory/snapshots","Snapshots"]]},{"id":"glossary","name":"Glossary","what":"Every defined term, with what it can support and what it cannot, as DefinedTermSet JSON-LD","cadence":"weekly","count":792,"count_of":"terms","links":[["/glossary.json","JSON-LD"],["/glossary","Page"]]}],"kept":[{"id":"archive","name":"The snapshot archive","what":"Every Resolve snapshot behind the public window, kept 90 days: the same signed files, listed and served on your key, so a change on any measured site can be placed to the day","held":44,"behind":37,"price":{"monthly":199,"yearly":1990,"included_in":"Growth"},"buy":"/pricing#data","access":"x-crawlcheck-key on GET /api/v1/snapshot"},{"id":"lanes","name":"Row-level lane files","what":"One row per measured site per day, 79 fields: SaaS, government and education, US local businesses by state, and the wider corpus","rows":724183,"days":50,"price":{"state_pack":149,"lane_monthly":349,"all_monthly":899,"enterprise_from_yearly":25000},"sample":[["/api/datasets/sample","CSV"],["/api/datasets/sample?format=ndjson","NDJSON"]],"buy":"/datasets#pricing"}]}}