{"dataset":"CrawlCheck Vendor Website Benchmark","dataset_version":"2026.09.17","snapshot":"2026-09-17","immutable":true,"source":"https://crawlcheck.io/directory","index_built":"2026-09-17T00:18:17.839Z","taken":"2026-09-17T00:18:17.912Z","score_version":23,"license":"https://creativecommons.org/licenses/by/4.0/","schema":"https://crawlcheck.io/api/v2/directory/schema","rows":[{"slug":"ahrefs","name":"Ahrefs","domain":"ahrefs.com","cat":"SEO suite","measured":true,"at":"2026-09-16T11:17:31.299Z","id":"y3lskdb9bm","refused":false,"alias_of":null,"grade":"C","score":79,"findings":5,"level":"medium","codes":["ROBOTS_NO_SITEMAP","RENDER_RESOURCE_BLOCKED","PAGE_IS_MOSTLY_CODE","NO_LLMS_TXT","MACHINE_CHAIN_HEAVY"],"reach":94,"read":67,"quote":72,"first_seen":"2026-08-18T21:13:13.275Z","scans":14,"llms":false,"entitymap":false},{"slug":"semrush","name":"Semrush","domain":"semrush.com","cat":"SEO suite","measured":true,"at":"2026-09-16T12:18:00.471Z","id":"05rhfrico4","refused":false,"alias_of":null,"grade":"C","score":79,"findings":4,"level":"medium","codes":["ROBOTS_RULES_SHADOWED","PAGE_IS_MOSTLY_CODE","SITEMAP_NO_LASTMOD","MACHINE_CHAIN_HEAVY"],"reach":89,"read":67,"quote":78,"first_seen":"2026-08-14T19:52:10.266Z","scans":17,"llms":true,"entitymap":false},{"slug":"moz","name":"Moz","domain":"moz.com","cat":"SEO suite","measured":true,"at":"2026-09-16T13:17:59.572Z","id":"tq4q0sn921","refused":false,"alias_of":null,"grade":"C","score":82,"findings":5,"level":"medium","codes":["ROBOTS_RULES_SHADOWED","ENTITY_COLLISION","STALE_CACHE_SERVED","NO_LLMS_TXT","MACHINE_CHAIN_HEAVY"],"reach":95,"read":75,"quote":70,"first_seen":"2026-08-18T21:45:34.811Z","scans":15,"llms":false,"entitymap":false},{"slug":"seranking","name":"SE Ranking","domain":"seranking.com","cat":"SEO suite","measured":true,"at":"2026-09-16T14:18:00.665Z","id":"f1s3nct3e1","refused":false,"alias_of":null,"grade":"C","score":81,"findings":6,"level":"medium","codes":["ROBOTS_RULES_SHADOWED","PAGE_IS_MOSTLY_CODE","HSTS_PRELOAD_INELIGIBLE","NO_LLMS_TXT","MACHINE_CHAIN_HEAVY","ENTITY_NO_COORDINATES"],"reach":92,"read":70,"quote":76,"first_seen":"2026-08-14T19:52:19.460Z","scans":14,"llms":false,"entitymap":false},{"slug":"similarweb","name":"Similarweb","domain":"similarweb.com","cat":"SEO suite","measured":true,"at":"2026-09-16T15:18:00.370Z","id":"eb8r9mxz4h","refused":false,"alias_of":null,"grade":"C","score":69,"findings":3,"level":"medium","codes":["ENTITY_COLLISION","PAGE_IS_MOSTLY_CODE","MACHINE_CHAIN_HEAVY"],"reach":77,"read":65,"quote":63,"first_seen":"2026-08-18T21:13:36.000Z","scans":16,"llms":true,"entitymap":false},{"slug":"screaming-frog","name":"Screaming Frog","domain":"screamingfrog.co.uk","cat":"Site crawler","measured":true,"at":"2026-09-16T16:17:32.925Z","id":"3jsg2h2ker","refused":false,"alias_of":null,"grade":"C","score":76,"findings":4,"level":"medium","codes":["ROBOTS_NO_SITEMAP","PAGE_IS_MOSTLY_CODE","NO_LLMS_TXT","MACHINE_CHAIN_HEAVY"],"reach":80,"read":64,"quote":83,"first_seen":"2026-08-14T19:52:10.454Z","scans":18,"llms":false,"entitymap":false},{"slug":"sitebulb","name":"Sitebulb","domain":"sitebulb.com","cat":"Site crawler","measured":true,"at":"2026-09-16T17:17:32.378Z","id":"mim2n47hii","refused":false,"alias_of":null,"grade":"A","score":85,"findings":3,"level":"info","codes":["NO_LLMS_TXT","HSTS_MISSING","MACHINE_CHAIN_HEAVY"],"reach":92,"read":85,"quote":76,"first_seen":"2026-08-18T21:16:10.791Z","scans":10,"llms":false,"entitymap":false},{"slug":"lumar","name":"Lumar","domain":"lumar.io","cat":"Site crawler","measured":true,"at":"2026-09-16T18:17:31.703Z","id":"u97zynd2nt","refused":false,"alias_of":null,"grade":"B","score":83,"findings":2,"level":"info","codes":["NO_LLMS_TXT","MACHINE_CHAIN_HEAVY"],"reach":89,"read":70,"quote":87,"first_seen":"2026-08-14T19:52:37.071Z","scans":9,"llms":false,"entitymap":false},{"slug":"botify","name":"Botify","domain":"botify.com","cat":"Site crawler","measured":true,"at":"2026-09-16T19:17:31.850Z","id":"j1uowmgvwg","refused":false,"alias_of":null,"grade":"C","score":85,"findings":6,"level":"medium","codes":["BROKEN_INTERNAL_LINK","PAGE_IS_MOSTLY_CODE","PLACEHOLDER_CONTENT","NO_LLMS_TXT","MACHINE_CHAIN_HEAVY","STALE_CACHE_SERVED"],"reach":96,"read":75,"quote":81,"first_seen":"2026-08-18T21:18:13.080Z","scans":13,"llms":false,"entitymap":false},{"slug":"oncrawl","name":"Oncrawl","domain":"oncrawl.com","cat":"Site crawler","measured":true,"at":"2026-09-16T20:17:32.868Z","id":"g2uoaxntul","refused":false,"alias_of":null,"grade":"B","score":76,"findings":3,"level":"info","codes":["HSTS_MISSING","MACHINE_CHAIN_HEAVY","ENTITY_NO_COORDINATES"],"reach":79,"read":80,"quote":67,"first_seen":"2026-08-14T19:52:36.430Z","scans":11,"llms":true,"entitymap":false},{"slug":"seobility","name":"Seobility","domain":"seobility.net","cat":"Site crawler","measured":true,"at":"2026-09-16T21:17:31.746Z","id":"dgwvwtrtht","refused":false,"alias_of":null,"grade":"C","score":78,"findings":5,"level":"medium","codes":["HOST_DUPLICATE_200","ROBOTS_RULES_SHADOWED","PAGE_IS_MOSTLY_CODE","STALE_CACHE_SERVED","HSTS_MISSING"],"reach":86,"read":67,"quote":77,"first_seen":"2026-08-18T21:22:24.261Z","scans":16,"llms":true,"entitymap":false},{"slug":"siteliner","name":"Siteliner","domain":"siteliner.com","cat":"Site crawler","measured":true,"at":"2026-09-16T22:17:56.321Z","id":"05a2vhwtsh","refused":false,"alias_of":null,"grade":"F","score":39,"findings":10,"level":"critical","codes":["ROBOTS_IS_HTML","SITEMAP_IS_HTML","SITEMAP_IS_HTML","SITEMAP_IS_HTML","HOST_DUPLICATE_200","ROBOTS_NO_SITEMAP","PAGE_IS_MOSTLY_CODE","NO_LLMS_TXT","HSTS_MISSING","MACHINE_CHAIN_HEAVY"],"reach":48,"read":20,"quote":46,"first_seen":"2026-08-18T21:21:35.891Z","scans":21,"llms":false,"entitymap":false},{"slug":"conductor","name":"Conductor","domain":"conductor.com","cat":"Enterprise SEO","measured":true,"at":"2026-09-16T23:17:46.753Z","id":"cuxcy429fc","refused":false,"alias_of":null,"grade":"C","score":70,"findings":7,"level":"medium","codes":["PAGE_IS_MOSTLY_CODE","NAP_NAME_DRIFT","ANSWER_ENGINE_REFUSED","MACHINE_CHAIN_HEAVY","ENTITY_NO_COORDINATES","NAP_UNDECLARED_LISTING","STALE_CACHE_SERVED"],"reach":78,"read":60,"quote":68,"first_seen":"2026-08-14T19:52:37.362Z","scans":17,"llms":true,"entitymap":false},{"slug":"brightedge","name":"BrightEdge","domain":"brightedge.com","cat":"Enterprise SEO","measured":true,"at":"2026-09-17T00:17:45.599Z","id":"c7q6626uvd","refused":false,"alias_of":null,"grade":"C","score":66,"findings":1,"level":"info","codes":["MACHINE_CHAIN_HEAVY"],"reach":81,"read":65,"quote":46,"first_seen":"2026-08-14T19:52:44.743Z","scans":14,"llms":true,"entitymap":false},{"slug":"otterly","name":"Otterly","domain":"otterly.ai","cat":"AI visibility monitoring","measured":true,"at":"2026-09-16T00:17:02.088Z","id":"szhp8qpk9y","refused":true,"alias_of":null,"grade":null,"score":null,"findings":0,"level":null,"codes":[],"reach":null,"read":null,"quote":null,"llms":null,"entitymap":null},{"slug":"peec","name":"Peec AI","domain":"peec.ai","cat":"AI visibility monitoring","measured":true,"at":"2026-09-16T01:17:01.001Z","id":"2vp839uwxy","refused":false,"alias_of":null,"grade":"C","score":69,"findings":4,"level":"medium","codes":["PAGE_IS_MOSTLY_CODE","CONTENT_NEEDS_JAVASCRIPT","SITEMAP_NO_LASTMOD","NO_LLMS_TXT"],"reach":100,"read":60,"quote":35,"first_seen":"2026-08-16T03:07:13.283Z","scans":15,"llms":false,"entitymap":false},{"slug":"profound","name":"Profound","domain":"tryprofound.com","cat":"AI visibility monitoring","measured":true,"at":"2026-09-16T02:17:00.935Z","id":"3ti6fi08ox","refused":false,"alias_of":null,"grade":"C","score":66,"findings":2,"level":"medium","codes":["PAGE_IS_MOSTLY_CODE","NO_LLMS_TXT"],"reach":100,"read":47,"quote":39,"first_seen":"2026-08-16T03:07:19.250Z","scans":15,"llms":false,"entitymap":false},{"slug":"scrunch","name":"Scrunch","domain":"scrunch.ai","cat":"AI visibility monitoring","measured":true,"at":"2026-09-16T03:17:00.929Z","id":"luevb24xbi","refused":true,"alias_of":null,"grade":null,"score":null,"findings":0,"level":null,"codes":[],"reach":null,"read":null,"quote":null,"llms":null,"entitymap":null},{"slug":"athena","name":"Athena","domain":"athenahq.ai","cat":"AI visibility monitoring","measured":true,"at":"2026-09-16T04:17:10.948Z","id":"ch9dqyag8n","refused":false,"alias_of":null,"grade":"B","score":79,"findings":2,"level":"low","codes":["MACHINE_CHAIN_HEAVY","STALE_CACHE_SERVED"],"reach":100,"read":73,"quote":56,"first_seen":"2026-09-01T17:28:07.416Z","scans":10,"llms":true,"entitymap":false},{"slug":"rankscale","name":"Rankscale","domain":"rankscale.ai","cat":"AI visibility monitoring","measured":true,"at":"2026-09-16T05:17:10.930Z","id":"4zpqyufnrn","refused":false,"alias_of":null,"grade":"C","score":84,"findings":2,"level":"medium","codes":["PAGE_IS_MOSTLY_CODE","ENTITY_NO_COORDINATES"],"reach":95,"read":67,"quote":85,"first_seen":"2026-09-01T17:29:02.476Z","scans":10,"llms":true,"entitymap":false},{"slug":"goodie","name":"Goodie","domain":"higoodie.com","cat":"AI visibility monitoring","measured":true,"at":"2026-09-16T06:17:14.161Z","id":"vkkpm7l6s2","refused":false,"alias_of":null,"grade":"C","score":80,"findings":3,"level":"medium","codes":["ENTITY_COLLISION","PAGE_IS_MOSTLY_CODE","HSTS_MISSING"],"reach":89,"read":67,"quote":80,"first_seen":"2026-09-01T21:08:20.525Z","scans":10,"llms":true,"entitymap":false},{"slug":"writesonic","name":"Writesonic","domain":"writesonic.com","cat":"Content and GEO","measured":true,"at":"2026-09-16T07:17:14.278Z","id":"uurgxzq7cr","refused":false,"alias_of":null,"grade":"C","score":80,"findings":2,"level":"medium","codes":["PAGE_IS_MOSTLY_CODE","ENTITY_NO_COORDINATES"],"reach":92,"read":67,"quote":76,"first_seen":"2026-08-18T21:14:22.537Z","scans":13,"llms":true,"entitymap":false},{"slug":"surfer","name":"Surfer","domain":"surferseo.com","cat":"Content and GEO","measured":true,"at":"2026-09-16T08:17:14.218Z","id":"33zrfevhr5","refused":false,"alias_of":null,"grade":"C","score":80,"findings":4,"level":"medium","codes":["ROBOTS_NO_SITEMAP","STALE_CACHE_SERVED","NO_LLMS_TXT","ENTITY_NO_COORDINATES"],"reach":92,"read":58,"quote":87,"first_seen":"2026-08-18T21:14:33.776Z","scans":13,"llms":false,"entitymap":false},{"slug":"clearscope","name":"Clearscope","domain":"clearscope.io","cat":"Content and GEO","measured":true,"at":"2026-09-16T09:17:14.526Z","id":"pem67b0j5k","refused":false,"alias_of":null,"grade":"D","score":63,"findings":3,"level":"high","codes":["NO_SITEMAP_FOUND","ROBOTS_NO_SITEMAP","NO_LLMS_TXT"],"reach":83,"read":60,"quote":40,"first_seen":"2026-08-18T22:00:32.227Z","scans":11,"llms":false,"entitymap":false},{"slug":"frase","name":"Frase","domain":"frase.io","cat":"Content and GEO","measured":true,"at":"2026-09-16T10:17:14.217Z","id":"uqkvyi5rxv","refused":false,"alias_of":null,"grade":"C","score":74,"findings":2,"level":"medium","codes":["ROBOTS_RULES_SHADOWED","ENTITY_COLLISION"],"reach":93,"read":60,"quote":63,"first_seen":"2026-08-18T21:14:51.493Z","scans":9,"llms":true,"entitymap":false}],"events":[{"vendor":"BrightEdge","slug":"brightedge","domain":"brightedge.com","at":"2026-09-17T00:17:45.599Z","kind":"appeared","code":"MACHINE_CHAIN_HEAVY","text":"a new finding appeared: the machine files cost more than the page"},{"vendor":"Conductor","slug":"conductor","domain":"conductor.com","at":"2026-09-16T23:17:46.753Z","kind":"appeared","code":"MACHINE_CHAIN_HEAVY","text":"a new finding appeared: the machine files cost more than the page"},{"vendor":"Siteliner","slug":"siteliner","domain":"siteliner.com","at":"2026-09-16T22:17:56.321Z","kind":"appeared","code":"MACHINE_CHAIN_HEAVY","text":"a new finding appeared: the machine files cost more than the page"},{"vendor":"Oncrawl","slug":"oncrawl","domain":"oncrawl.com","at":"2026-09-16T20:17:32.868Z","kind":"appeared","code":"MACHINE_CHAIN_HEAVY","text":"a new finding appeared: the machine files cost more than the page"},{"vendor":"Botify","slug":"botify","domain":"botify.com","at":"2026-09-16T19:17:31.850Z","kind":"appeared","code":"BROKEN_INTERNAL_LINK","text":"a new finding appeared: links on this site point at pages that do not answer"},{"vendor":"Botify","slug":"botify","domain":"botify.com","at":"2026-09-16T19:17:31.850Z","kind":"appeared","code":"MACHINE_CHAIN_HEAVY","text":"a new finding appeared: the machine files cost more than the page"},{"vendor":"Lumar","slug":"lumar","domain":"lumar.io","at":"2026-09-16T18:17:31.703Z","kind":"appeared","code":"MACHINE_CHAIN_HEAVY","text":"a new finding appeared: the machine files cost more than the page"},{"vendor":"Sitebulb","slug":"sitebulb","domain":"sitebulb.com","at":"2026-09-16T17:17:32.378Z","kind":"appeared","code":"MACHINE_CHAIN_HEAVY","text":"a new finding appeared: the machine files cost more than the page"},{"vendor":"Screaming Frog","slug":"screaming-frog","domain":"screamingfrog.co.uk","at":"2026-09-16T16:17:32.925Z","kind":"appeared","code":"MACHINE_CHAIN_HEAVY","text":"a new finding appeared: the machine files cost more than the page"},{"vendor":"Screaming Frog","slug":"screaming-frog","domain":"screamingfrog.co.uk","at":"2026-09-16T16:17:32.925Z","kind":"resolved","code":"STALE_CACHE_SERVED","text":"the stale edge copy cleared"},{"vendor":"Similarweb","slug":"similarweb","domain":"similarweb.com","at":"2026-09-16T15:18:00.370Z","kind":"appeared","code":"MACHINE_CHAIN_HEAVY","text":"a new finding appeared: the machine files cost more than the page"},{"vendor":"SE Ranking","slug":"seranking","domain":"seranking.com","at":"2026-09-16T14:18:00.665Z","kind":"appeared","code":"MACHINE_CHAIN_HEAVY","text":"a new finding appeared: the machine files cost more than the page"},{"vendor":"Moz","slug":"moz","domain":"moz.com","at":"2026-09-16T13:17:59.572Z","kind":"appeared","code":"MACHINE_CHAIN_HEAVY","text":"a new finding appeared: the machine files cost more than the page"},{"vendor":"Semrush","slug":"semrush","domain":"semrush.com","at":"2026-09-16T12:18:00.471Z","kind":"appeared","code":"MACHINE_CHAIN_HEAVY","text":"a new finding appeared: the machine files cost more than the page"},{"vendor":"Ahrefs","slug":"ahrefs","domain":"ahrefs.com","at":"2026-09-16T11:58:25.586Z","kind":"returned","code":"UNIFORM_REFUSAL","text":"a new finding appeared: this origin refused the scanner, so the site was never measured again"},{"vendor":"Ahrefs","slug":"ahrefs","domain":"ahrefs.com","at":"2026-09-16T11:58:25.586Z","kind":"resolved","code":"MACHINE_CHAIN_HEAVY","text":"resolved: the machine files cost more than the page"},{"vendor":"Ahrefs","slug":"ahrefs","domain":"ahrefs.com","at":"2026-09-16T11:58:25.586Z","kind":"resolved","code":"NO_LLMS_TXT","text":"an llms.txt became readable"},{"vendor":"Ahrefs","slug":"ahrefs","domain":"ahrefs.com","at":"2026-09-16T11:58:25.586Z","kind":"resolved","code":"PAGE_IS_MOSTLY_CODE","text":"the homepage no longer read as mostly markup"},{"vendor":"Ahrefs","slug":"ahrefs","domain":"ahrefs.com","at":"2026-09-16T11:58:25.586Z","kind":"resolved","code":"RENDER_RESOURCE_BLOCKED","text":"the blocked render resource cleared"},{"vendor":"Ahrefs","slug":"ahrefs","domain":"ahrefs.com","at":"2026-09-16T11:58:25.586Z","kind":"resolved","code":"ROBOTS_NO_SITEMAP","text":"robots.txt named a sitemap"},{"vendor":"Surfer","slug":"surfer","domain":"surferseo.com","at":"2026-09-16T08:17:14.218Z","kind":"returned","code":"STALE_CACHE_SERVED","text":"a stale edge copy was served again"},{"vendor":"Athena","slug":"athena","domain":"athenahq.ai","at":"2026-09-16T04:17:10.948Z","kind":"appeared","code":"MACHINE_CHAIN_HEAVY","text":"a new finding appeared: the machine files cost more than the page"},{"vendor":"Profound","slug":"profound","domain":"tryprofound.com","at":"2026-09-16T02:17:00.935Z","kind":"score-down","from":77,"to":75,"text":"score moved 77 → 75"},{"vendor":"Seobility","slug":"seobility","domain":"seobility.net","at":"2026-09-15T20:17:01.240Z","kind":"resolved","code":"ANSWER_ENGINE_REFUSED","text":"no answer engine was refused"},{"vendor":"Seobility","slug":"seobility","domain":"seobility.net","at":"2026-09-15T20:17:01.240Z","kind":"score-up","from":74,"to":80,"text":"score moved 74 → 80"},{"vendor":"Botify","slug":"botify","domain":"botify.com","at":"2026-09-15T18:17:03.099Z","kind":"returned","code":"STALE_CACHE_SERVED","text":"a stale edge copy was served again"},{"vendor":"Sitebulb","slug":"sitebulb","domain":"sitebulb.com","at":"2026-09-15T16:17:02.261Z","kind":"score-up","from":83,"to":84,"text":"score moved 83 → 84"},{"vendor":"Similarweb","slug":"similarweb","domain":"similarweb.com","at":"2026-09-15T14:17:44.442Z","kind":"score-up","from":73,"to":74,"text":"score moved 73 → 74"},{"vendor":"SE Ranking","slug":"seranking","domain":"seranking.com","at":"2026-09-15T13:17:44.535Z","kind":"score-up","from":78,"to":79,"text":"score moved 78 → 79"},{"vendor":"Moz","slug":"moz","domain":"moz.com","at":"2026-09-15T12:17:43.828Z","kind":"score-up","from":85,"to":86,"text":"score moved 85 → 86"},{"vendor":"Semrush","slug":"semrush","domain":"semrush.com","at":"2026-09-15T11:17:44.242Z","kind":"resolved","code":"HSTS_PRELOAD_REMOVED","text":"resolved: this domain was removed from the browser hsts preload list"},{"vendor":"Athena","slug":"athena","domain":"athenahq.ai","at":"2026-09-15T03:17:43.984Z","kind":"returned","code":"STALE_CACHE_SERVED","text":"a stale edge copy was served again"},{"vendor":"Profound","slug":"profound","domain":"tryprofound.com","at":"2026-09-15T01:17:44.171Z","kind":"score-up","from":76,"to":77,"text":"score moved 76 → 77"},{"vendor":"Seobility","slug":"seobility","domain":"seobility.net","at":"2026-09-14T20:17:44.140Z","kind":"score-up","from":73,"to":74,"text":"score moved 73 → 74"},{"vendor":"Screaming Frog","slug":"screaming-frog","domain":"screamingfrog.co.uk","at":"2026-09-14T15:17:44.618Z","kind":"returned","code":"STALE_CACHE_SERVED","text":"a stale edge copy was served again"},{"vendor":"Screaming Frog","slug":"screaming-frog","domain":"screamingfrog.co.uk","at":"2026-09-14T15:17:44.618Z","kind":"score-up","from":75,"to":76,"text":"score moved 75 → 76"},{"vendor":"Surfer","slug":"surfer","domain":"surferseo.com","at":"2026-09-14T07:17:44.908Z","kind":"resolved","code":"STALE_CACHE_SERVED","text":"the stale edge copy cleared"},{"vendor":"Conductor","slug":"conductor","domain":"conductor.com","at":"2026-09-13T21:17:44.372Z","kind":"returned","code":"STALE_CACHE_SERVED","text":"a stale edge copy was served again"},{"vendor":"Seobility","slug":"seobility","domain":"seobility.net","at":"2026-09-13T19:17:44.108Z","kind":"score-down","from":75,"to":73,"text":"score moved 75 → 73"},{"vendor":"Screaming Frog","slug":"screaming-frog","domain":"screamingfrog.co.uk","at":"2026-09-13T14:17:44.635Z","kind":"score-down","from":76,"to":75,"text":"score moved 76 → 75"}]}