{"source":"https://crawlcheck.io/registry","retrieved":"2026-10-19T05:35:08.257Z","note":"Rows are derived from scans this service performed. A site is listed because a fetch produced the row, never because anyone submitted it, and no payment moves a shelf.","shelves":[{"code":"llms","name":"llms.txt","means":"The file resolves and its links point at pages that exist on the same host.","count":313},{"code":"agents","name":"agents.md","means":"An /agents.md is served and is not an HTML 404 dressed as one.","count":39},{"code":"mediakit","name":"Media kit","means":"A /.well-known/media-kit.json is served — machine-readable press facts, assets and usage terms.","count":15},{"code":"entity","name":"Entity graph","means":"At least one sameAs was fetched and found pointing back. Reciprocity tested, not declared.","count":335},{"code":"aipolicy","name":"AI access policy","means":"robots.txt names AI crawlers by name rather than leaving them to the wildcard.","count":282},{"code":"agentapi","name":"Agent-callable","means":"An API catalog, MCP server card or agent-skills index is served.","count":73}],"stale_after_days":30,"shelf":null,"total":820,"corpus":820,"population":"corpus counts registry rows: domains with a complete, graded full scan that can be published and have not opted out. It is not the homepage sites_measured, which also counts lighter census readings that get no registry row.","offset":0,"limit":100,"returned":100,"complete":false,"next":"https://crawlcheck.io/api/registry?limit=100&offset=100","cursor":null,"rows":[{"host":"303fence.com","at":"2026-10-14T06:48:04.211Z","grade":"C","score":70,"findings":5,"id":"x7bve01ibz","shelves":["llms","aipolicy"],"stale":false,"verified":false},{"host":"abgoyal.com","at":"2026-10-15T18:28:06.526Z","grade":"C","score":85,"findings":2,"id":"exjmlxlmeo","shelves":[],"stale":false,"verified":false},{"host":"abridge.com","at":"2026-10-15T05:02:04.183Z","grade":"C","score":80,"findings":4,"id":"dvb1bycdst","shelves":["entity","aipolicy"],"stale":false,"verified":false},{"host":"acsoldiers.com","at":"2026-10-01T15:13:35.928Z","grade":"C","score":76,"findings":5,"id":"rforoicjm7","shelves":[],"stale":false,"verified":false},{"host":"activeloop.ai","at":"2026-10-15T05:03:21.430Z","grade":"C","score":77,"findings":3,"id":"lc3jet88a2","shelves":["llms","entity","aipolicy"],"stale":false,"verified":false},{"host":"acxiom.com","at":"2026-10-10T04:32:53.347Z","grade":"C","score":80,"findings":2,"id":"qqzezmf3d5","shelves":["llms","entity"],"stale":false,"verified":false},{"host":"adactio.com","at":"2026-09-01T22:55:55.250Z","grade":"C","score":76,"findings":3,"id":"z30wrxnh73","shelves":["aipolicy"],"stale":true,"verified":false},{"host":"adept.ai","at":"2026-10-18T20:02:37.413Z","grade":"D","score":86,"findings":8,"id":"hw0yp6f068","shelves":[],"stale":false,"verified":false},{"host":"adobe.io","at":"2026-10-04T04:23:01.331Z","grade":"C","score":71,"findings":3,"id":"akpsz7v763","shelves":[],"stale":false,"verified":false},{"host":"adrevdigital.com","at":"2026-10-14T04:48:02.892Z","grade":"B","score":85,"findings":0,"id":"urwqfpoxr9","shelves":["llms"],"stale":false,"verified":false},{"host":"aethelauto.in","at":"2026-10-14T05:03:02.901Z","grade":"A","score":87,"findings":1,"id":"df79sybrpx","shelves":["llms","aipolicy"],"stale":false,"verified":false},{"host":"agentic.ai","at":"2026-08-13T20:02:08.847Z","grade":"B","score":80,"findings":0,"id":"9n3f5c2rwo","shelves":["llms"],"stale":true,"verified":false},{"host":"ahrefs.com","at":"2026-10-18T15:10:35.348Z","grade":"C","score":83,"findings":9,"id":"r2fbjwexyq","shelves":["entity"],"stale":false,"verified":false},{"host":"ai21.com","at":"2026-10-15T05:05:55.315Z","grade":"C","score":81,"findings":3,"id":"6k6lehw0td","shelves":["llms","entity"],"stale":false,"verified":false},{"host":"aiclicks.io","at":"2026-10-14T06:49:55.455Z","grade":"C","score":83,"findings":3,"id":"tv9sz36q8e","shelves":["llms","entity"],"stale":false,"verified":false},{"host":"aicrawlercheck.com","at":"2026-10-13T22:32:41.860Z","grade":"C","score":79,"findings":3,"id":"bugwk3d6ky","shelves":["llms","aipolicy"],"stale":false,"verified":false},{"host":"aidoc.com","at":"2026-10-15T06:02:04.427Z","grade":"C","score":80,"findings":2,"id":"6e9ckzgs66","shelves":["llms","entity"],"stale":false,"verified":false},{"host":"airbnb.com","at":"2026-10-04T04:38:40.011Z","grade":"D","score":69,"findings":4,"id":"7396ndwuhy","shelves":["aipolicy"],"stale":false,"verified":false},{"host":"airbridge.io","at":"2026-10-04T05:10:42.345Z","grade":"C","score":79,"findings":3,"id":"iseult3c3y","shelves":["llms","entity","aipolicy"],"stale":false,"verified":false},{"host":"aircanada.com","at":"2026-10-04T05:08:46.778Z","grade":"C","score":66,"findings":6,"id":"g0xl0vtadf","shelves":[],"stale":false,"verified":false},{"host":"aireserv.com","at":"2026-10-12T08:32:50.908Z","grade":"C","score":80,"findings":4,"id":"5c675go8ne","shelves":["llms","aipolicy"],"stale":false,"verified":false},{"host":"algolia.com","at":"2026-10-19T05:02:37.638Z","grade":"D","score":67,"findings":4,"id":"e0586dksk3","shelves":[],"stale":false,"verified":false},{"host":"allbirds.com","at":"2026-10-04T04:03:08.126Z","grade":"C","score":76,"findings":2,"id":"v09df1ca9s","shelves":["agents","aipolicy"],"stale":false,"verified":false},{"host":"allguardroofing.ie","at":"2026-09-10T06:07:47.892Z","grade":"C","score":72,"findings":4,"id":"hb6bromwn4","shelves":[],"stale":true,"verified":false},{"host":"alli.ai","at":"2026-10-15T06:06:38.981Z","grade":"C","score":68,"findings":4,"id":"9q0pepsu8y","shelves":["entity"],"stale":false,"verified":false},{"host":"allstateroofingaz.com","at":"2026-10-14T20:03:05.107Z","grade":"C","score":85,"findings":3,"id":"8d3fkmeo8b","shelves":["llms","entity"],"stale":false,"verified":false},{"host":"americanarborcare.com","at":"2026-10-16T03:02:29.352Z","grade":"C","score":86,"findings":6,"id":"wjumy5tl6o","shelves":["llms","entity","aipolicy"],"stale":false,"verified":false},{"host":"amtrak.com","at":"2026-10-18T05:01:41.150Z","grade":"B","score":73,"findings":2,"id":"ezwljwg8b2","shelves":[],"stale":false,"verified":false},{"host":"angi.com","at":"2026-10-04T05:09:28.175Z","grade":"C","score":66,"findings":4,"id":"bupxzfjvbj","shelves":["aipolicy"],"stale":false,"verified":false},{"host":"anthropic.com","at":"2026-10-15T06:02:24.082Z","grade":"C","score":80,"findings":3,"id":"0k3qfdtdfx","shelves":[],"stale":false,"verified":false},{"host":"anyscale.com","at":"2026-10-15T06:04:13.572Z","grade":"C","score":83,"findings":4,"id":"bb9bg25kg5","shelves":["entity"],"stale":false,"verified":false},{"host":"anyword.com","at":"2026-10-15T06:09:32.173Z","grade":"B","score":77,"findings":3,"id":"p8q79h7pb1","shelves":[],"stale":false,"verified":false},{"host":"apache.org","at":"2026-10-04T04:22:26.799Z","grade":"D","score":72,"findings":4,"id":"t2jqmejxkf","shelves":[],"stale":false,"verified":false},{"host":"apify.com","at":"2026-10-15T06:10:03.506Z","grade":"C","score":83,"findings":5,"id":"zwcptq4sfj","shelves":["llms","agents","entity","agentapi"],"stale":false,"verified":false},{"host":"app.callrail.com","at":"2026-10-11T21:48:38.528Z","grade":"D","score":69,"findings":6,"id":"23q3n212dp","shelves":[],"stale":false,"verified":false},{"host":"app.neilpatel.com","at":"2026-10-08T15:35:53.595Z","grade":"C","score":71,"findings":3,"id":"z2or5n6xt8","shelves":["llms","aipolicy"],"stale":false,"verified":false},{"host":"appen.com","at":"2026-10-15T06:14:36.469Z","grade":"C","score":85,"findings":6,"id":"wg51sbus48","shelves":["entity"],"stale":false,"verified":false},{"host":"apple.com","at":"2026-10-04T04:18:39.836Z","grade":"C","score":83,"findings":3,"id":"ilcm7g42qb","shelves":["entity","aipolicy"],"stale":false,"verified":false},{"host":"appsflyer.com","at":"2026-10-04T04:32:30.361Z","grade":"C","score":75,"findings":6,"id":"azmsv5647g","shelves":["entity"],"stale":false,"verified":false},{"host":"arborcaretreesolutions.com","at":"2026-10-14T20:09:20.529Z","grade":"A","score":84,"findings":4,"id":"hhzq2vtxvk","shelves":["llms","entity","aipolicy"],"stale":false,"verified":false},{"host":"arborday.org","at":"2026-10-13T21:02:15.051Z","grade":"C","score":81,"findings":5,"id":"dc3jsvhqsw","shelves":[],"stale":false,"verified":false},{"host":"arborscapes.net","at":"2026-10-14T06:33:03.866Z","grade":"C","score":70,"findings":4,"id":"jeyq068pig","shelves":["entity"],"stale":false,"verified":false},{"host":"arbortectree.com","at":"2026-09-01T17:54:21.943Z","grade":"D","score":61,"findings":6,"id":"a4cux2vfq7","shelves":[],"stale":true,"verified":false},{"host":"archive.org","at":"2026-10-04T04:23:01.441Z","grade":"C","score":74,"findings":3,"id":"iokswcy13l","shelves":[],"stale":false,"verified":false},{"host":"argilla.io","at":"2026-10-15T06:18:08.705Z","grade":"C","score":77,"findings":4,"id":"t7xsdvx6zn","shelves":[],"stale":false,"verified":false},{"host":"arize.com","at":"2026-10-15T06:18:33.573Z","grade":"C","score":87,"findings":2,"id":"skp7gm3z1g","shelves":["llms","entity"],"stale":false,"verified":false},{"host":"arngren.net","at":"2026-10-18T03:20:57.756Z","grade":"C","score":71,"findings":4,"id":"5vmyrlamf1","shelves":[],"stale":false,"verified":false},{"host":"arnoldsmithlaw.com","at":"2026-10-14T20:33:03.272Z","grade":"B","score":82,"findings":7,"id":"snq1aebrme","shelves":["aipolicy"],"stale":false,"verified":false},{"host":"articleforge.com","at":"2026-10-15T06:19:26.697Z","grade":"D","score":58,"findings":5,"id":"6is6b9np9s","shelves":[],"stale":false,"verified":false},{"host":"arvadaco.gov","at":"2026-08-26T05:10:50.602Z","grade":"C","score":75,"findings":3,"id":"k1c0fryl6w","shelves":["aipolicy"],"stale":true,"verified":false},{"host":"arvadatreeservicellc.com","at":"2026-10-17T13:23:07.445Z","grade":"A","score":97,"findings":0,"id":"vwgjd2bfza","shelves":["llms","agents","mediakit","entity","aipolicy","agentapi"],"stale":false,"verified":true},{"host":"arxiv.org","at":"2026-10-04T04:34:59.275Z","grade":"D","score":73,"findings":5,"id":"6geu14smuj","shelves":["aipolicy"],"stale":false,"verified":false},{"host":"assemblyai.com","at":"2026-10-15T07:02:25.218Z","grade":"B","score":76,"findings":3,"id":"rr88dbsfsl","shelves":["llms"],"stale":false,"verified":false},{"host":"astro.build","at":"2026-10-04T05:22:00.927Z","grade":"C","score":77,"findings":3,"id":"58b95exefm","shelves":[],"stale":false,"verified":false},{"host":"astronomy.com","at":"2026-09-18T13:23:02.052Z","grade":"C","score":83,"findings":4,"id":"oae2zup4hg","shelves":["entity","aipolicy"],"stale":true,"verified":false},{"host":"athenahq.ai","at":"2026-10-15T03:02:12.868Z","grade":"B","score":79,"findings":4,"id":"nt5hb3p3w4","shelves":["llms"],"stale":false,"verified":false},{"host":"atomwise.com","at":"2026-10-15T07:03:41.916Z","grade":"C","score":67,"findings":3,"id":"8nvslih5an","shelves":[],"stale":false,"verified":false},{"host":"augmedix.com","at":"2026-10-15T07:08:51.530Z","grade":"B","score":85,"findings":4,"id":"fl52zrv39u","shelves":[],"stale":false,"verified":false},{"host":"augmentcode.com","at":"2026-10-15T08:02:23.309Z","grade":"C","score":77,"findings":3,"id":"gcar56f6wk","shelves":["llms","entity","agentapi"],"stale":false,"verified":false},{"host":"autoeurope.com","at":"2026-10-11T18:21:52.227Z","grade":"B","score":72,"findings":3,"id":"h0qnufrjmz","shelves":["entity"],"stale":false,"verified":false},{"host":"autogpt.net","at":"2026-10-15T08:03:08.228Z","grade":"A","score":84,"findings":2,"id":"rhie3c1vfb","shelves":["aipolicy"],"stale":false,"verified":false},{"host":"awin1.com","at":"2026-10-04T05:02:30.158Z","grade":"D","score":72,"findings":6,"id":"30cdd6ab7o","shelves":["aipolicy"],"stale":false,"verified":false},{"host":"backlinko.com","at":"2026-10-04T06:17:46.723Z","grade":"C","score":81,"findings":4,"id":"09za0bv00z","shelves":["entity","aipolicy"],"stale":false,"verified":false},{"host":"banana.dev","at":"2026-10-15T08:05:23.317Z","grade":"D","score":71,"findings":7,"id":"l8abrkmt8x","shelves":[],"stale":false,"verified":false},{"host":"baseten.co","at":"2026-10-15T08:05:57.386Z","grade":"C","score":77,"findings":7,"id":"w0kk5qx3z1","shelves":["llms"],"stale":false,"verified":false},{"host":"bbc.co.uk","at":"2026-10-13T08:03:19.761Z","grade":"C","score":70,"findings":5,"id":"mfres02l1v","shelves":["aipolicy"],"stale":false,"verified":false},{"host":"bbc.com","at":"2026-10-04T04:40:17.671Z","grade":"C","score":68,"findings":4,"id":"49fu991f7z","shelves":["entity","aipolicy"],"stale":false,"verified":false},{"host":"behinmehr.com","at":"2026-10-14T06:04:28.007Z","grade":"C","score":70,"findings":4,"id":"fg4y3iqasx","shelves":["llms"],"stale":false,"verified":false},{"host":"benjaminfranklinplumbing.com","at":"2026-10-14T07:03:05.800Z","grade":"C","score":76,"findings":3,"id":"s6jjojegxt","shelves":["llms","aipolicy"],"stale":false,"verified":false},{"host":"bentoml.com","at":"2026-10-15T09:02:37.325Z","grade":"C","score":72,"findings":6,"id":"wvracsjfza","shelves":[],"stale":false,"verified":false},{"host":"bestbuy.com","at":"2026-10-18T05:12:38.200Z","grade":"C","score":67,"findings":5,"id":"6gts0h2wg0","shelves":["aipolicy"],"stale":false,"verified":false},{"host":"bestseopodcast.com","at":"2026-10-14T17:03:04.505Z","grade":"B","score":76,"findings":3,"id":"9f9hnvllm0","shelves":["entity"],"stale":false,"verified":false},{"host":"beyondbilingual.net","at":"2026-10-17T06:02:10.661Z","grade":"B","score":79,"findings":3,"id":"h2opmp3d1y","shelves":[],"stale":false,"verified":false},{"host":"billhartzer.com","at":"2026-08-20T03:02:31.045Z","grade":"C","score":85,"findings":2,"id":"n3ftfdqtdv","shelves":["llms","entity"],"stale":true,"verified":false},{"host":"bitly.com","at":"2026-10-04T04:55:19.959Z","grade":"C","score":80,"findings":3,"id":"qpxzaeqqiw","shelves":["entity","aipolicy"],"stale":false,"verified":false},{"host":"bloomreach.com","at":"2026-10-15T09:03:49.723Z","grade":"C","score":78,"findings":7,"id":"ym7j31h49l","shelves":["entity"],"stale":false,"verified":false},{"host":"bls.gov","at":"2026-10-18T05:04:40.772Z","grade":"B","score":78,"findings":4,"id":"mxo84rv8z2","shelves":["aipolicy"],"stale":false,"verified":false},{"host":"bolt.new","at":"2026-10-15T09:08:18.252Z","grade":"C","score":81,"findings":2,"id":"0mqde7kdqz","shelves":["entity"],"stale":false,"verified":false},{"host":"booking.com","at":"2026-09-01T18:17:26.482Z","grade":"C","score":67,"findings":2,"id":"v5rgw8pfj6","shelves":["aipolicy"],"stale":true,"verified":false},{"host":"botify.com","at":"2026-10-18T17:10:51.979Z","grade":"C","score":85,"findings":5,"id":"4z7go5u7mr","shelves":["entity"],"stale":false,"verified":false},{"host":"botview.app","at":"2026-10-14T07:33:03.591Z","grade":"B","score":82,"findings":3,"id":"grmkaqkmm0","shelves":["llms","aipolicy"],"stale":false,"verified":false},{"host":"braintrust.dev","at":"2026-10-02T13:21:07.229Z","grade":"C","score":83,"findings":2,"id":"6obgbksg1k","shelves":["llms","entity"],"stale":false,"verified":false},{"host":"brand24.com","at":"2026-10-15T10:02:23.943Z","grade":"C","score":77,"findings":5,"id":"xbui0ck7cx","shelves":["llms","entity"],"stale":false,"verified":false},{"host":"brandlight.ai","at":"2026-10-15T10:04:17.447Z","grade":"B","score":80,"findings":2,"id":"llgfpt08n5","shelves":["llms","entity"],"stale":false,"verified":false},{"host":"brandwatch.com","at":"2026-10-15T10:08:09.061Z","grade":"C","score":81,"findings":5,"id":"v138cb67ey","shelves":["entity","aipolicy"],"stale":false,"verified":false},{"host":"brightdata.com","at":"2026-10-15T10:10:58.831Z","grade":"C","score":90,"findings":4,"id":"qc5oi3slcq","shelves":["agents","aipolicy","agentapi"],"stale":false,"verified":false},{"host":"brightedge.com","at":"2026-10-18T19:02:38.419Z","grade":"C","score":72,"findings":1,"id":"wd3di329p4","shelves":["llms"],"stale":false,"verified":false},{"host":"brightonco.gov","at":"2026-10-18T05:17:15.164Z","grade":"C","score":64,"findings":6,"id":"ps89mej4o6","shelves":["aipolicy"],"stale":false,"verified":false},{"host":"brokervsadvisor.com","at":"2026-10-09T16:33:03.234Z","grade":"A","score":84,"findings":1,"id":"gijxza6oej","shelves":["llms","aipolicy","agentapi"],"stale":false,"verified":false},{"host":"browserbase.com","at":"2026-10-15T11:02:23.859Z","grade":"C","score":73,"findings":4,"id":"zmb83etaxh","shelves":["llms"],"stale":false,"verified":false},{"host":"browserless.io","at":"2026-10-15T11:03:34.073Z","grade":"C","score":80,"findings":3,"id":"satnsgnfpm","shelves":["entity","agentapi"],"stale":false,"verified":false},{"host":"builder.io","at":"2026-10-15T11:07:28.262Z","grade":"C","score":79,"findings":3,"id":"jjugjge9pn","shelves":["llms"],"stale":false,"verified":false},{"host":"businessinsider.com","at":"2026-10-04T06:17:46.390Z","grade":"C","score":86,"findings":4,"id":"2ofnxfc8wn","shelves":["entity","aipolicy"],"stale":false,"verified":false},{"host":"businessvaluenow.com","at":"2026-10-09T16:33:57.835Z","grade":"A","score":83,"findings":1,"id":"z1ajvcxuzk","shelves":["llms","aipolicy","agentapi"],"stale":false,"verified":false},{"host":"byword.ai","at":"2026-10-15T11:08:00.448Z","grade":"C","score":80,"findings":3,"id":"gkyznfnt8h","shelves":["llms","aipolicy"],"stale":false,"verified":false},{"host":"calibreapp.com","at":"2026-10-02T15:18:05.756Z","grade":"C","score":79,"findings":3,"id":"790lv4z1eh","shelves":["llms","agents","entity","aipolicy"],"stale":false,"verified":false},{"host":"callrail.com","at":"2026-10-04T04:50:46.216Z","grade":"C","score":77,"findings":5,"id":"4viyiqnlnq","shelves":["aipolicy"],"stale":false,"verified":false},{"host":"calltower.com","at":"2026-10-11T18:23:53.906Z","grade":"D","score":75,"findings":4,"id":"0bew813j2k","shelves":["llms"],"stale":false,"verified":false},{"host":"calypsoai.com","at":"2026-10-15T12:02:22.708Z","grade":"D","score":62,"findings":7,"id":"4euct2w6gf","shelves":[],"stale":false,"verified":false},{"host":"canonical.com","at":"2026-10-04T04:49:17.873Z","grade":"C","score":78,"findings":3,"id":"e36rp0zxcl","shelves":["llms","entity","aipolicy"],"stale":false,"verified":false}]}