{"title":"CrawlCheck evidence and methods notes — 30 October 2026","version":1,"author":"CrawlCheck / VSNARY","prepared":"2026-10-30","evidence_types":["synthetic fixtures","first-party HTTP readings","archived manual notes reanalysis","dated primary-document review","secondary review of an earlier published case"],"notice":"This bundle does not contain a completed thirty-day field study or controlled AI ranking intervention. Raw external documentation is not redistributed. No new model queries were performed for the archived-answer analyses.","reuse":"Original tables and aggregate readings may be reused with attribution to CrawlCheck and this edition; third-party documentation retains its own terms.","files":{"published-case-review.json":{"type":"secondary review of the operator-published article, not a new model evaluation","source":{"id":"crawlcheck-four-models","url":"https://crawlcheck.io/blog/what-four-ai-models-told-customers-about-our-three-businesses","final_url":"https://crawlcheck.io/blog/what-four-ai-models-told-customers-about-our-three-businesses","observed_at":"2026-10-30T19:56:39.371123+00:00","status":200,"content_type":"text/html; charset=utf-8","bytes":62414,"sha256":"28ffdc6cdd8427ad168a2b79a8790492f58f0a1a3cd6de55980fc2f6effe6be8"},"original_observation_date":"2026-10-09","review_date":"2026-10-30","owned_businesses":3,"conditions":4,"search_enabled_conditions":1,"limitations":["No raw model transcript independently reverified in this release","Owned convenience sample","Search-enabled and no-search conditions differ","No general population error rate inferred"]},"answer-observations.json":[{"date":"2026-10-03","record_sha256":"84e5152df804d04470662d9e326da3416fec0172879507a7c46b418a0bfd53df","record_path":"history/answers/2026-10-03.json","source_type":"archived manual observation notes","signed_in":{"google_ai_mode":true,"perplexity":false},"personalization_warning":false,"rows":[{"question":"how to test robots.txt for GPTBot","guide":"how-to-test-robots-txt-for-gptbot-and-claudebot","ai_mode":{"crawlcheck_cited":false,"source_labels":["ahrefs (youtube)","seocrawl.ai","asva ai","keywords everywhere","scrappey"]},"perplexity":{"crawlcheck_cited":false,"source_labels":["seositecheckup","technicalseo.com","tamethebots","aeoengine.ai"]}},{"question":"does Cloudflare block AI crawlers even if robots.txt allows them","guide":"cloudflare-ai-crawl-control-vs-crawlcheck","ai_mode":{"crawlcheck_cited":false,"source_labels":["nytro seo","cloudflare blog","playwire","cloudflare community","reddit r/CloudFlare"]},"perplexity":{"crawlcheck_cited":false,"source_labels":["developers.cloudflare.com","cloudflare.com"]}},{"question":"how to test color contrast on hover and focus states","guide":"how-to-test-color-contrast-in-hover-and-focus-states","ai_mode":{"crawlcheck_cited":false,"source_labels":["webaim","microsoft learn","youtube (stefany newman)","accessible web","linkedin","ux stack exchange"]},"perplexity":{"crawlcheck_cited":false,"source_labels":["w3.org","webaim"]}},{"question":"how to find other websites using my brand name","guide":"how-to-check-who-else-uses-your-brand-name-online","ai_mode":{"crawlcheck_cited":false,"source_labels":["spiceworks community","whoisxml api","valimail","namecheck","reddit r/SEO"]},"perplexity":{"crawlcheck_cited":false,"source_labels":["redpoints","novagraaf","uspto"]}},{"question":"how to find broken internal links and anchor links","guide":"how-to-audit-internal-links-and-dead-anchors","ai_mode":{"crawlcheck_cited":false,"source_labels":["screaming frog","youtube (josiah roche, scott redgate)","ahrefs","semrush","quora","valve + meter"]},"perplexity":{"crawlcheck_cited":false,"source_labels":["luckyorange","screamingfrog","sitebulb","linkstorm"]}},{"question":"how to move DMARC from p=none to p=quarantine","guide":"how-to-move-dmarc-from-none-to-quarantine-without-losing-mail","ai_mode":{"crawlcheck_cited":false,"source_labels":["powerdmarc","valimail","cloudflare community","sh.consulting"]},"perplexity":{"crawlcheck_cited":false,"source_labels":["learn.microsoft.com","dmarcreport"]}},{"question":"what is CrawlCheck","guide":null,"ai_mode":{"crawlcheck_cited":false,"source_labels":["chrome web store (CrawlCheck extension, ours)","crawlercheck.com (different product)"]},"perplexity":{"crawlcheck_cited":true,"source_labels":["crawlcheck.io"]}}]},{"date":"2026-10-05","record_sha256":"8a0cdc68433aa015f7603267732a02d43913ba53f4c33b4e871aba739056adfe","record_path":"history/answers/2026-10-05.json","source_type":"archived manual observation notes","signed_in":{"google_ai_mode":true,"perplexity":true},"personalization_warning":true,"rows":[{"question":"how to test robots.txt for GPTBot","guide":"how-to-test-robots-txt-for-gptbot-and-claudebot","ai_mode":{"crawlcheck_cited":false,"source_labels":["llm pulse","seocrawl.ai","jasonburns.co.uk","asva ai","keywords everywhere","youtube (ahrefs, glasp)","openai developers","genrank.io","fennecseo.app","llmoptimisation.fr"]},"perplexity":{"crawlcheck_cited":false,"source_labels":["technicalseo.com","tamethebots","llmpulse.ai","aeoengine.ai","brandarmor.ai","seositecheckup","quattr","reddit","seranking","aiclicks.io"]}},{"question":"does Cloudflare block AI crawlers even if robots.txt allows them","guide":"cloudflare-ai-crawl-control-vs-crawlcheck","ai_mode":{"crawlcheck_cited":false,"source_labels":["nytro seo","idx.inc","cloudflare developer docs","reddit r/CloudFlare","playwire","no hacks","llmrefs","metricspot"]},"perplexity":{"crawlcheck_cited":false,"source_labels":["developers.cloudflare.com (+3, list not readable in shared-session view)"]}},{"question":"how to test color contrast on hover and focus states","guide":"how-to-test-color-contrast-in-hover-and-focus-states","ai_mode":{"crawlcheck_cited":false,"source_labels":[]},"perplexity":{"crawlcheck_cited":false,"source_labels":["w3.org (+1; 25 sources, list partly rendered)"]}},{"question":"how to find other websites using my brand name","guide":"how-to-check-who-else-uses-your-brand-name-online","ai_mode":{"crawlcheck_cited":false,"source_labels":[]},"perplexity":{"crawlcheck_cited":false,"source_labels":["help.ahrefs.com","community.spiceworks.com","uspto"]}},{"question":"how to find broken internal links and anchor links","guide":"how-to-audit-internal-links-and-dead-anchors","ai_mode":{"crawlcheck_cited":false,"source_labels":["screaming frog","youtube (sreenidhi rajakrishnan, josiah roche, marketing o'clock, she knows seo)","ahrefs","semrush","linkedin","luckyorange","hosting.com","valve + meter","finsweet","kathyjacobs.com","natehoffelder.com","elementor","bestseo.sg","scalenut"]},"perplexity":{"crawlcheck_cited":false,"source_labels":["screamingfrog","luckyorange"]}},{"question":"how to move DMARC from p=none to p=quarantine","guide":"how-to-move-dmarc-from-none-to-quarantine-without-losing-mail","ai_mode":{"crawlcheck_cited":false,"source_labels":["powerdmarc (youtube)","sh.consulting","reddit r/cybersecurity","ssd nodes","cloudflare community","valimail","simpledmarc","youtube (easydmarc)"]},"perplexity":{"crawlcheck_cited":false,"source_labels":["learn.microsoft.com","dmarcreport"]}},{"question":"what is CrawlCheck","guide":null,"ai_mode":{"crawlcheck_cited":false,"source_labels":["chrome web store (CrawlCheck extension, ours)","crawlercheck.com (different product)"]},"perplexity":{"crawlcheck_cited":true,"source_labels":["crawlcheck.io (+1)"]}}]},{"date":"2026-10-29","record_sha256":"733b0e4f1f14582a2e66e2ad771846bd1c89d01470a8e9c0d412cd76493f243d","record_path":"history/answers/2026-10-29.json","source_type":"archived manual observation notes","signed_in":{"google_ai_mode":true,"perplexity":true},"personalization_warning":true,"rows":[{"question":"how to test robots.txt for GPTBot","guide":"how-to-test-robots-txt-for-gptbot-and-claudebot","ai_mode":{"crawlcheck_cited":false,"source_labels":["llm pulse","holp.ai","context.dev","robotstxt.com","axiolo","asva ai","youtube (ahrefs)","reddit r/AI_SearchOptimization"]},"perplexity":{"crawlcheck_cited":true,"source_labels":["crawlcheck.io (lead citation, 3 inline)","patrickstox","context.dev","developers.openai","dev.to","centinelanalytica","sona","quanticdata","deepseoanalysis","aiclicks","proxies.sx","github","aischemagen","soar","searchatlas","mencoro","reddit","windmillstrategy"]}},{"question":"does Cloudflare block AI crawlers even if robots.txt allows them","guide":"cloudflare-ai-crawl-control-vs-crawlcheck","ai_mode":{"crawlcheck_cited":false,"source_labels":["nytro seo","idx.inc","cloudflare developer docs","reddit r/CloudFlare","cloudflare community","flatdot marketing"]},"perplexity":{"crawlcheck_cited":false,"source_labels":["developers.cloudflare.com (x7)","cloudflare.com","blog.cloudflare.com (x2)"]}},{"question":"how to test color contrast on hover and focus states","guide":"how-to-test-color-contrast-in-hover-and-focus-states","ai_mode":{"crawlcheck_cited":false,"source_labels":["webaim","microsoft learn","youtube (stefany newman, accessible web)","level access","w3schools","sponsored: audioeye"]},"perplexity":{"crawlcheck_cited":true,"source_labels":["webaim","learn.microsoft","makethingsaccessible","w3.org (x4)","crawlcheck.io (sources list, not inline)","chrome web store (A11y State Contrast Checker)","accessibility.(guide)"]}},{"question":"how to find other websites using my brand name","guide":"how-to-check-who-else-uses-your-brand-name-online","ai_mode":{"crawlcheck_cited":true,"source_labels":["youtube (cybersudo)","lexsynergy","crawlcheck.io (inline, manual domain sweeps across TLDs)","foresiet","brand guarde","brandshield","spiceworks community","sponsored: spyfu"]},"perplexity":{"crawlcheck_cited":false,"source_labels":["clickrank","landrypllc","novagraaf","lexsynergy","brandguarde","uspto (x2)","katiecharlestonlaw","atom","icoda"]}},{"question":"how to find broken internal links and anchor links","guide":"how-to-audit-internal-links-and-dead-anchors","ai_mode":{"crawlcheck_cited":false,"source_labels":["screaming frog","youtube (josiah roche, computersluggish)","seoclarity","ahrefs","sponsored: dubbot"]},"perplexity":{"crawlcheck_cited":false,"source_labels":["screamingfrog (x8)","ilanshabad","sitebulb","luckyorange (x2)","finsweet","patrickstox","seoclarity","brokenlinkcheck","semrush","reddit r/webdev","webseosg","aubreyyung"]}},{"question":"how to move DMARC from p=none to p=quarantine","guide":"how-to-move-dmarc-from-none-to-quarantine-without-losing-mail","ai_mode":{"crawlcheck_cited":false,"source_labels":["youtube (powerdmarc)","reddit r/cybersecurity","ssd nodes","valimail","sh.consulting","sponsored: powerdmarc"]},"perplexity":{"crawlcheck_cited":false,"source_labels":["learn.microsoft","support.google","generate.folderly","sitelemetry","dmarcreport","knowledgebase.mxtoolbox","reddit","mailtester (x2)","ssdnodes"]}},{"question":"what is CrawlCheck","guide":null,"ai_mode":{"crawlcheck_cited":true,"source_labels":["g2","chrome web store (CrawlCheck extension, ours)","crawlcheck.io","fazier"]},"perplexity":{"crawlcheck_cited":true,"source_labels":["crawlcheck.io (homepage + press page)","cran.r-project","portswigger","browserstack","zenrows","woorank","reddit","evendigit","webmasters.stackexchange"]}},{"question":"Google-Extended and Applebot-Extended: opt out of AI training and stay in search","guide":"google-extended-and-applebot-extended-opt-out-of-ai-training-keep-search","ai_mode":{"crawlcheck_cited":false,"source_labels":["search engine journal","coywolf.com","apple support","ai crawler check (aicrawlercheck)","search engine watch"]},"perplexity":{"crawlcheck_cited":false,"source_labels":["developers.google (x2)","apple (x2)","promptwatch","ppc.land","google community","cultofmac","blog.cloudflare","agilitycms","reddit","dev.to (x2)","crawlbeacon","brightsideofnews","google (grounding docs)","cloud.google","robotstxtlab","isitready","aischemagen","aicrawlercheck","noirtrack","robotstxtstudio","thatware","mariehaynes"]}},{"question":"Website crawlers in 2026: who crawls your site, how to verify them, and which tool to check with","guide":"website-crawlers-2026-who-crawls-your-site-and-how-to-check","ai_mode":{"crawlcheck_cited":false,"source_labels":["digital applied","exa","webyes","google for developers","xseek","ai crawler check","cloudflare","ivris tech","sponsored: gofishdigital, dataforseo"]},"perplexity":{"crawlcheck_cited":false,"source_labels":["swetrix","developers.cloudflare (x2)","screamingfrog","developers.google","lumengeo","empact","seo-stuff","nohacks","promptcloud","reddit","neilpatel","prerenderbuddy","capsicummediaworks","cloudflare (x3)","workspace.google","aisearch.similarweb","apis.io","organikpi","shattered","centinelanalytica","linkedin","mencoro"]}},{"question":"How to tell if an SEO fix worked: compare fixed and untouched pages","guide":"how-to-tell-if-a-website-fix-moved-traffic-fixed-vs-untouched-pages","ai_mode":{"crawlcheck_cited":false,"source_labels":["aio copilot","seomatic","youtube (semrush, the seo pub)","complete gurus","reddit r/SEO","sponsored: █████████"]},"perplexity":{"crawlcheck_cited":false,"source_labels":["searchpilot (x12)","cogny","9clouds","nytroseo","searchatlas","lovarank","seoraft","linkedin","seositecheckup","img courses","(part of list truncated in page text; DOM check found no crawlcheck link)"]}},{"question":"Crawl budget: what it is, and how to measure what a crawler reads","guide":"crawl-budget-how-much-of-your-site-a-crawler-reads-in-50-fetches","ai_mode":{"crawlcheck_cited":false,"source_labels":["google for developers","firecrawl","nytro seo","youtube (google search central, nerd up)","crawl.sarv.com"]},"perplexity":{"crawlcheck_cited":false,"source_labels":["developers.google (x11)","patrickstox","search.google","google support","firecrawl","yoast","lumar","nytroseo","conductor","captaindns","gr0","semji","seosly","rightblogger","webmasterworld","entail"]}},{"question":"How to verify Amazonbot and Amzn-SearchBot: check the IP against Amazon's list","guide":"how-to-verify-amazonbot-and-amzn-searchbot","ai_mode":{"crawlcheck_cited":false,"source_labels":["centinel analytica","mencoro","amazon.com / amazon developer","cloud-ip-ranges.com"]},"perplexity":{"crawlcheck_cited":true,"source_labels":["developer.amazon (x2)","sona","aibotschecker","ipscanner","centinelanalytica","crawlcheck.io (bot directory page /amazonbot, not the guide; sources list only)","unsourced","log-hero","tryprefer"]}},{"question":"How to check if your website supports HTTP/3","guide":"http-3-how-to-check-whether-your-site-offers-it","ai_mode":{"crawlcheck_cited":false,"source_labels":["plesk","http3check (litespeed)","speedvitals","testmu ai","cloudflare blog","stack overflow","youtube (contabo, litespeed)"]},"perplexity":{"crawlcheck_cited":false,"source_labels":["plesk","cloudflare blog","speedvitals","http3check","cloudflare-quic","stackoverflow","superuser","http3checker","caddy community","browserleaks"]}},{"question":"How to get your local business recommended by AI assistants","guide":"how-to-get-your-local-business-recommended-by-ai-assistants","ai_mode":{"crawlcheck_cited":false,"source_labels":["reddit r/DigitalMarketing","youtube (pushleads, pluspoint, paving marketers)","seo inc","consultwithkyle.com","search engine journal","sponsored: stan ventures, danmatt media"]},"perplexity":{"crawlcheck_cited":false,"source_labels":["durable","seoinc","developers.google (x4)","developers.openai","google support","pluspoint","lighthouselocal","locallister","medalisthq","votrixai","seotal","foundlocalgeo","academy.brightlocal","openai (x2)","scalevise","searchenginegeeks","ronincomms","optimalleap","basedrep","difusionseo","earlyseo"]}},{"question":"How an AI agent should verify a website before reading, citing or paying","guide":"how-an-ai-agent-verifies-a-website-before-it-acts","ai_mode":{"crawlcheck_cited":false,"source_labels":["search engine journal","medium (saran kumar s.)","akamai","stanford university"]},"perplexity":{"crawlcheck_cited":true,"source_labels":["crawlcheck.io (lead citation, ~7 inline, named 'CrawlCheck's guide')","doppel","semrush","zspace","verygoodsecurity","techpolicy","edikka","experian","brightedge","salespeak"]}},{"question":"What is the agentic web? What it means for your website","guide":"what-the-agentic-web-means-for-a-site-owner","ai_mode":{"crawlcheck_cited":false,"source_labels":["richard macmanus","custom ai studio","quora","indusmart.digital","techtio","linkedin (thomas ross)","olostep","sponsored: sinequa, brightspot"]},"perplexity":{"crawlcheck_cited":false,"source_labels":["mitsloan.mit","fibr","ibm","developers.google (x2)","developers.openai","cloudflare (x6)","developers.cloudflare","aws","cloud.google","databricks","redhat","salesforce","wikipedia","reddit","openai help","searchenginejournal","scalevise (x2)","dev.to"]}},{"question":"How to check an MCP server is safe before an AI agent connects to it","guide":"how-to-check-an-mcp-server-before-an-agent-connects","ai_mode":{"crawlcheck_cited":false,"source_labels":["youtube (aws events, thomas roccia)","phrasly.ai","deimoscore.com","github","reddit r/mcp","workos","techrepublic","microsoft community hub","sponsored: wiz, snyk"]},"perplexity":{"crawlcheck_cited":false,"source_labels":["cheatsheetseries.owasp","dev.to (x2)","workos (x2)","techcommunity.microsoft","fruggr","oxylabs","iloveaiagent","mcpplaygroundonline"]}},{"question":"Can an AI agent use your contact form? A two-minute test","guide":"can-an-ai-agent-use-your-contact-form","ai_mode":{"crawlcheck_cited":true,"source_labels":["crawlcheck.io (lead source, 7 inline, 'based on the CrawlCheck Guide')","pravinkumar.co"]},"perplexity":{"crawlcheck_cited":false,"source_labels":["webinvolved","agentiqa","reddit","linkedin (cube forms)","qacraft","cocopilot","testiva","wdcstechnology","globalcx","help.dmchamp"]}},{"question":"AI website audit tools compared: Semrush, Ahrefs, Screaming Frog, Sitebulb and CrawlCheck","guide":"ai-website-audit-tools-compared-semrush-ahrefs-screaming-frog-sitebulb-crawlcheck","ai_mode":{"crawlcheck_cited":false,"source_labels":["webenture","utilityseo.com","zapier","youtube (geo mastery with adam, promoseo)","pagelensai","closefa.st","sponsored: gofishdigital, pixis"]},"perplexity":{"crawlcheck_cited":true,"source_labels":["semrush","ahrefs (x9)","screamingfrog","sitebulb","crawlcheck.io (guides hub, best-ai-visibility-tools, seo-audit-tools, glossary agentic-web, about page, vendors dataset; ~4 inline)","searchatlas","vezadigital","honeyb","onelittleweb","wizgrowth","ighenatt","discoveredlabs"]}},{"question":"Do AI crawlers and agents respect robots.txt?","guide":"do-ai-crawlers-respect-robots-txt","ai_mode":{"crawlcheck_cited":false,"source_labels":["searchengineworld","dataimpulse","alien.club","dev community","reddit r/neocities","cloudflare docs","sponsored: cheq ai"]},"perplexity":{"crawlcheck_cited":false,"source_labels":["paulcalvano","developers.openai","arxiv","better-robots","searchengineworld","botrank","perplexity help","bankinfosecurity","robotstxtlab (x2)","aivisibilitycheck","dataimpulse","reddit","reidify","alien","amicited","news.ycombinator","searchenginejournal","inc","linkedin","pritamr.substack","elementera","openai community","realmacsoftware"]}},{"question":"Do AI crawlers render JavaScript? How to check what they actually see","guide":"do-ai-crawlers-render-javascript","ai_mode":{"crawlcheck_cited":false,"source_labels":["maslonlabs.com","braindo","contentmonk","linkedin (harry sanders)","youtube (whitespark)","searchviu","geotoolbox.ai","omnibound","sponsored: gofishdigital, ayzeo"]},"perplexity":{"crawlcheck_cited":false,"source_labels":["vercel (x2)","developers.google (x2)","fokal","bugviso","mihirnaik","qwairy","serpmentor","aisyndicate","crawlmouse","webalyzer","dev.to","bignewsnetwork","developer.mozilla","aigcmkt","ambientarray","digitalmagma","omnibound","aischemagen","hadoseo","agents.suedeai","getintel","victorious"]}},{"question":"AI crawler list: the user agents, what each one does, and which you can verify","guide":"ai-crawler-list-what-each-one-does","ai_mode":{"crawlcheck_cited":false,"source_labels":["contextbolt","search engine journal","google for developers","geotoolbox.ai","sponsored: promptwatch"]},"perplexity":{"crawlcheck_cited":false,"source_labels":["arcjet","centinelanalytica (x4)","platform.openai","claude.com","wandered","primores","perplexity (x2)","commoncrawl","cloudflare","developers.google","deviceatlas","searchenginejournal","momenticmarketing","humansecurity","technologychecker","lumengeo","geotoolbox","51degrees","robotsense","udger","captaindns","alleo","aiola"]}},{"question":"GEO vs AEO vs LLM SEO: what the terms mean, and what you can measure","guide":"geo-aeo-llm-seo-what-the-terms-mean","ai_mode":{"crawlcheck_cited":false,"source_labels":["onely","serpstat","ctaio.dev"]},"perplexity":{"crawlcheck_cited":false,"source_labels":["business.adobe","searchengineland","llmbrandscan","blogs.bing","onely","ahrefs","peppereffect","developers.google","developers.openai","seosherpa","shadowdigital","redclawlabs","progressionagency","learn.microsoft","searchenginejournal","linkedin (x2)","advertisingbusiness","vasundhara","topify","explodingtopics","ansvisor","aicrawlercheck","ultraseosolutions","searchinfluence","chrisraulf","reddit"]}},{"question":"Free AI visibility checkers: what they really measure, and how to tell if one is any good","guide":"free-ai-visibility-checkers-what-they-measure","ai_mode":{"crawlcheck_cited":false,"source_labels":["veuno","amicited","ahrefs","linkeddit","flisko.si","youtube (searchlogistics)","frizerly","christianstrunk.com"]},"perplexity":{"crawlcheck_cited":true,"source_labels":["webtrek","rankfender","verifiedmarketresearch","amicited","frizerly","amplitude","lightsite","█████████","crawlcheck.io (this guide; 1 inline, 'what would make this test fail')","reddit"]}},{"question":"How to measure AI visibility yourself, without trusting a vendor's score","guide":"how-to-measure-ai-visibility","ai_mode":{"crawlcheck_cited":false,"source_labels":["youtube (bcjr podcast)","supermetrics","nuance collaborative","asp events","gwenth.com","reddit r/GEO_optimization, next level ai","vyara stefcheva","nalaber"]},"perplexity":{"crawlcheck_cited":true,"source_labels":["crawlcheck.io (this guide; lead citation, 3 inline)","growleads","aideploymentpartner","supermetrics","llmpulse","developers.openai","google support","openai help (x2)","reddit","linkedin","influencermarketinghub","dev.to (x2)","techbullion","developers.google (x3)","search.google","flangapp","github","searchengineworld","copylume","ipinfo","proxies.sx"]}},{"question":"How to block AI crawlers like GPTBot and ClaudeBot, and what each block costs","guide":"how-to-block-ai-crawlers","ai_mode":{"crawlcheck_cited":false,"source_labels":["stytch.com","centinel analytica","youtube (edward sturm, wptuts)","cloudflare blog","amicited","trustmyip","geotoolbox.ai"]},"perplexity":{"crawlcheck_cited":false,"source_labels":["developers.cloudflare (x6)","claude.com","developers.openai","dev.to (x2)","searchenginejournal (x2)","neil-clarke","sitepoint","cyberciti","rabbitclip","indexernow","vixi","thestacc","indexmesh","seroundtable","cloudflare (x3)","centinelanalytica","seo-kreativ","transistordigital","promptwatch","byfaro","linkedin","sona","oasy","outrank","built2winweb","onelence","primores","organikpi","yogreet","capconvert"]}},{"question":"YouTube SEO case study: a dormant channel with 81 videos rewritten, measured before and after","guide":"youtube-seo-case-study-81-videos-rewritten-measured-before-and-after","ai_mode":{"crawlcheck_cited":true,"source_labels":["crawlcheck.io (sole source, 4 inline; library page)"]},"perplexity":{"crawlcheck_cited":true,"source_labels":["crawlcheck.io (case study, library, glossary Product; ~10 inline)","seouniversity","clickminded","alanspicer (x2)","overlap","ahrsumon","designbossstudio"]}}]}],"source-captures.json":[{"id":"google-ai-features","url":"https://developers.google.com/search/docs/appearance/ai-features","final_url":"https://developers.google.com/search/docs/appearance/ai-features","observed_at":"2026-10-30T19:56:26.340037+00:00","status":200,"content_type":"text/html; charset=utf-8","bytes":180003,"sha256":"7da1117fc5431ae79cc313c3b927b940a9ee58b57a160f097ec257399df2e035"},{"id":"google-javascript","url":"https://developers.google.com/search/docs/crawling-indexing/javascript/javascript-seo-basics","final_url":"https://developers.google.com/search/docs/crawling-indexing/javascript/javascript-seo-basics","observed_at":"2026-10-30T19:56:26.341132+00:00","status":200,"content_type":"text/html; charset=utf-8","bytes":221906,"sha256":"fe415dbe9111bd684de3de75d78181d37f9aa054cf65085fa89e21b9eaa8ebf9"},{"id":"google-localbusiness","url":"https://developers.google.com/search/docs/appearance/structured-data/local-business","final_url":"https://developers.google.com/search/docs/appearance/structured-data/local-business","observed_at":"2026-10-30T19:56:26.341798+00:00","status":200,"content_type":"text/html; charset=utf-8","bytes":226222,"sha256":"d620e24f303451f6665c5029d10dd36806946f056a36de50f362f8284c1bd41f"},{"id":"google-verification","url":"https://developers.google.com/search/docs/crawling-indexing/verifying-googlebot","final_url":"https://developers.google.com/crawling/docs/crawlers-fetchers/verify-google-requests","observed_at":"2026-10-30T19:56:26.342390+00:00","status":200,"content_type":"text/html; charset=utf-8","bytes":69980,"sha256":"c12ae4c5dc8fb5fbd0bc78c068e52b1b5373da53996dbf8c7af46b18a950c460"},{"id":"google-recrawl","url":"https://developers.google.com/search/docs/crawling-indexing/ask-google-to-recrawl","final_url":"https://developers.google.com/search/docs/crawling-indexing/ask-google-to-recrawl","observed_at":"2026-10-30T19:56:29.866138+00:00","status":200,"content_type":"text/html; charset=utf-8","bytes":170954,"sha256":"acc27b12bea4779ebc2826c30752b0cc24ab28583ec876cf8b9a57fbf386cf9f"},{"id":"google-crawlers","url":"https://developers.google.com/search/docs/crawling-indexing/overview-google-crawlers","final_url":"https://developers.google.com/crawling/docs/crawlers-fetchers/overview-google-crawlers","observed_at":"2026-10-30T19:56:29.885280+00:00","status":200,"content_type":"text/html; charset=utf-8","bytes":74668,"sha256":"45ab0440b7fbf9b0ffe24213d7f4adf61d215278667abb569644d43ee8e1a92e"},{"id":"google-extended","url":"https://developers.google.com/search/docs/crawling-indexing/google-common-crawlers","final_url":"https://developers.google.com/crawling/docs/crawlers-fetchers/google-common-crawlers","observed_at":"2026-10-30T19:56:30.199764+00:00","status":200,"content_type":"text/html; charset=utf-8","bytes":89489,"sha256":"38becab182c7af6e3147ebcba06b2d9941ca88658f65c84188bf780fdd703723"},{"id":"openai-bots","url":"https://platform.openai.com/docs/bots","final_url":"https://developers.openai.com/api/docs/bots","observed_at":"2026-10-30T19:56:33.428853+00:00","status":200,"content_type":"text/html; charset=utf-8","bytes":365640,"sha256":"e08c54c07ac0a9306b6441ba38359c723ef3262bd97aa2a209785e15219e6d31"},{"id":"cloudflare-ai-control","url":"https://developers.cloudflare.com/ai-crawl-control/","final_url":"https://developers.cloudflare.com/ai-crawl-control/","observed_at":"2026-10-30T19:56:33.441293+00:00","status":200,"content_type":"text/html","bytes":116365,"sha256":"d3db5ae299dd590a5a83563c375755466c307ef35f0165da7d37f702f32f4808"},{"id":"cloudflare-traffic","url":"https://developers.cloudflare.com/ai-crawl-control/features/analyze-ai-traffic/","final_url":"https://developers.cloudflare.com/ai-crawl-control/features/analyze-ai-traffic/","observed_at":"2026-10-30T19:56:36.450727+00:00","status":200,"content_type":"text/html","bytes":130001,"sha256":"f537c6ad4b58c7bf605876ce829acdfa6d92267b7689768a97dcbb271ed402b3"},{"id":"http-semantics","url":"https://www.rfc-editor.org/rfc/rfc9110.html","final_url":"https://www.rfc-editor.org/rfc/rfc9110.html","observed_at":"2026-10-30T19:56:36.545372+00:00","status":200,"content_type":"text/html","bytes":1187554,"sha256":"d431760660ea44e130f6e919dab216df2d0b3a490567a98089267523368fe1e5"},{"id":"robots-standard","url":"https://www.rfc-editor.org/rfc/rfc9309.html","final_url":"https://www.rfc-editor.org/rfc/rfc9309.html","observed_at":"2026-10-30T19:56:36.667694+00:00","status":200,"content_type":"text/html","bytes":82367,"sha256":"bbaa280636d3e38f0a1a130a5ce40e9753db1ed18c860945b0bf795ff8dc1a87"},{"id":"applebot","url":"https://support.apple.com/en-us/119829","final_url":"https://support.apple.com/en-us/119829","observed_at":"2026-10-30T19:56:38.070646+00:00","status":200,"content_type":"text/html;charset=utf-8","bytes":180475,"sha256":"30c9ef308e140287d2a6310879414df7f87a65314234b51d00bd971d6f4daad3"},{"id":"schema-localbusiness","url":"https://schema.org/LocalBusiness","final_url":"https://schema.org/LocalBusiness","observed_at":"2026-10-30T19:56:38.939596+00:00","status":200,"content_type":"text/html","bytes":40870,"sha256":"57fddb8e029db70d8a82398ce4d0225b7db281e0a3dd6d3d700ae0beb2b3e7c3"},{"id":"crawlcheck-builder-study","url":"https://crawlcheck.io/blog/what-ai-crawlers-get-from-wix-shopify-squarespace-webflow-sites","final_url":"https://crawlcheck.io/blog/what-ai-crawlers-get-from-wix-shopify-squarespace-webflow-sites","observed_at":"2026-10-30T19:56:39.139236+00:00","status":200,"content_type":"text/html; charset=utf-8","bytes":68613,"sha256":"22748cfddb19b713b1e0263ff682f93ebc3723774e404fa39fedff41fe06a3ad"},{"id":"crawlcheck-four-models","url":"https://crawlcheck.io/blog/what-four-ai-models-told-customers-about-our-three-businesses","final_url":"https://crawlcheck.io/blog/what-four-ai-models-told-customers-about-our-three-businesses","observed_at":"2026-10-30T19:56:39.371123+00:00","status":200,"content_type":"text/html; charset=utf-8","bytes":62414,"sha256":"28ffdc6cdd8427ad168a2b79a8790492f58f0a1a3cd6de55980fc2f6effe6be8"}],"representation-readings.json":[{"path":"/blog","accept":"text/html","observed_at":"2026-10-30T19:56:42.764162+00:00","status":200,"content_type":"text/html; charset=utf-8","vary":"Accept","bytes":89867,"sha256":"eef19ecdae3aee44b623ca9066a7a1c99f945a6d286609aa7e8805b34f9dcd77"},{"path":"/blog","accept":"text/markdown","observed_at":"2026-10-30T19:56:45.604392+00:00","status":200,"content_type":"text/markdown; charset=utf-8","vary":"Accept","bytes":21094,"sha256":"7ed125ad873db169778531d9b70b694ddf66327b17c1f4e671e1bc72234fb020"},{"path":"/blog","accept":"*/*","observed_at":"2026-10-30T19:56:48.392229+00:00","status":200,"content_type":"text/html; charset=utf-8","vary":"Accept","bytes":89469,"sha256":"df92ff14bb19777389c1fceb1f120d124ccf3a58f27b322403be94a05a34949a"},{"path":"/guides/does-llms-txt-actually-work","accept":"text/html","observed_at":"2026-10-30T19:56:51.312114+00:00","status":200,"content_type":"text/html; charset=utf-8","vary":"Accept","bytes":75808,"sha256":"81fe648be48d8713287d5fd8c8c1fe14edf11195ef0840cdbcfa5ef558570720"},{"path":"/guides/does-llms-txt-actually-work","accept":"text/markdown","observed_at":"2026-10-30T19:56:54.152119+00:00","status":200,"content_type":"text/markdown; charset=utf-8","vary":"Accept","bytes":18280,"sha256":"c63927a670b9d9d3394ea27fa0663cdf67e8f3b6e0e91847af4f39db3b7f9f09"},{"path":"/guides/does-llms-txt-actually-work","accept":"*/*","observed_at":"2026-10-30T19:56:57.376221+00:00","status":200,"content_type":"text/html; charset=utf-8","vary":"Accept","bytes":75404,"sha256":"b60fba0e3dca625103450cb90057badc39370db38a55d36341acf9ced97ecb47"},{"path":"/guides/how-to-measure-ai-visibility","accept":"text/html","observed_at":"2026-10-30T19:57:00.584015+00:00","status":200,"content_type":"text/html; charset=utf-8","vary":"Accept","bytes":66422,"sha256":"1da7bfee6edbb43b861d363d4fdae9a3cf71d6f5f717f892b1e67052ff61ad90"},{"path":"/guides/how-to-measure-ai-visibility","accept":"text/markdown","observed_at":"2026-10-30T19:57:03.459228+00:00","status":200,"content_type":"text/markdown; charset=utf-8","vary":"Accept","bytes":15366,"sha256":"0c41f2d0264e7823e805abeb26ed2fdaffc24234750d122dc9a82a77be7b9ebf"},{"path":"/guides/how-to-measure-ai-visibility","accept":"*/*","observed_at":"2026-10-30T19:57:06.824505+00:00","status":200,"content_type":"text/html; charset=utf-8","vary":"Accept","bytes":66024,"sha256":"05a98d7107c237c1f512b0ffe70a7051a78bf9ec4f6d3a9f4c4e879a14d3de38"},{"path":"/blog/what-ai-crawlers-get-from-wix-shopify-squarespace-webflow-sites","accept":"text/html","observed_at":"2026-10-30T19:57:09.648055+00:00","status":200,"content_type":"text/html; charset=utf-8","vary":"Accept","bytes":68613,"sha256":"1603604f8376cadfda277a2d75cb18749016a8670996460b96701dc753f20dd2"},{"path":"/blog/what-ai-crawlers-get-from-wix-shopify-squarespace-webflow-sites","accept":"text/markdown","observed_at":"2026-10-30T19:57:12.760226+00:00","status":200,"content_type":"text/markdown; charset=utf-8","vary":"Accept","bytes":13690,"sha256":"0cde1073be7b5540f5ba515b22e5de647285e37c0f20976f2bb64a4fa6f2b7ef"},{"path":"/blog/what-ai-crawlers-get-from-wix-shopify-squarespace-webflow-sites","accept":"*/*","observed_at":"2026-10-30T19:57:15.620391+00:00","status":200,"content_type":"text/html; charset=utf-8","vary":"Accept","bytes":68215,"sha256":"50dd4967779cc3a5df43584fb2eeb73ecc63b07df5361c630b87dce945346184"}],"initial-text-fixtures.json":{"type":"synthetic initial-response text extraction; no JavaScript rendering or AI surface tested","results":[{"id":"server-html","html":"<h1>Demo</h1><p>Service code: DEMO-731</p>","raw_bytes_contains_fact":true,"initial_nonscript_text_contains_fact":true},{"id":"script-literal","html":"<h1>Demo</h1><script>document.body.append(\"Service code: DEMO-731\")</script>","raw_bytes_contains_fact":true,"initial_nonscript_text_contains_fact":false},{"id":"external-client-fetch","html":"<h1>Demo</h1><script src=\"/client.js\"></script>","raw_bytes_contains_fact":false,"initial_nonscript_text_contains_fact":false},{"id":"noscript-fallback","html":"<h1>Demo</h1><noscript>Service code: DEMO-731</noscript>","raw_bytes_contains_fact":true,"initial_nonscript_text_contains_fact":true}]},"verification-fixtures.json":{"type":"synthetic policy demonstration on reserved documentation addresses and fictional bot names; not a test of a third-party library","policy":"exact identity-specific range membership; stale/missing evidence remains unknown","results":[{"id":"right-range","claim":"DemoSearchBot","ip":"192.0.2.1","expected":"verified","actual":"verified"},{"id":"last-in-range","claim":"DemoSearchBot","ip":"192.0.2.127","expected":"verified","actual":"verified"},{"id":"outside-boundary","claim":"DemoSearchBot","ip":"192.0.2.128","expected":"contradicted","actual":"contradicted"},{"id":"same-provider-other-role","claim":"DemoSearchBot","ip":"198.51.100.1","expected":"contradicted","actual":"contradicted"},{"id":"right-training-role","claim":"DemoTrainingBot","ip":"198.51.100.1","expected":"verified","actual":"verified"},{"id":"name-only","claim":"UnknownBot","ip":"192.0.2.1","expected":"unverifiable","actual":"unverifiable"},{"id":"feed-unavailable","claim":"DemoSearchBot","ip":"192.0.2.1","feed_unavailable":true,"expected":"unverifiable","actual":"unverifiable"},{"id":"feed-stale","claim":"DemoSearchBot","ip":"192.0.2.1","feed_stale":true,"expected":"unverifiable","actual":"unverifiable"},{"id":"ipv6-correct","claim":"DemoSearchBot","ip":"2001:db8:1::1","expected":"verified","actual":"verified"},{"id":"ipv6-wrong","claim":"DemoSearchBot","ip":"2001:db8:2::1","expected":"contradicted","actual":"contradicted"},{"id":"mapped-ipv4","claim":"DemoSearchBot","ip":"::ffff:192.0.2.1","expected":"verified","actual":"verified"},{"id":"training-wrong-range","claim":"DemoTrainingBot","ip":"192.0.2.1","expected":"contradicted","actual":"contradicted"}],"passed":12},"robots-fixtures.json":{"type":"synthetic deterministic parser fixtures; not crawler behavior or ranking measurements","runtime":"Python standard library urllib.robotparser","policies":{"general-only":"User-agent: *\nDisallow: /private/\n","named-allow":"User-agent: *\nDisallow: /private/\n\nUser-agent: GPTBot\nAllow: /\n","named-repeat":"User-agent: *\nDisallow: /private/\n\nUser-agent: GPTBot\nDisallow: /private/\n","training-deny-search-allow":"User-agent: GPTBot\nDisallow: /\n\nUser-agent: OAI-SearchBot\nAllow: /\n\nUser-agent: *\nDisallow: /private/\n"},"results":[{"policy":"general-only","user_agent":"GPTBot","path":"/private/a","expected_allowed":false,"observed_allowed":false},{"policy":"general-only","user_agent":"GPTBot","path":"/public/a","expected_allowed":true,"observed_allowed":true},{"policy":"general-only","user_agent":"OAI-SearchBot","path":"/private/a","expected_allowed":false,"observed_allowed":false},{"policy":"named-allow","user_agent":"GPTBot","path":"/private/a","expected_allowed":true,"observed_allowed":true},{"policy":"named-allow","user_agent":"OAI-SearchBot","path":"/private/a","expected_allowed":false,"observed_allowed":false},{"policy":"named-allow","user_agent":"OtherBot","path":"/private/a","expected_allowed":false,"observed_allowed":false},{"policy":"named-repeat","user_agent":"GPTBot","path":"/private/a","expected_allowed":false,"observed_allowed":false},{"policy":"named-repeat","user_agent":"GPTBot","path":"/public/a","expected_allowed":true,"observed_allowed":true},{"policy":"named-repeat","user_agent":"OAI-SearchBot","path":"/private/a","expected_allowed":false,"observed_allowed":false},{"policy":"training-deny-search-allow","user_agent":"GPTBot","path":"/public/a","expected_allowed":false,"observed_allowed":false},{"policy":"training-deny-search-allow","user_agent":"OAI-SearchBot","path":"/private/a","expected_allowed":true,"observed_allowed":true},{"policy":"training-deny-search-allow","user_agent":"OtherBot","path":"/private/a","expected_allowed":false,"observed_allowed":false}],"passed":12},"citation-comparison.json":{"measured_dates":["2026-10-05","2026-10-29"],"unit":"one question on one named surface in one archived manual pass","raw_answers_retained_in_this_release":false,"summary":{"ai_mode":{"matched_questions":7,"oct05_cited":0,"oct29_cited":2,"false_to_true":2,"true_to_false":0,"oct29_all_questions":27,"oct29_all_cited":4,"oct29_records_with_source_labels":27},"perplexity":{"matched_questions":7,"oct05_cited":1,"oct29_cited":3,"false_to_true":2,"true_to_false":0,"oct29_all_questions":27,"oct29_all_cited":9,"oct29_records_with_source_labels":27}},"matched":[{"question":"how to test robots.txt for GPTBot","surface":"ai_mode","oct05":false,"oct29":false},{"question":"how to test robots.txt for GPTBot","surface":"perplexity","oct05":false,"oct29":true},{"question":"does Cloudflare block AI crawlers even if robots.txt allows them","surface":"ai_mode","oct05":false,"oct29":false},{"question":"does Cloudflare block AI crawlers even if robots.txt allows them","surface":"perplexity","oct05":false,"oct29":false},{"question":"how to test color contrast on hover and focus states","surface":"ai_mode","oct05":false,"oct29":false},{"question":"how to test color contrast on hover and focus states","surface":"perplexity","oct05":false,"oct29":true},{"question":"how to find other websites using my brand name","surface":"ai_mode","oct05":false,"oct29":true},{"question":"how to find other websites using my brand name","surface":"perplexity","oct05":false,"oct29":false},{"question":"how to find broken internal links and anchor links","surface":"ai_mode","oct05":false,"oct29":false},{"question":"how to find broken internal links and anchor links","surface":"perplexity","oct05":false,"oct29":false},{"question":"how to move DMARC from p=none to p=quarantine","surface":"ai_mode","oct05":false,"oct29":false},{"question":"how to move DMARC from p=none to p=quarantine","surface":"perplexity","oct05":false,"oct29":false},{"question":"what is CrawlCheck","surface":"ai_mode","oct05":false,"oct29":true},{"question":"what is CrawlCheck","surface":"perplexity","oct05":true,"oct29":true}],"limitations":["Small nonrandom self-selected panel","Signed-in personalized sessions","Source labels are manual notes, not normalized URLs or complete transcripts","No randomized intervention","Only two comparable dates; no thirty-day repeated panel","No causal attribution or significance claim"]},"schema-parity-fixtures.json":{"type":"synthetic literal field consistency fixtures; not Schema.org or Google eligibility validation and not an AI outcome test","results":[{"id":"matching-phone","jsonld":{"@context":"https://schema.org","@type":"LocalBusiness","telephone":"+1-202-555-0100"},"visible_text":"Phone: +1-202-555-0100","json_syntax_valid":true,"field_comparison":"match"},{"id":"conflicting-phone","jsonld":{"@context":"https://schema.org","@type":"LocalBusiness","telephone":"+1-202-555-0100"},"visible_text":"Phone: +1-202-555-0199","json_syntax_valid":true,"field_comparison":"conflict"},{"id":"conflicting-hours","jsonld":{"@context":"https://schema.org","@type":"LocalBusiness","openingHours":"Mo-Fr 09:00-17:00"},"visible_text":"Hours: Mo-Fr 10:00-16:00","json_syntax_valid":true,"field_comparison":"conflict"},{"id":"conflicting-name","jsonld":{"@context":"https://schema.org","@type":"LocalBusiness","name":"Example North"},"visible_text":"Name: Example South","json_syntax_valid":true,"field_comparison":"conflict"},{"id":"not-on-page","jsonld":{"@context":"https://schema.org","@type":"LocalBusiness","telephone":"+1-202-555-0100"},"visible_text":"Contact us","json_syntax_valid":true,"field_comparison":"not-established"},{"id":"no-ground-truth","jsonld":{"@context":"https://schema.org","@type":"LocalBusiness","name":"Example Business"},"visible_text":"Welcome","json_syntax_valid":true,"field_comparison":"not-established"}]}},"reproduction":{"language":"Python 3","source":"#!/usr/bin/env python3\n\"\"\"Recompute released aggregates and fixture outcomes offline from the evidence bundle.\"\"\"\nimport ipaddress\nimport json\nfrom pathlib import Path\nimport sys\nfrom html.parser import HTMLParser\nfrom urllib.robotparser import RobotFileParser\n\npath=Path(sys.argv[1]) if len(sys.argv)>1 else Path(__file__).with_name('evidence-bundle.json')\nbundle=json.loads(path.read_text());files=bundle['files'];checks=0\nrows=files['citation-comparison.json']['matched']\nfor surface,expected in files['citation-comparison.json']['summary'].items():\n selected=[r for r in rows if r['surface']==surface]\n actual=dict(matched_questions=len(selected),oct05_cited=sum(r['oct05'] for r in selected),oct29_cited=sum(r['oct29'] for r in selected),false_to_true=sum(not r['oct05'] and r['oct29'] for r in selected),true_to_false=sum(r['oct05'] and not r['oct29'] for r in selected))\n for key,value in actual.items():assert expected[key]==value;checks+=1\n latest=files['answer-observations.json'][-1]['rows']\n assert expected['oct29_all_questions']==len(latest);checks+=1\n assert expected['oct29_all_cited']==sum(r[surface]['crawlcheck_cited'] for r in latest);checks+=1\nf=files['robots-fixtures.json']\nfor r in f['results']:\n p=RobotFileParser();p.parse(f['policies'][r['policy']].splitlines());assert p.can_fetch(r['user_agent'],r['path'])==r['expected_allowed']==r['observed_allowed'];checks+=1\nclass Text(HTMLParser):\n def __init__(self):super().__init__();self.skip=0;self.parts=[]\n def handle_starttag(self,t,a):\n  if t in ('script','style'):self.skip+=1\n def handle_endtag(self,t):\n  if t in ('script','style'):self.skip-=1\n def handle_data(self,t):\n  if not self.skip:self.parts.append(t)\nfor r in files['initial-text-fixtures.json']['results']:\n p=Text();p.feed(r['html']);assert ('DEMO-731' in ''.join(p.parts))==r['initial_nonscript_text_contains_fact'];checks+=1\nfor r in files['schema-parity-fixtures.json']['results']:\n j=json.loads(json.dumps(r['jsonld']));field=next(k for k in j if not k.startswith('@'));value=j[field]\n actual='match' if value in r['visible_text'] else ('conflict' if ':' in r['visible_text'] else 'not-established')\n assert actual==r['field_comparison'];checks+=1\nnetworks={'DemoSearchBot':['192.0.2.0/25','2001:db8:1::/48'],'DemoTrainingBot':['198.51.100.0/24']}\nfor r in files['verification-fixtures.json']['results']:\n if r.get('feed_stale') or r.get('feed_unavailable') or r['claim'] not in networks:actual='unverifiable'\n else:\n  address=ipaddress.ip_address(r['ip'])\n  if isinstance(address,ipaddress.IPv6Address) and address.ipv4_mapped:address=address.ipv4_mapped\n  actual='verified' if any(address.version==ipaddress.ip_network(n).version and address in ipaddress.ip_network(n) for n in networks[r['claim']]) else 'contradicted'\n assert actual==r['expected']==r['actual'];checks+=1\nprint(f'PASS: {checks} aggregate/fixture assertions. Historic answer labels and HTTP captures are inputs, not independently replayed observations.')\n","usage":"Save source as reproduce.py alongside this downloaded JSON, then run python3 reproduce.py citation-evidence-2026-10-30.json. It performs no network requests."}}