{"url":"https://lab2-clean.crawlcheck.io/","finalUrl":"https://lab2-clean.crawlcheck.io/","checkedAt":"2026-11-04T05:59:47.811Z","ms":77,"control":{"status":200,"refused":false,"browserRefused":false,"error":null},"robots":{"state":"ok","note":"robots.txt at https://lab2-clean.crawlcheck.io returned 200 with 1 group","groups":1},"score":{"reachable":16,"blocked":0,"verify":0,"unverifiable":0,"total":16,"pct":100},"issues":[],"results":[{"token":"GPTBot","name":"GPTBot","operator":"OpenAI","purpose":"training","profile":"/directory/ai-bots/gptbot","verdict":"Reachable","evidence":"All six gates passed","fix":"Nothing to fix.","issueKey":"","gates":{"robots":{"ok":true,"detail":"the \"User-agent: *\" group allows / (line 3: Allow: /)"},"directive":{"ok":true,"detail":"No blocking directive"},"status":{"ok":true,"detail":"HTTP 200","code":200},"redirects":{"ok":true,"detail":"No redirects","hops":0},"interstitial":{"ok":true,"detail":"No challenge or block page detected"},"js":{"ok":true,"detail":"1,128 characters of text are present without JavaScript."}}},{"token":"OAI-SearchBot","name":"OAI-SearchBot","operator":"OpenAI","purpose":"search","profile":"/directory/ai-bots/oai-searchbot","verdict":"Reachable","evidence":"All six gates passed","fix":"Nothing to fix.","issueKey":"","gates":{"robots":{"ok":true,"detail":"the \"User-agent: *\" group allows / (line 3: Allow: /)"},"directive":{"ok":true,"detail":"No blocking directive"},"status":{"ok":true,"detail":"HTTP 200","code":200},"redirects":{"ok":true,"detail":"No redirects","hops":0},"interstitial":{"ok":true,"detail":"No challenge or block page detected"},"js":{"ok":true,"detail":"1,128 characters of text are present without JavaScript."}}},{"token":"ChatGPT-User","name":"ChatGPT-User","operator":"OpenAI","purpose":"user","profile":"/directory/ai-bots/chatgpt-user","verdict":"Reachable","evidence":"All six gates passed","fix":"Nothing to fix.","issueKey":"","gates":{"robots":{"ok":true,"detail":"the \"User-agent: *\" group allows / (line 3: Allow: /)"},"directive":{"ok":true,"detail":"No blocking directive"},"status":{"ok":true,"detail":"HTTP 200","code":200},"redirects":{"ok":true,"detail":"No redirects","hops":0},"interstitial":{"ok":true,"detail":"No challenge or block page detected"},"js":{"ok":true,"detail":"1,128 characters of text are present without JavaScript."}}},{"token":"ClaudeBot","name":"ClaudeBot","operator":"Anthropic","purpose":"training","profile":"/directory/ai-bots/claudebot","verdict":"Reachable","evidence":"All six gates passed","fix":"Nothing to fix.","issueKey":"","gates":{"robots":{"ok":true,"detail":"the \"User-agent: *\" group allows / (line 3: Allow: /)"},"directive":{"ok":true,"detail":"No blocking directive"},"status":{"ok":true,"detail":"HTTP 200","code":200},"redirects":{"ok":true,"detail":"No redirects","hops":0},"interstitial":{"ok":true,"detail":"No challenge or block page detected"},"js":{"ok":true,"detail":"1,128 characters of text are present without JavaScript."}}},{"token":"Claude-SearchBot","name":"Claude-SearchBot","operator":"Anthropic","purpose":"search","profile":"/directory/ai-bots/claude-searchbot","verdict":"Reachable","evidence":"All six gates passed","fix":"Nothing to fix.","issueKey":"","gates":{"robots":{"ok":true,"detail":"the \"User-agent: *\" group allows / (line 3: Allow: /)"},"directive":{"ok":true,"detail":"No blocking directive"},"status":{"ok":true,"detail":"HTTP 200","code":200},"redirects":{"ok":true,"detail":"No redirects","hops":0},"interstitial":{"ok":true,"detail":"No challenge or block page detected"},"js":{"ok":true,"detail":"1,128 characters of text are present without JavaScript."}}},{"token":"Claude-User","name":"Claude-User","operator":"Anthropic","purpose":"user","profile":"/directory/ai-bots/claude-user","verdict":"Reachable","evidence":"All six gates passed","fix":"Nothing to fix.","issueKey":"","gates":{"robots":{"ok":true,"detail":"the \"User-agent: *\" group allows / (line 3: Allow: /)"},"directive":{"ok":true,"detail":"No blocking directive"},"status":{"ok":true,"detail":"HTTP 200","code":200},"redirects":{"ok":true,"detail":"No redirects","hops":0},"interstitial":{"ok":true,"detail":"No challenge or block page detected"},"js":{"ok":true,"detail":"1,128 characters of text are present without JavaScript."}}},{"token":"PerplexityBot","name":"PerplexityBot","operator":"Perplexity AI","purpose":"search","profile":"/directory/ai-bots/perplexitybot","verdict":"Reachable","evidence":"All six gates passed","fix":"Nothing to fix.","issueKey":"","gates":{"robots":{"ok":true,"detail":"the \"User-agent: *\" group allows / (line 3: Allow: /)"},"directive":{"ok":true,"detail":"No blocking directive"},"status":{"ok":true,"detail":"HTTP 200","code":200},"redirects":{"ok":true,"detail":"No redirects","hops":0},"interstitial":{"ok":true,"detail":"No challenge or block page detected"},"js":{"ok":true,"detail":"1,128 characters of text are present without JavaScript."}}},{"token":"Perplexity-User","name":"Perplexity-User","operator":"Perplexity AI","purpose":"user","profile":"/directory/ai-bots/perplexity-user","verdict":"Reachable","evidence":"All six gates passed","fix":"Nothing to fix.","issueKey":"","gates":{"robots":{"ok":true,"detail":"the \"User-agent: *\" group allows / (line 3: Allow: /)"},"directive":{"ok":true,"detail":"No blocking directive"},"status":{"ok":true,"detail":"HTTP 200","code":200},"redirects":{"ok":true,"detail":"No redirects","hops":0},"interstitial":{"ok":true,"detail":"No challenge or block page detected"},"js":{"ok":true,"detail":"1,128 characters of text are present without JavaScript."}}},{"token":"Google-Extended","name":"Google-Extended","operator":"Google","purpose":"training","profile":"/directory/ai-bots/google-extended","verdict":"Opt-out token: allowed","evidence":"Robots.txt control token only: it never fetches pages, so gates 2 to 6 do not apply. the \"User-agent: *\" group allows / (line 3: Allow: /).","fix":"Nothing to fix, unless you want to opt out: add a \"User-agent: Google-Extended\" group with \"Disallow: /\".","issueKey":"","gates":{"robots":{"ok":true,"detail":"the \"User-agent: *\" group allows / (line 3: Allow: /)"},"directive":{"ok":true,"detail":"No blocking directive"},"status":{"ok":true,"detail":"HTTP 200","code":200},"redirects":{"ok":true,"detail":"No redirects","hops":0},"interstitial":{"ok":true,"detail":"No challenge or block page detected"},"js":{"ok":true,"detail":"1,128 characters of text are present without JavaScript."}}},{"token":"Googlebot","name":"Googlebot","operator":"Google","purpose":"search-engine","profile":"/directory/search-engines/googlebot","verdict":"Reachable","evidence":"All six gates passed","fix":"Nothing to fix.","issueKey":"","gates":{"robots":{"ok":true,"detail":"the \"User-agent: *\" group allows / (line 3: Allow: /)"},"directive":{"ok":true,"detail":"No blocking directive"},"status":{"ok":true,"detail":"HTTP 200","code":200},"redirects":{"ok":true,"detail":"No redirects","hops":0},"interstitial":{"ok":true,"detail":"No challenge or block page detected"},"js":{"ok":true,"detail":"1,128 characters of text are present without JavaScript."}}},{"token":"bingbot","name":"Bingbot","operator":"Microsoft","purpose":"search-engine","profile":"/directory/search-engines/bingbot","verdict":"Reachable","evidence":"All six gates passed","fix":"Nothing to fix.","issueKey":"","gates":{"robots":{"ok":true,"detail":"the \"User-agent: *\" group allows / (line 3: Allow: /)"},"directive":{"ok":true,"detail":"No blocking directive"},"status":{"ok":true,"detail":"HTTP 200","code":200},"redirects":{"ok":true,"detail":"No redirects","hops":0},"interstitial":{"ok":true,"detail":"No challenge or block page detected"},"js":{"ok":true,"detail":"1,128 characters of text are present without JavaScript."}}},{"token":"Applebot-Extended","name":"Applebot-Extended","operator":"Apple","purpose":"training","profile":"/directory/search-engines/applebot-extended","verdict":"Opt-out token: allowed","evidence":"Robots.txt control token only: it never fetches pages, so gates 2 to 6 do not apply. the \"User-agent: *\" group allows / (line 3: Allow: /).","fix":"Nothing to fix, unless you want to opt out: add a \"User-agent: Applebot-Extended\" group with \"Disallow: /\".","issueKey":"","gates":{"robots":{"ok":true,"detail":"the \"User-agent: *\" group allows / (line 3: Allow: /)"},"directive":{"ok":true,"detail":"No blocking directive"},"status":{"ok":true,"detail":"HTTP 200","code":200},"redirects":{"ok":true,"detail":"No redirects","hops":0},"interstitial":{"ok":true,"detail":"No challenge or block page detected"},"js":{"ok":true,"detail":"1,128 characters of text are present without JavaScript."}}},{"token":"Meta-ExternalAgent","name":"Meta-ExternalAgent","operator":"Meta","purpose":"training","profile":"/directory/ai-bots/meta-externalagent","verdict":"Reachable","evidence":"All six gates passed","fix":"Nothing to fix.","issueKey":"","gates":{"robots":{"ok":true,"detail":"the \"User-agent: *\" group allows / (line 3: Allow: /)"},"directive":{"ok":true,"detail":"No blocking directive"},"status":{"ok":true,"detail":"HTTP 200","code":200},"redirects":{"ok":true,"detail":"No redirects","hops":0},"interstitial":{"ok":true,"detail":"No challenge or block page detected"},"js":{"ok":true,"detail":"1,128 characters of text are present without JavaScript."}}},{"token":"Amazonbot","name":"Amazonbot","operator":"Amazon","purpose":"search","profile":"/directory/cloud-services/amazonbot","verdict":"Reachable","evidence":"All six gates passed","fix":"Nothing to fix.","issueKey":"","gates":{"robots":{"ok":true,"detail":"the \"User-agent: *\" group allows / (line 3: Allow: /)"},"directive":{"ok":true,"detail":"No blocking directive"},"status":{"ok":true,"detail":"HTTP 200","code":200},"redirects":{"ok":true,"detail":"No redirects","hops":0},"interstitial":{"ok":true,"detail":"No challenge or block page detected"},"js":{"ok":true,"detail":"1,128 characters of text are present without JavaScript."}}},{"token":"Bytespider","name":"Bytespider","operator":"ByteDance","purpose":"training","profile":"/directory/scrapers/bytespider","verdict":"Reachable","evidence":"All six gates passed","fix":"Nothing to fix.","issueKey":"","gates":{"robots":{"ok":true,"detail":"the \"User-agent: *\" group allows / (line 3: Allow: /)"},"directive":{"ok":true,"detail":"No blocking directive"},"status":{"ok":true,"detail":"HTTP 200","code":200},"redirects":{"ok":true,"detail":"No redirects","hops":0},"interstitial":{"ok":true,"detail":"No challenge or block page detected"},"js":{"ok":true,"detail":"1,128 characters of text are present without JavaScript."}}},{"token":"CCBot","name":"CCBot","operator":"Common Crawl","purpose":"training","profile":"/directory/scrapers/ccbot","verdict":"Reachable","evidence":"All six gates passed","fix":"Nothing to fix.","issueKey":"","gates":{"robots":{"ok":true,"detail":"the \"User-agent: *\" group allows / (line 3: Allow: /)"},"directive":{"ok":true,"detail":"No blocking directive"},"status":{"ok":true,"detail":"HTTP 200","code":200},"redirects":{"ok":true,"detail":"No redirects","hops":0},"interstitial":{"ok":true,"detail":"No challenge or block page detected"},"js":{"ok":true,"detail":"1,128 characters of text are present without JavaScript."}}},{"token":"DuckAssistBot","name":"DuckAssistBot","operator":"DuckDuckGo","purpose":"user","profile":"/directory/ai-bots/duckassistbot","verdict":"Reachable","evidence":"All six gates passed","fix":"Nothing to fix.","issueKey":"","gates":{"robots":{"ok":true,"detail":"the \"User-agent: *\" group allows / (line 3: Allow: /)"},"directive":{"ok":true,"detail":"No blocking directive"},"status":{"ok":true,"detail":"HTTP 200","code":200},"redirects":{"ok":true,"detail":"No redirects","hops":0},"interstitial":{"ok":true,"detail":"No challenge or block page detected"},"js":{"ok":true,"detail":"1,128 characters of text are present without JavaScript."}}},{"token":"MistralAI-User","name":"MistralAI-User","operator":"Mistral AI","purpose":"user","profile":"/directory/ai-bots/mistralai-user","verdict":"Reachable","evidence":"All six gates passed","fix":"Nothing to fix.","issueKey":"","gates":{"robots":{"ok":true,"detail":"the \"User-agent: *\" group allows / (line 3: Allow: /)"},"directive":{"ok":true,"detail":"No blocking directive"},"status":{"ok":true,"detail":"HTTP 200","code":200},"redirects":{"ok":true,"detail":"No redirects","hops":0},"interstitial":{"ok":true,"detail":"No challenge or block page detected"},"js":{"ok":true,"detail":"1,128 characters of text are present without JavaScript."}}},{"token":"cohere-ai","name":"cohere-ai","operator":"Cohere","purpose":"user","profile":"/directory/ai-bots/cohere-ai","verdict":"Reachable","evidence":"All six gates passed","fix":"Nothing to fix.","issueKey":"","gates":{"robots":{"ok":true,"detail":"the \"User-agent: *\" group allows / (line 3: Allow: /)"},"directive":{"ok":true,"detail":"No blocking directive"},"status":{"ok":true,"detail":"HTTP 200","code":200},"redirects":{"ok":true,"detail":"No redirects","hops":0},"interstitial":{"ok":true,"detail":"No challenge or block page detected"},"js":{"ok":true,"detail":"1,128 characters of text are present without JavaScript."}}},{"token":"YouBot","name":"YouBot","operator":"You.com","purpose":"search","profile":"/directory/ai-bots/youbot","verdict":"Reachable","evidence":"All six gates passed","fix":"Nothing to fix.","issueKey":"","gates":{"robots":{"ok":true,"detail":"the \"User-agent: *\" group allows / (line 3: Allow: /)"},"directive":{"ok":true,"detail":"No blocking directive"},"status":{"ok":true,"detail":"HTTP 200","code":200},"redirects":{"ok":true,"detail":"No redirects","hops":0},"interstitial":{"ok":true,"detail":"No challenge or block page detected"},"js":{"ok":true,"detail":"1,128 characters of text are present without JavaScript."}}}]}