{
 "kind": "crawlcheck-ai-consent",
 "v": 1,
 "built_at": "2026-10-17T13:46:33.000Z",
 "days": [
  "2026-10-17",
  "2026-10-16",
  "2026-10-15",
  "2026-10-14",
  "2026-10-13",
  "2026-10-12",
  "2026-10-11"
 ],
 "newest": "2026-10-17",
 "fields": [
  [
   "domain",
   "Registrable domain"
  ],
  [
   "read_at",
   "When robots.txt and the homepage were read (UTC, ISO 8601)"
  ],
  [
   "lane",
   "saas, gov, local or filler"
  ],
  [
   "kind",
   "Site kind from the page"
  ],
  [
   "robots",
   "read = robots.txt parsed; unread = absent or unreadable when read (the three agent lists are then empty: an absent file allows every agent, an unreadable one cannot be judged)"
  ],
  [
   "blocked",
   "Panel agents robots.txt closes the homepage to"
  ],
  [
   "allowed",
   "Panel agents robots.txt lets read the homepage"
  ],
  [
   "unknown",
   "Panel agents not determinable from this read (older rows kept only the first 20 closed agents)"
  ],
  [
   "ai_closed_all",
   "Every AI agent closed, of the ~60 AI agents tested (rows read from 16 Oct 2026)"
  ],
  [
   "content_signal",
   "The Content-Signal line, verbatim, when one was read"
  ],
  [
   "signal_ai_train",
   "yes, no, or empty"
  ],
  [
   "signal_ai_input",
   "yes, no, or empty"
  ],
  [
   "signal_search",
   "yes, no, or empty"
  ],
  [
   "llms_txt",
   "Serves /llms.txt"
  ]
 ],
 "panel": [
  {
   "agent": "GPTBot",
   "company": "OpenAI",
   "use": "training"
  },
  {
   "agent": "OAI-SearchBot",
   "company": "OpenAI",
   "use": "search"
  },
  {
   "agent": "ChatGPT-User",
   "company": "OpenAI",
   "use": "user fetch"
  },
  {
   "agent": "ClaudeBot",
   "company": "Anthropic",
   "use": "training"
  },
  {
   "agent": "Claude-SearchBot",
   "company": "Anthropic",
   "use": "search"
  },
  {
   "agent": "Claude-User",
   "company": "Anthropic",
   "use": "user fetch"
  },
  {
   "agent": "Google-Extended",
   "company": "Google",
   "use": "training"
  },
  {
   "agent": "PerplexityBot",
   "company": "Perplexity",
   "use": "search"
  },
  {
   "agent": "Perplexity-User",
   "company": "Perplexity",
   "use": "user fetch"
  },
  {
   "agent": "Applebot-Extended",
   "company": "Apple",
   "use": "training"
  },
  {
   "agent": "CCBot",
   "company": "Common Crawl",
   "use": "training"
  },
  {
   "agent": "meta-externalagent",
   "company": "Meta",
   "use": "training"
  },
  {
   "agent": "Bytespider",
   "company": "ByteDance",
   "use": "training"
  },
  {
   "agent": "cohere-ai",
   "company": "Cohere",
   "use": "training"
  },
  {
   "agent": "MistralAI-User",
   "company": "Mistral",
   "use": "user fetch"
  },
  {
   "agent": "DeepSeekBot",
   "company": "DeepSeek",
   "use": "training"
  }
 ],
 "licence": "https://crawlcheck.io/datasets/ai-consent#licence",
 "scope": "free sample: 1000 sites, SaaS and government lanes, evenly spread",
 "sites": 1000,
 "parts": [
  {
   "lane": "saas+gov",
   "file": "sample.ndjson",
   "rows": 1000,
   "bytes": 442631,
   "sha256": "f23405a0010e3c966776ced04dfef1887e94d78cb61c8dba1f5e06a1847f4514"
  }
 ],
 "signature": {
  "alg": "Ed25519",
  "kid": "1hHmE3fo1rg5Ig70wuFtJ5wBc7G-sUTgUJkxyPWFRps",
  "message": "crawlcheck-ai-consent/v1:cd7cac79a6101a63da92759789fe4e6cae3bb6298b7f056ebe4b302ebe37609e",
  "manifest_sha256": "cd7cac79a6101a63da92759789fe4e6cae3bb6298b7f056ebe4b302ebe37609e",
  "sig": "vR3Rph2Akp4D8sMcMRCIiTSBCEgmvFsesFg12mM5IThAHH5T-cj0pzlGDhkS_qeqaTdkdfz38V2BWyF_ofheCg",
  "signed_at": "2026-10-17T13:46:35.142Z",
  "key": {
   "kid": "1hHmE3fo1rg5Ig70wuFtJ5wBc7G-sUTgUJkxyPWFRps",
   "jwk": {
    "kty": "OKP",
    "crv": "Ed25519",
    "x": "5TmKCOx0TTHepq7HwyXBHvGQpVCSBd2lV8cLzYVqFxQ"
   },
   "published_in": "https://crawlcheck.io/.well-known/http-message-signatures-directory"
  },
  "how": "manifest_sha256 = SHA-256 (hex) of the canonical JSON (keys sorted at every level, no whitespace) of this manifest without its signature field; sig = Ed25519 over the UTF-8 bytes of message, base64url"
 }
}