{"ok":true,"kind":"crawlcheck-census-records","frame":"top1k","run":43,"complete":true,"measured":991,"of":991,"shards":8,"offset":0,"returned":100,"next":"https://crawlcheck.io/api/census/records?run=43&offset=100&limit=100","records":[{"v":1,"kind":"crawlcheck-census-record","run":43,"domain":"google.com","rank":1,"measured_at":"2026-11-13T07:56:55.238Z","home":{"browser":{"status":429,"content_type":"text/html","bytes":3137,"sha256":"c2c91c9d792a304c0fa583e28944bb780ad4ddcfcf1ea1578f89c52980540e30","final_host":"www.google.com","final_path":"/sorry/index","ms":527,"identity":"Mobile browser","html":true},"gptbot":{"status":429,"content_type":"text/html","bytes":3137,"sha256":"a6a3ec6cb4c9a853030c746f059e67139786801b931b79bda50849db06949e37","final_host":"www.google.com","final_path":"/sorry/index","ms":392,"identity":"GPTBot","html":true},"claudebot":{"status":429,"content_type":"text/html","bytes":3137,"sha256":"68412b9a6848787cd72579a4212890441c5c945570be17b4cecb07c88de60955","final_host":"www.google.com","final_path":"/sorry/index","ms":647,"identity":"ClaudeBot","html":true},"oaisearch":{"status":429,"content_type":"text/html","bytes":3137,"sha256":"020d61f97dee91b2fd5c999e7d61d558b11375d4fd7c9bda976f9caacef35651","final_host":"www.google.com","final_path":"/sorry/index","ms":415,"identity":"OAI-SearchBot","html":true},"perplexity":{"status":429,"content_type":"text/html","bytes":3139,"sha256":"bd98320d4987ffe61f962b183ab01bf8b5edbdd219cd81b26bb4ccae20dab993","final_host":"www.google.com","final_path":"/sorry/index","ms":547,"identity":"PerplexityBot","html":true}},"robots":{"status":200,"content_type":"text/plain","bytes":6681,"sha256":"e8bf1ce79810901a9fef98deac93aaa532b1be5091ccd19f944dc8d86028a5c3","final_host":"www.google.com","final_path":"/robots.txt","ms":104,"html":false,"text":"User-agent: *\nUser-agent: Yandex\nDisallow: /search\nAllow: /search/about\nAllow: /search/howsearchworks\nDisallow: /sdch\nDisallow: /groups\nDisallow: /index.html?\nDisallow: /?\nDisallow: /goto?\nAllow: /?hl=\nDisallow: /?hl=*&\nAllow: /?hl=*&gws_rd=ssl$\nDisallow: /?hl=*&*&gws_rd=ssl\nAllow: /?gws_rd=ssl$\nAllow: /?pt1=true$\nDisallow: /imgres\nDisallow: /u/\nDisallow: /setprefs\nDisallow: /m?\nDisallow: /m/\nAllow:    /m/finance\nDisallow: /wml?\nDisallow: /wml/?\nDisallow: /wml/search?\nDisallow: /xhtml?\nDisallow: /xhtml/?\nDisallow: /xhtml/search?\nDisallow: /xml?\nDisallow: /imode?\nDisallow: /imode/?\nDisallow: /imode/search?\nDisallow: /jsky?\nDisallow: /jsky/?\nDisallow: /jsky/search?\nDisallow: /pda?\nDisallow: /pda/?\nDisallow: /pda/search?\nDisallow: /local?\nDisallow: /local_url\nDisallow: /products?\nDisallow: /product_\nDisallow: /products_\nDisallow: /products;\nDisallow: /print\nDisallow: /books/\nDisallow: /bkshp?*dq=\nDisallow: /bkshp?*q=\nDisallow: /books?*dq=\nDisallow: /books?*q=\nDisallow: /books?*qtid=\nDisallow: /books?*output=\nDisallow: /books?*pg=\nDisallow: /books?*jtp=\nDisallow: /books?*jscmd=\nDisallow: /books?*buy=\nDisallow: /books?*zoom=\nAllow: /books/about\nAllow: /books?*zoom=1\nAllow: /books?*zoom=5\nAllow: /books/content?*zoom=1\nAllow: /books/content?*zoom=5\nDisallow: /patents?\nDisallow: /patents/download/\nDisallow: /patents/pdf/\nDisallow: /patents/related/\nDisallow: /scholar\nDisallow: /citations?\nAllow: /citations?user=\nAllow: /citations?view_op=new_profile\nAllow: /citations?view_op=top_venues\nAllow: /scholar_share\nDisallow: /s?\nDisallow: /maps?\nAllow: /maps?daddr=\nAllow: /maps?entry=wc\nAllow: /maps?f=\nAllow: /maps?hl=\nAllow: /maps?q=\nAllow: /maps?saddr=\nAllow: /maps?sid=\nAllow: /maps?*output=classic\nAllow: /maps?*file=\nDisallow: /mapslt?\nDisallow: /maphp?\nDisallow: /maps/\nAllow: /maps/$\nAllow: /maps/@\nAllow: /maps/?daddr=\nAllow: /maps/?entry=wc\nAllow: /maps/?f=\nAllow: /maps/?hl=\nAllow: /maps/?q=\nAllow: /maps/?saddr=\nAllow: /maps/?sid=\nAllow: /maps/api/staticmap?\nAllow: /maps/search/\nAllow: /maps/sitemap.xml\nAllow: /maps/sitemaps/\nAllow: /maps/dir/\nAllow: /maps/d/\nAllow: /maps/reserve\nAllow: /maps/about\nAllow: /maps/contrib/\nAllow: /maps/match\nAllow: /maps/ms?\nAllow: /maps/place/\nAllow: /maps/_/\nAllow: /search?*tbm=map\nAllow: /maps/vt?\nAllow: /maps/preview\nDisallow: /maps/api/js/\nAllow: /maps/api/js\nDisallow: /mld?\nDisallow: /staticmap?\nDisallow: /help/maps/streetview/partners/welcome/\nDisallow: /help/maps/indoormaps/partners/\nDisallow: /lochp?\nDisallow: /ie?\nDisallow: /uds/\nDisallow: /transit?\nDisallow: /trends?\nDisallow: /trends/music?\nDisallow: /trends/hottrends?\nDisallow: /trends/viz?\nDisallow: /trends/embed.js?\nDisallow: /trends/fetchComponent?\nDisallow: /trends/beta\nDisallow: /trends/topics\nDisallow: /trends/explore?\nDisallow: /trends/api\nDisallow: /musica\nDisallow: /musicl\nDisallow: /musics\nDisallow: /urchin_test/\nDisallow: /movies?\nDisallow: /wapsearch?\nDisallow: /reviews/search?\nDisallow: /cbk\nDisallow: /profiles/me\nDisallow: /s2/profiles/me\nAllow: /s2/profiles\nAllow: /s2/oz\nAllow: /s2/photos\nAllow: /s2/search/social\nAllow: /s2/static\nDisallow: /s2\nDisallow: /transconsole/portal/\nDisallow: /aclk\nDisallow: /tbproxy/\nDisallow: /support/forum/search?\nDisallow: /reviews/polls/\nDisallow: /hosted/images/\nDisallow: /accounts/ClientLogin\nDisallow: /accounts/ClientAuth\nDisallow: /accounts/o8\nAllow: /accounts/o8/id\nDisallow: /quality_form?\nDisallow: /labs/popgadget/search\nDisallow: /compressiontest/\nDisallow: /analytics/feeds/\nDisallow: /analytics/partners/comments/\nDisallow: /analytics/portal/\nDisallow: /analytics/uploads/\nAllow: /alerts/manage\nAllow: /alerts/remove\nDisallow: /alerts/\nAllow: /alerts/$\nDisallow: /phone/compare/?\nDisallow: /travel/clk\nDisallow: /travel/lodging/clk\nDisallow: /travel/lodging/imp\nDisallow: /travel/entity\nDisallow: /travel/search\nDisallow: /travel/flights/booking\nDisallow: /travel/flights/s/\nDisallow: /travel/flights/search\nDisallow: /travel/hotels/stories\nDisallow: /travel/hotels/*/stories\nDisallow: /travel/story\nDisallow: /hotelfinder/rpc\nDisallow: /hotels/rpc\nDisallow: /evaluation/\nDisallow: /forms/perks/\nDisallow: /shopping/suppliers/search\nDisallow: /edu/cs4hs/\nDisallow: /trustedstores/s/\nDisallow: /trustedstores/tm2\nDisallow: /trustedstores/verify\nDisallow: /shopping?\nDisallow: /shopping/product/\nDisallow: /shopping/seller\nDisallow: /shopping/ratings/account/metrics\nDisallow: /shopping/ratings/merchant/immersivedetails\nDisallow: /shopping/reviewer\nDisallow: /shopping/search\nDisallow: /shopping/deals\nAllow: /shopping?udm=28$\nAllow: /shopping?udm=shop$\nDisallow: /storefront\nDisallow: /storepicker\nDisallow: /about/careers/applications/candidate-prep\nDisallow: /about/careers/applications/connect-with-a-googler\nDisallow: /about/careers/applications/jobs/results?page=\nDisallow: /about/careers/applications/jobs/results/?page=\nDisallow: /about/careers/applications/jobs/results?*&page=\nDisallow: /about/careers/applications/jobs/results/?*&page=\nDisallow: /landing/signout.html\nDisallow: /gallery/\nDisallow: /landing/now/ontap/\nAllow: /maps/reserve\nAllow: /maps/reserve/partners\nDisallow: /maps/reserve/api/\nDisallow: /maps/reserve/search\nDisallow: /maps/reserve/bookings\nDisallow: /maps/reserve/settings\nDisallow: /maps/reserve/manage\nDisallow: /maps/reserve/payment\nDisallow: /maps/reserve/receipt\nDisallow: /maps/reserve/sellersignup\nDisallow: /maps/reserve/feedback\nDisallow: /maps/reserve/terms\nDisallow: /maps/reserve/m/\nDisallow: /maps/reserve/b/\nDisallow: /maps/reserve/partner-dashboard\nDisallow: /local/cars\nDisallow: /local/dealership/\nDisallow: /local/dining/\nDisallow: /local/place/products/\nDisallow: /local/place/reviews/\nDisallow: /local/place/rap/\nDisallow: /local/tab/\nDisallow: /localservices/\nAllow: /localservices/coupon/\nDisallow: /nonprofits/account/\nDisallow: /uviewer\nDisallow: /landing/cmsnext-root/\n\n# AdsBot\nUser-agent: AdsBot-Google\nDisallow: /maps/api/js/\nAllow: /maps/api/js\nDisallow: /maps/api/place/js/\nDisallow: /maps/api/staticmap\nDisallow: /maps/api/streetview\n\n# New user agent groups must also have a user agent reference in the global (*)\n# group. See \"Order of precedence\" section in\n# https://goo.gle/rep#order-of-precedence-for-user-agents\nUser-agent: Yandex\nDisallow: /about/careers/applications/jobs/results\nDisallow: /about/careers/applications-a/jobs/results\n\n# Crawlers of certain social media sites are allowed to access page markup when\n# google.com/imgres* links are shared. To learn more, please contact\n# images-robots-allowlist@google.com.\nUser-agent: facebookexternalhit\nUser-agent: Twitterbot\nAllow: /imgres\nAllow: /search\nDisallow: /groups\nDisallow: /hosted/images/\nDisallow: /m/\n\nSitemap: https://www.google.com/sitemap.xml\n","agents":{"GPTBot":true,"OAI-SearchBot":true,"ChatGPT-User":true,"ClaudeBot":true,"Claude-SearchBot":true,"PerplexityBot":true,"Google-Extended":true,"Applebot-Extended":true,"CCBot":true,"Bytespider":true,"Meta-ExternalAgent":true,"Amazonbot":true}},"sitemap":null,"files":null,"derived":{"reachable":false,"home_class":"refused_us","robots_served":false,"robots_sitemap":false,"content_signal":false,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":[],"blocks_any_ai":false,"files":{"llms_txt":false,"agents_md":false,"agent_skills":false,"mcp_card":false,"api_catalog":false,"a2a_card":false},"any_capability":false,"ai_refused":[],"ai_thin":[],"rings":{"mcp_declared":false,"mcp_live":false,"mcp_auth":false,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":3,"fh":"www.google.com","fp_other_host":[]},"derive_v":5,"frame":"top1k","shard":0},{"v":1,"kind":"crawlcheck-census-record","run":43,"domain":"cloudflare.com","rank":2,"measured_at":"2026-11-13T07:57:55.549Z","home":{"browser":{"status":200,"content_type":"text/html","bytes":1334957,"sha256":"00ee845a0aec1159aa945abb5ca47747933dd009bd0842544f62c320a7cba150","final_host":"www.cloudflare.com","final_path":"/","ms":231,"identity":"Mobile browser","html":true,"fp":"52bec2ee8b21bd1f"},"gptbot":{"status":200,"content_type":"text/html","bytes":1334957,"sha256":"00ee845a0aec1159aa945abb5ca47747933dd009bd0842544f62c320a7cba150","final_host":"www.cloudflare.com","final_path":"/","ms":221,"identity":"GPTBot","html":true,"fp":"52bec2ee8b21bd1f"},"claudebot":{"status":200,"content_type":"text/html","bytes":1334957,"sha256":"00ee845a0aec1159aa945abb5ca47747933dd009bd0842544f62c320a7cba150","final_host":"www.cloudflare.com","final_path":"/","ms":228,"identity":"ClaudeBot","html":true,"fp":"52bec2ee8b21bd1f"},"oaisearch":{"status":200,"content_type":"text/html","bytes":1334957,"sha256":"00ee845a0aec1159aa945abb5ca47747933dd009bd0842544f62c320a7cba150","final_host":"www.cloudflare.com","final_path":"/","ms":229,"identity":"OAI-SearchBot","html":true,"fp":"52bec2ee8b21bd1f"},"perplexity":{"status":200,"content_type":"text/html","bytes":1334957,"sha256":"00ee845a0aec1159aa945abb5ca47747933dd009bd0842544f62c320a7cba150","final_host":"www.cloudflare.com","final_path":"/","ms":226,"identity":"PerplexityBot","html":true,"fp":"52bec2ee8b21bd1f"}},"robots":{"status":200,"content_type":"text/plain","bytes":1099,"sha256":"c6e360cc6e931ea35c33ae917ff9e2194eb55f79f68e949ec54bcd4533324d98","final_host":"www.cloudflare.com","final_path":"/robots.txt","ms":20,"html":false,"text":"# Robots.txt for www.cloudflare.com\n\nUser-agent: *\nAllow: /\n\n# Sitemap\nSitemap: https://www.cloudflare.com/sitemap.xml\n\n# AI/LLM friendly content\n# See https://llmstxt.org for the llms.txt specification\n# llms.txt provides curated content for AI assistants and LLMs\n\n# Allow AI crawlers to access markdown versions of pages\nUser-agent: GPTBot\nAllow: /\n\nUser-agent: ChatGPT-User\nAllow: /\n\nUser-agent: Google-Extended\nAllow: /\n\nUser-agent: Anthropic-AI\nAllow: /\n\nUser-agent: Claude-Web\nAllow: /\n\nUser-agent: CCBot\nAllow: /\n\nUser-agent: PerplexityBot\nAllow: /\n\nUser-agent: Cohere-ai\nAllow: /\n\n# Content Signals — declare AI content usage preferences\n# See https://contentsignals.org/ and https://datatracker.ietf.org/doc/draft-romm-aipref-contentsignals/\nContent-Signal: ai-train=yes, search=yes, ai-input=yes\n\n# AI-friendly content locations\n# - /llms.txt - Curated overview for AI/LLMs (markdown)\n# - /llms-full.txt - Full expanded content for larger context windows\n# - /*.md - Markdown versions of all pages (append .md to any URL)\n# - /.well-known/agents.json - Agent discovery and capabilities\n","agents":{"GPTBot":true,"OAI-SearchBot":true,"ChatGPT-User":true,"ClaudeBot":true,"Claude-SearchBot":true,"PerplexityBot":true,"Google-Extended":true,"Applebot-Extended":true,"CCBot":true,"Bytespider":true,"Meta-ExternalAgent":true,"Amazonbot":true}},"sitemap":{"status":200,"content_type":"application/xml","bytes":1639563,"sha256":"954a72fa35a37343d1da2fbe2cd33860f552c4c6347901d10ac27aaefde28af2","final_host":"www.cloudflare.com","final_path":"/sitemap.xml","ms":14,"url":"https://www.cloudflare.com/sitemap.xml","declared":true,"xml":true},"files":{"llms_txt":{"status":200,"content_type":"text/plain","bytes":17173,"sha256":"81d53b543800b2a5fb3521421b0b522d09097c19f8e19ba092c25901d97db7ac","final_host":"www.cloudflare.com","final_path":"/llms.txt","ms":21,"path":"/llms.txt","html":false,"json":null,"present":true},"agents_md":{"status":404,"content_type":"text/plain","bytes":12,"sha256":"69967e8148f9451562c4ef70a9ba15c29cd14b13376b2595a71da547f9a03802","final_host":"www.cloudflare.com","final_path":"/agents.md","ms":284,"path":"/agents.md","html":false,"json":null,"present":false},"agent_skills":{"status":200,"content_type":"application/json","bytes":11207,"sha256":"339186caba56b94036ddfbc50ce40585e99d4218befa7bc244a49e863a9a6e15","final_host":"www.cloudflare.com","final_path":"/.well-known/agent-skills/index.json","ms":240,"path":"/.well-known/agent-skills/index.json","html":false,"json":true,"present":true},"mcp_card":{"status":200,"content_type":"application/json","bytes":1320,"sha256":"296657d7bd0330bee2700f5de88edc054acc8637f6deb2b5cce8b9030f13706f","final_host":"www.cloudflare.com","final_path":"/.well-known/mcp/server-card.json","ms":214,"path":"/.well-known/mcp/server-card.json","html":false,"json":true,"present":true},"api_catalog":{"status":404,"content_type":"text/html","bytes":176757,"sha256":"b45c362a81ae9b05e644d9544924aa12d3796c0581b8657f33693f1b9dcba4f9","final_host":"www.cloudflare.com","final_path":"/.well-known/api-catalog","ms":288,"path":"/.well-known/api-catalog","html":true,"json":null,"present":false},"a2a_card":{"status":404,"content_type":"text/plain","bytes":12,"sha256":"69967e8148f9451562c4ef70a9ba15c29cd14b13376b2595a71da547f9a03802","final_host":"www.cloudflare.com","final_path":"/.well-known/agent-card.json","ms":100,"path":"/.well-known/agent-card.json","html":false,"json":null,"present":false}},"rings":{"v":1,"candidates":[{"url":"https://docs.mcp.cloudflare.com/mcp","source":"mcp_card"},{"url":"https://blog.mcp.cloudflare.com/mcp","source":"mcp_card"},{"url":"https://mcp.cloudflare.com/mcp","source":"mcp_card"}],"mcp":[{"url":"https://docs.mcp.cloudflare.com/mcp","source":"mcp_card","outcome":"initialized","status":200,"ms":12,"initialized":true,"protocol_version":"2025-06-18","server":{"name":"docs-ai-search","version":"0.5.0"},"tools":2,"tools_sha256":"16fd520bee852c5de3a0ec249dac7322b1152156c6012d9102160e830e374c92","auth_header":false},{"url":"https://blog.mcp.cloudflare.com/mcp","source":"mcp_card","outcome":"initialized","status":200,"ms":26,"initialized":true,"protocol_version":"2025-06-18","server":{"name":"cloudflare-blog","version":"0.3.0"},"tools":4,"tools_sha256":"4995c0bbb213a965db3f46d62682b9eb69b29c40d876b3bd3648716bf750927d","auth_header":false},{"url":"https://mcp.cloudflare.com/mcp","source":"mcp_card","outcome":"auth_required","status":401,"ms":4,"initialized":false,"protocol_version":null,"server":null,"tools":null,"tools_sha256":null,"auth_header":true}],"a2a":null,"webmcp":{"model_context":false,"bridge":false,"mcp_json":false,"link_rel":false,"any":false},"handshakes":3,"get_path":{"status":404,"content_type":"text/html","ms":752,"error":null}},"sf":[1,0,0,0,0,0,0,0,1],"derived":{"reachable":true,"home_class":"answers","robots_served":true,"robots_sitemap":true,"content_signal":true,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":[],"blocks_any_ai":false,"files":{"llms_txt":true,"agents_md":false,"agent_skills":true,"mcp_card":true,"api_catalog":false,"a2a_card":false},"any_capability":true,"ai_refused":[],"ai_thin":[],"aipref":0,"aipref_measured":true,"sf":[1,0,0,0,0,0,0,0,1],"rings":{"mcp_declared":true,"mcp_live":true,"mcp_auth":true,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":3,"fh":"www.cloudflare.com","fp_other_host":[],"fp":"52bec2ee8b21bd1f","fp_bits":{"gptbot":0,"claudebot":0,"oaisearch":0,"perplexity":0}},"derive_v":5,"frame":"top1k","shard":1},{"v":1,"kind":"crawlcheck-census-record","run":43,"domain":"facebook.com","rank":3,"measured_at":"2026-11-13T07:58:58.649Z","home":{"browser":{"status":400,"content_type":"text/html","bytes":3676,"sha256":"a2a23547c4444ef5a3d182376466e9184df374f0f8f9c0a0a6cdf68f7bcc01e1","final_host":"m.facebook.com","final_path":"/","ms":467,"identity":"Mobile browser","html":true},"gptbot":{"status":200,"content_type":"text/html","bytes":366909,"sha256":"c6c9f99d21bd1c494e215cb6f6dccda359ba63b4d0e06181096a2bdd6ad04743","final_host":"www.facebook.com","final_path":"/","ms":451,"identity":"GPTBot","html":true,"fp":"1eb59c9679d8cb2f"},"claudebot":{"status":200,"content_type":"text/html","bytes":366737,"sha256":"b10af0d169f38283485d305e14199e9cc2adf817d3537adab8742d2783351415","final_host":"www.facebook.com","final_path":"/","ms":478,"identity":"ClaudeBot","html":true,"fp":"1eb59c9679d8cb2f"},"oaisearch":{"status":200,"content_type":"text/html","bytes":366967,"sha256":"2fd8c5c57c3df2ffbaeee4f6109047a5497aa58caa496f7fcbcf785768961778","final_host":"www.facebook.com","final_path":"/","ms":372,"identity":"OAI-SearchBot","html":true,"fp":"1eb59c9679d8cb2f"},"perplexity":{"status":200,"content_type":"text/html","bytes":367821,"sha256":"5c6e4d095e54a44914f8a0bc743ae022f2031e57ecb18c5806c7cbb58a65dcdc","final_host":"www.facebook.com","final_path":"/","ms":370,"identity":"PerplexityBot","html":true,"fp":"1eb59c9679d8cb2f"}},"robots":{"status":200,"content_type":"text/plain","bytes":26917,"sha256":"7fc92d8059cb08a27e73f63d490ae62f4f0dd4d58eac0c9fca0dec452b0224bc","final_host":"facebook.com","final_path":"/robots.txt","ms":115,"html":false,"text":"# Notice: Collection of data on Facebook through automated means is\n# prohibited unless you have express written permission from Facebook\n# and may only be conducted for the limited purpose contained in said\n# permission.\n# All authorized user-agents listed on this page must comply with Meta’s\n# Automated Data Collection Terms available at:\n# https://www.facebook.com/legal/automated_data_collection_terms\nUser-agent: __usage_ai-training\nDisallow: /\n\nUser-agent: Amazonbot\nDisallow: /\n\nUser-agent: Applebot-Extended\nDisallow: /\n\nUser-agent: Brightbot\nDisallow: /\n\nUser-agent: ClaudeBot\nDisallow: /\n\nUser-agent: Google-Extended\nDisallow: /\n\nUser-agent: GPTBot\nDisallow: /\n\nUser-agent: PerplexityBot\nDisallow: /\n\nUser-agent: PetalBot\nDisallow: /\n\nUser-agent: Scrapy\nDisallow: /\n\nUser-agent: uptimerobot\nDisallow: /\n\nUser-agent: viberbot\nDisallow: /\n\nUser-agent: YaK\nDisallow: /\n\nUser-agent: Yandex\nDisallow: /\n\nUser-agent: Yeti\nDisallow: /\n\nUser-agent: Applebot\nDisallow: /*/plugins/*\nDisallow: /?*next=\nDisallow: /a/bz?\nDisallow: /ajax/\nDisallow: /album.php\nDisallow: /business/*&categories\nDisallow: /business/*?categories\nDisallow: /business/help/search*&query=\nDisallow: /business/help/search*?query=\nDisallow: /business/news*&tags\nDisallow: /business/news*?tags\nDisallow: /business/search*&q=\nDisallow: /business/search*?q=\nDisallow: /checkpoint/\nDisallow: /contact_importer/\nDisallow: /dialog/\nDisallow: /fbml/ajax/dialog/\nDisallow: /feeds/\nDisallow: /file_download.php\nDisallow: /job_application/\nDisallow: /l.php\nDisallow: /login.php*&next=\nDisallow: /login.php/?next=\nDisallow: /login.php?next=\nDisallow: /login/*&next=\nDisallow: /login/?next=\nDisallow: /login/device-based/regular/login/*&next=\nDisallow: /login/device-based/regular/login/?next=\nDisallow: /meta-ai-answers/\nDisallow: /moments_app/\nDisallow: /p.php\nDisallow: /photos.php\nDisallow: /plugins/\nDisallow: /share.php\nDisallow: /share/\nDisallow: /sharer.php\nDisallow: /sharer/\nDisallow: /tr/\nDisallow: /tr?\nDisallow: /ufi/reaction/profile/browser/\nDisallow: /x/oauth/\nAllow: /ajax/bootloader-endpoint/\nAllow: /ajax/pagelet/generic.php/PagePostsSectionPagelet\nAllow: /careers/\nAllow: /safetycheck/\n\nUser-agent: Bingbot\nDisallow: /*/plugins/*\nDisallow: /?*next=\nDisallow: /a/bz?\nDisallow: /ajax/\nDisallow: /album.php\nDisallow: /business/*&categories\nDisallow: /business/*?categories\nDisallow: /business/help/search*&query=\nDisallow: /business/help/search*?query=\nDisallow: /business/news*&tags\nDisallow: /business/news*?tags\nDisallow: /business/search*&q=\nDisallow: /business/search*?q=\nDisallow: /checkpoint/\nDisallow: /contact_importer/\nDisallow: /dialog/\nDisallow: /fbml/ajax/dialog/\nDisallow: /feeds/\nDisallow: /file_download.php\nDisallow: /job_application/\nDisallow: /l.php\nDisallow: /login.php*&next=\nDisallow: /login.php/?next=\nDisallow: /login.php?next=\nDisallow: /login/*&next=\nDisallow: /login/?next=\nDisallow: /login/device-based/regular/login/*&next=\nDisallow: /login/device-based/regular/login/?next=\nDisallow: /meta-ai-answers/\nDisallow: /moments_app/\nDisallow: /p.php\nDisallow: /photos.php\nDisallow: /plugins/\nDisallow: /share.php\nDisallow: /share/\nDisallow: /sharer.php\nDisallow: /sharer/\nDisallow: /tr/\nDisallow: /tr?\nDisallow: /ufi/reaction/profile/browser/\nDisallow: /x/oauth/\nAllow: /ajax/bootloader-endpoint/\nAllow: /ajax/pagelet/generic.php/PagePostsSectionPagelet\nAllow: /careers/\nAllow: /safetycheck/\n\nUser-agent: deepcrawl\nDisallow: /\nDisallow: /*/plugins/*\nDisallow: /?*next=\nDisallow: /a/bz?\nDisallow: /ajax/\nDisallow: /album.php\nDisallow: /business/*&categories\nDisallow: /business/*?categories\nDisallow: /business/help/search*&query=\nDisallow: /business/help/search*?query=\nDisallow: /business/news*&tags\nDisallow: /business/news*?tags\nDisallow: /business/search*&q=\nDisallow: /business/search*?q=\nDisallow: /checkpoint/\nDisallow: /contact_importer/\nDisallow: /dialog/\nDisallow: /fbml/ajax/dialog/\nDisallow: /feeds/\nDisallow: /file_download.php\nDisallow: /job_application/\nDisallow: /l.php\nDisallow: /login.php*&next=\nDisallow: /login.php/?next=\nDisallow: /login.php?next=\nDisallow: /login/*&next=\nDisallow: /login/?next=\nDisallow: /login/device-based/regular/login/*&next=\nDisallow: /login/device-based/regular/login/?next=\nDisallow: /meta-ai-answers/\nDisallow: /moments_app/\nDisallow: /p.php\nDisallow: /photos.php\nDisallow: /plugins/\nDisallow: /share.php\nDisallow: /share/\nDisallow: /sharer.php\nDisallow: /sharer/\nDisallow: /tr/\nDisallow: /tr?\nDisallow: /ufi/reaction/profile/browser/\nDisallow: /x/oauth/\nAllow: /ajax/bootloader-endpoint/\nAllow: /ajax/pagelet/generic.php/PagePostsSectionPagelet\nAllow: /business/\nAllow: /careers/\nAllow: /safetycheck/\n\nUser-agent: Discordbot\nDisallow: /*/plugins/*\nDisallow: /?*next=\nDisallow: /a/bz?\nDisallow: /ajax/\nDisallow: /album.php\nDisallow: /business/*&categories\nDisallow: /business/*?categories\nDisallow: /business/help/search*&query=\nDisallow: /business/help/search*?query=\nDisallow: /business/news*&tags\nDisallow: /business/news*?tags\nDisallow: /business/search*&q=\nDisallow: /business/search*?q=\nDisallow: /checkpoint/\nDisallow: /contact_importer/\nDisallow: /dialog/\nDisallow: /fbml/ajax/dialog/\nDisallow: /feeds/\nDisallow: /file_download.php\nDisallow: /job_application/\nDisallow: /l.php\nDisallow: /login.php*&next=\nDisallow: /login.php/?next=\nDisallow: /login.php?next=\nDisallow: /login/*&next=\nDisallow: /login/?next=\nDisallow: /login/device-based/regular/login/*&next=\nDisallow: /login/device-based/regular/login/?next=\nDisallow: /meta-ai-answers/\nDisallow: /moments_app/\nDisallow: /p.php\nDisallow: /photos.php\nDisallow: /plugins/\nDisallow: /share.php\nDisallow: /share/\nDisallow: /sharer.php\nDisallow: /sharer/\nDisallow: /tr/\nDisallow: /tr?\nDisallow: /ufi/reaction/profile/browser/\nDisallow: /x/oauth/\nAllow: /ajax/bootloader-endpoint/\nAllow: /ajax/pagelet/generic.php/PagePostsSectionPagelet\nAllow: /careers/\nAllow: /safetycheck/\n\nUser-agent: DuckDuckBot\nDisallow: /*/plugins/*\nDisallow: /?*next=\nDisallow: /a/bz?\nDisallow: /ajax/\nDisallow: /album.php\nDisallow: /business/*&categories\nDisallow: /business/*?categories\nDisallow: /business/help/search*&query=\nDisallow: /business/help/search*?query=\nDisallow: /business/news*&tags\nDisallow: /business/news*?tags\nDisallow: /business/search*&q=\nDisallow: /business/search*?q=\nDisallow: /checkpoint/\nDisallow: /contact_importer/\nDisallow: /dialog/\nDisallow: /fbml/ajax/dialog/\nDisallow: /feeds/\nDisallow: /file_download.php\nDisallow: /job_application/\nDisallow: /l.php\nDisallow: /login.php*&next=\nDisallow: /login.php/?next=\nDisallow: /login.php?next=\nDisallow: /login/*&next=\nDisallow: /login/?next=\nDisallow: /login/device-based/regular/login/*&next=\nDisallow: /login/device-based/regular/login/?next=\nDisallow: /meta-ai-answers/\nDisallow: /moments_app/\nDisallow: /p.php\nDisallow: /photos.php\nDisallow: /plugins/\nDisallow: /share.php\nDisallow: /share/\nDisallow: /sharer.php\nDisallow: /sharer/\nDisallow: /tr/\nDisallow: /tr?\nDisallow: /ufi/reaction/profile/browser/\nDisallow: /x/oauth/\nAllow: /ajax/bootloader-endpoint/\nAllow: /ajax/pagelet/generic.php/PagePostsSectionPagelet\nAllow: /careers/\nAllow: /safetycheck/\n\nUser-agent: facebookexternalhit\nDisallow: /*/plugins/*\nDisallow: /?*next=\nDisallow: /a/bz?\nDisallow: /ajax/\nDisallow: /album.php\nDisallow: /business/*&categories\nDisallow: /business/*?categories\nDisallow: /business/help/search*&query=\nDisallow: /business/help/search*?query=\nDisallow: /business/news*&tags\nDisallow: /business/news*?tags\nDisallow: /business/search*&q=\nDisallow: /business/search*?q=\nDisallow: /checkpoint/\nDisallow: /contact_importer/\nDisallow: /dialog/\nDisallow: /fbml/ajax/dialog/\nDisallow: /feeds/\nDisallow: /file_download.php\nDisallow: /job_application/\nDisallow: /l.php\nDisallow: /login.php*&next=\nDisallow: /login.php/?next=\nDisallow: /login.php?next=\nDisallow: /login/*&next=\nDisallow: /login/?next=\nDisallow: /login/device-based/regular/login/*&next=\nDisallow: /login/device-based/regular/login/?next=\nDisallow: /meta-ai-answers/\nDisallow: /moments_app/\nDisallow: /p.php\nDisallow: /photos.php\nDisallow: /plugins/\nDisallow: /share.php\nDisallow: /share/\nDisallow: /sharer.php\nDisallow: /sharer/\nDisallow: /tr/\nDisallow: /tr?\nDisallow: /ufi/reaction/profile/browser/\nDisallow: /x/oauth/\nAllow: /ajax/bootloader-endpoint/\nAllow: /ajax/pagelet/generic.php/PagePostsSectionPagelet\nAllow: /careers/\nAllow: /safetycheck/\n\nUser-agent: Googlebot\nDisallow: /*/plugins/*\nDisallow: /?*next=\nDisallow: /a/bz?\nDisallow: /ajax/\nDisallow: /album.php\nDisallow: /business/*&categories\nDisallow: /business/*?categories\nDisallow: /business/help/search*&query=\nDisallow: /business/help/search*?query=\nDisallow: /business/news*&tags\nDisallow: /business/news*?tags\nDisallow: /business/search*&q=\nDisallow: /business/search*?q=\nDisallow: /checkpoint/\nDisallow: /contact_importer/\nDisallow: /dialog/\nDisallow: /fbml/ajax/dialog/\nDisallow: /feeds/\nDisallow: /file_download.php\nDisallow: /job_application/\nDisallow: /l.php\nDisallow: /login.php*&next=\nDisallow: /login.php/?next=\nDisallow: /login.php?next=\nDisallow: /login/*&next=\nDisallow: /login/?next=\nDisallow: /login/device-based/regular/login/*&next=\nDisallow: /login/device-based/regular/login/?next=\nDisallow: /meta-ai-answers/\nDisallow: /moments_app/\nDisallow: /p.php\nDisallow: /photos.php\nDisallow: /plugins/\nDisallow: /share.php\nDisallow: /share/\nDisallow: /sharer.php\nDisallow: /sharer/\nDisallow: /tr/\nDisallow: /tr?\nDisallow: /ufi/reaction/profile/browser/\nDisallow: /x/oauth/\nAllow: /ajax/bootloader-endpoint/\nAllow: /ajax/pagelet/generic.php/PagePostsSectionPagelet\nAllow: /careers/\nAllow: /safetycheck/\n\nUser-agent: Googlebot-Image\nDisallow: /*/plugins/*\nDisallow: /?*next=\nDisallow: /a/bz?\nDisallow: /ajax/\nDisallow: /album.php\nDisallow: /business/*&categories\nDisallow: /business/*?categories\nDisallow: /business/help/search*&query=\nDisallow: /business/help/search*?query=\nDisallow: /business/news*&tags\nDisallow: /business/news*?tags\nDisallow: /business/search*&q=\nDisallow: /business/search*?q=\nDisallow: /checkpoint/\nDisallow: /contact_importer/\nDisallow: /dialog/\nDisallow: /fbml/ajax/dialog/\nDisallow: /feeds/\nDisallow: /file_download.php\nDisallow: /job_application/\nDisallow: /l.php\nDisallow: /login.php*&next=\nDisallow: /login.php/?next=\nDisallow: /login.php?next=\nDisallow: /login/*&next=\nDisallow: /login/?next=\nDisallow: /login/device-based/regular/login/*&next=\nDisallow: /login/device-based/regular/login/?next=\nDisallow: /map_tile.php\nDisallow: /meta-ai-answers/\nDisallow: /moments_app/\nDisallow: /p.php\nDisallow: /photos.php\nDisallow: /plugins/\nDisallow: /share.php\nDisallow: /share/\nDisallow: /sharer.php\nDisallow: /sharer/\nDisallow: /static_map.php\nDisallow: /tr/\nDisallow: /tr?\nDisallow: /ufi/reaction/profile/browser/\nDisallow: /x/oauth/\nAllow: /ajax/bootloader-endpoint/\nAllow: /ajax/pagelet/generic.php/PagePostsSectionPagelet\nAllow: /careers/\nAllow: /safetycheck/\n\nUser-agent: Googlebot-Video\nDisallow: /*/plugins/*\nDisallow: /?*next=\nDisallow: /a/bz?\nDisallow: /ajax/\nDisallow: /album.php\nDisallow: /business/*&categories\nDisallow: /business/*?categories\nDisallow: /business/help/search*&query=\nDisallow: /business/help/search*?query=\nDisallow: /business/news*&tags\nDisallow: /business/news*?tags\nDisallow: /business/search*&q=\nDisallow: /business/search*?q=\nDisallow: /checkpoint/\nDisallow: /contact_importer/\nDisallow: /dialog/\nDisallow: /fbml/ajax/dialog/\nDisallow: /feeds/\nDisallow: /file_download.php\nDisallow: /job_application/\nDisallow: /l.php\nDisallow: /login.php*&next=\nDisallow: /login.php/?next=\nDisallow: /login.php?next=\nDisallow: /login/*&next=\nDisallow: /login/?next=\nDisallow: /login/device-based/regular/login/*&next=\nDisallow: /login/device-based/regular/login/?next=\nDisallow: /meta-ai-answers/\nDisallow: /moments_app/\nDisallow: /p.php\nDisallow: /photos.php\nDisallow: /plugins/\nDisallow: /share.php\nDisallow: /share/\nDisallow: /sharer.php\nDisallow: /sharer/\nDisallow: /tr/\nDisallow: /tr?\nDisallow: /ufi/reaction/profile/browser/\nDisallow: /x/oauth/\nAllow: /ajax/bootloader-endpoint/\nAllow: /ajax/pagelet/generic.php/PagePostsSectionPagelet\nAllow: /careers/\nAllow: /safetycheck/\n\nUser-agent: Googlebot-News\nDisallow: /*/plugins/*\nDisallow: /?*next=\nDisallow: /a/bz?\nDisallow: /ajax/\nDisallow: /album.php\nDisallow: /business/*&categories\nDisallow: /business/*?categories\nDisallow: /business/help/search*&query=\nDisallow: /business/help/search*?query=\nDisallow: /business/news*&tags\nDisallow: /business/news*?tags\nDisallow: /business/search*&q=\nDisallow: /business/search*?q=\nDisallow: /checkpoint/\nDisallow: /contact_importer/\nDisallow: /dialog/\nDisallow: /fbml/ajax/dialog/\nDisallow: /feeds/\nDisallow: /file_download.php\nDisallow: /job_application/\nDisallow: /l.php\nDisallow: /login.php*&next=\nDisallow: /login.php/?next=\nDisallow: /login.php?next=\nDisallow: /login/*&next=\nDisallow: /login/?next=\nDisallow: /login/device-based/regular/login/*&next=\nDisallow: /login/device-based/regular/login/?next=\nDisallow: /meta-ai-answers/\nDisallow: /moments_app/\nDisallow: /p.php\nDisallow: /photos.php\nDisallow: /plugins/\nDisallow: /share.php\nDisallow: /share/\nDisallow: /sharer.php\nDisallow: /sharer/\nDisallow: /tr/\nDisallow: /tr?\nDisallow: /ufi/reaction/profile/browser/\nDisallow: /x/oauth/\nAllow: /ajax/bootloader-endpoint/\nAllow: /ajax/pagelet/generic.php/PagePostsSectionPagelet\nAllow: /careers/\nAllow: /safetycheck/\n\nUser-agent: Google-InspectionTool\nDisallow: /*/plugins/*\nDisallow: /?*next=\nDisallow: /a/bz?\nDisallow: /ajax/\nDisallow: /album.php\nDisallow: /business/*&categories\nDisallow: /business/*?categories\nDisallow: /business/help/search*&query=\nDisallow: /business/help/search*?query=\nDisallow: /business/news*&tags\nDisallow: /business/news*?tags\nDisallow: /business/search*&q=\nDisallow: /business/search*?q=\nDisallow: /checkpoint/\nDisallow: /contact_importer/\nDisallow: /dialog/\nDisallow: /fbml/ajax/dialog/\nDisallow: /feeds/\nDisallow: /file_download.php\nDisallow: /job_application/\nDisallow: /l.php\nDisallow: /login.php*&next=\nDisallow: /login.php/?next=\nDisallow: /login.php?next=\nDisallow: /login/*&next=\nDisallow: /login/?next=\nDisallow: /login/device-based/regular/login/*&next=\nDisallow: /login/device-based/regular/login/?next=\nDisallow: /meta-ai-answers/\nDisallow: /moments_app/\nDisallow: /p.php\nDisallow: /photos.php\nDisallow: /plugins/\nDisallow: /share.php\nDisallow: /share/\nDisallow: /sharer.php\nDisallow: /sharer/\nDisallow: /tr/\nDisallow: /tr?\nDisallow: /ufi/reaction/profile/browser/\nDisallow: /x/oauth/\nAllow: /ajax/bootloader-endpoint/\nAllow: /ajax/pagelet/generic.php/PagePostsSectionPagelet\nAllow: /careers/\nAllow: /safetycheck/\n\nUser-agent: lumar\nDisallow: /\nDisallow: /*/plugins/*\nDisallow: /?*next=\nDisallow: /a/bz?\nDisallow: /ajax/\nDisallow: /album.php\nDisallow: /business/*&categories\nDisallow: /business/*?categories\nDisallow: /business/help/search*&query=\nDisallow: /business/help/search*?query=\nDisallow: /business/news*&tags\nDisallow: /business/news*?tags\nDisallow: /business/search*&q=\nDisallow: /business/search*?q=\nDisallow: /checkpoint/\nDisallow: /contact_importer/\nDisallow: /dialog/\nDisallow: /fbml/ajax/dialog/\nDisallow: /feeds/\nDisallow: /file_download.php\nDisallow: /job_application/\nDisallow: /l.php\nDisallow: /login.php*&next=\nDisallow: /login.php/?next=\nDisallow: /login.php?next=\nDisallow: /login/*&next=\nDisallow: /login/?next=\nDisallow: /login/device-based/regular/login/*&next=\nDisallow: /login/device-based/regular/login/?next=\nDisallow: /meta-ai-answers/\nDisallow: /moments_app/\nDisallow: /p.php\nDisallow: /photos.php\nDisallow: /plugins/\nDisallow: /share.php\nDisallow: /share/\nDisallow: /sharer.php\nDisallow: /sharer/\nDisallow: /tr/\nDisallow: /tr?\nDisallow: /ufi/reaction/profile/browser/\nDisallow: /x/oauth/\nAllow: /ajax/bootloader-endpoint/\nAllow: /ajax/pagelet/generic.php/PagePostsSectionPagelet\nAllow: /business/\nAllow: /careers/\nAllow: /safetycheck/\n\nUser-agent: LinkedInBot\nDisallow: /*/plugins/*\nDisallow: /?*next=\nDisallow: /a/bz?\nDisallow: /ajax/\nDisallow: /album.php\nDisallow: /business/*&categories\nDisallow: /business/*?categories\nDisallow: /business/help/search*&query=\nDisallow: /business/help/search*?query=\nDisallow: /business/news*&tags\nDisallow: /business/news*?tags\nDisallow: /business/search*&q=\nDisallow: /business/search*?q=\nDisallow: /checkpoint/\nDisallow: /contact_importer/\nDisallow: /dialog/\nDisallow: /fbml/ajax/dialog/\nDisallow: /feeds/\nDisallow: /file_download.php\nDisallow: /job_application/\nDisallow: /l.php\nDisallow: /login.php*&next=\nDisallow: /login.php/?next=\nDisallow: /login.php?next=\nDisallow: /login/*&next=\nDisallow: /login/?next=\nDisallow: /login/device-based/regular/login/*&next=\nDisallow: /login/device-based/regular/login/?next=\nDisallow: /meta-ai-answers/\nDisallow: /moments_app/\nDisallow: /p.php\nDisallow: /photos.php\nDisallow: /plugins/\nDisallow: /share.php\nDisallow: /share/\nDisallow: /sharer.php\nDisallow: /sharer/\nDisallow: /tr/\nDisallow: /tr?\nDisallow: /ufi/reaction/profile/browser/\nDisallow: /x/oauth/\nAllow: /ajax/bootloader-endpoint/\nAllow: /ajax/pagelet/generic.php/PagePostsSectionPagelet\nAllow: /careers/\nAllow: /safetycheck/\n\nUser-agent: msnbot\nDisallow: /*/plugins/*\nDisallow: /?*next=\nDisallow: /a/bz?\nDisallow: /ajax/\nDisallow: /album.php\nDisallow: /business/*&categories\nDisallow: /business/*?categories\nDisallow: /business/help/search*&query=\nDisallow: /business/help/search*?query=\nDisallow: /business/news*&tags\nDisallow: /business/news*?tags\nDisallow: /business/search*&q=\nDisallow: /business/search*?q=\nDisallow: /checkpoint/\nDisallow: /contact_importer/\nDisallow: /dialog/\nDisallow: /fbml/ajax/dialog/\nDisallow: /feeds/\nDisallow: /file_download.php\nDisallow: /job_application/\nDisallow: /l.php\nDisallow: /login.php*&next=\nDisallow: /login.php/?next=\nDisallow: /login.php?next=\nDisallow: /login/*&next=\nDisallow: /login/?next=\nDisallow: /login/device-based/regular/login/*&next=\nDisallow: /login/device-based/regular/login/?next=\nDisallow: /meta-ai-answers/\nDisallow: /moments_app/\nDisallow: /p.php\nDisallow: /photos.php\nDisallow: /plugins/\nDisallow: /share.php\nDisallow: /share/\nDisallow: /sharer.php\nDisallow: /sharer/\nDisallow: /tr/\nDisallow: /tr?\nDisallow: /ufi/reaction/profile/browser/\nDisallow: /x/oauth/\nAllow: /ajax/bootloader-endpoint/\nAllow: /ajax/pagelet/generic.php/PagePostsSectionPagelet\nAllow: /careers/\nAllow: /safetycheck/\n\nUser-agent: Pinterestbot\nDisallow: /*/plugins/*\nDisallow: /?*next=\nDisallow: /a/bz?\nDisallow: /ajax/\nDisallow: /album.php\nDisallow: /business/*&categories\nDisallow: /business/*?categories\nDisallow: /business/help/search*&query=\nDisallow: /business/help/search*?query=\nDisallow: /business/news*&tags\nDisallow: /business/news*?tags\nDisallow: /business/search*&q=\nDisallow: /business/search*?q=\nDisallow: /checkpoint/\nDisallow: /contact_importer/\nDisallow: /dialog/\nDisallow: /fbml/ajax/dialog/\nDisallow: /feeds/\nDisallow: /file_download.php\nDisallow: /job_application/\nDisallow: /l.php\nDisallow: /login.php*&next=\nDisallow: /login.php/?next=\nDisallow: /login.php?next=\nDisallow: /login/*&next=\nDisallow: /login/?next=\nDisallow: /login/device-based/regular/login/*&next=\nDisallow: /login/device-based/regular/login/?next=\nDisallow: /meta-ai-answers/\nDisallow: /moments_app/\nDisallow: /p.php\nDisallow: /photos.php\nDisallow: /plugins/\nDisallow: /share.php\nDisallow: /share/\nDisallow: /sharer.php\nDisallow: /sharer/\nDisallow: /tr/\nDisallow: /tr?\nDisallow: /ufi/reaction/profile/browser/\nDisallow: /x/oauth/\nAllow: /ajax/bootloader-endpoint/\nAllow: /ajax/pagelet/generic.php/PagePostsSectionPagelet\nAllow: /careers/\nAllow: /safetycheck/\n\nUser-agent: Screaming Frog SEO Spider\nDisallow: /*/plugins/*\nDisallow: /?*next=\nDisallow: /a/bz?\nDisallow: /ajax/\nDisallow: /album.php\nDisallow: /business/*&categories\nDisallow: /business/*?categories\nDisallow: /business/help/search*&query=\nDisallow: /business/help/search*?query=\nDisallow: /business/news*&tags\nDisallow: /business/news*?tags\nDisallow: /business/search*&q=\nDisallow: /business/search*?q=\nDisallow: /checkpoint/\nDisallow: /contact_importer/\nDisallow: /dialog/\nDisallow: /fbml/ajax/dialog/\nDisallow: /feeds/\nDisallow: /file_download.php\nDisallow: /job_application/\nDisallow: /l.php\nDisallow: /login.php*&next=\nDisallow: /login.php/?next=\nDisallow: /login.php?next=\nDisallow: /login/*&next=\nDisallow: /login/?next=\nDisallow: /login/device-based/regular/login/*&next=\nDisallow: /login/device-based/regular/login/?next=\nDisallow: /meta-ai-answers/\nDisallow: /moments_app/\nDisallow: /p.php\nDisallow: /photos.php\nDisallow: /plugins/\nDisallow: /share.php\nDisallow: /share/\nDisallow: /sharer.php\nDisallow: /sharer/\nDisallow: /tr/\nDisallow: /tr?\nDisallow: /ufi/reaction/profile/browser/\nDisallow: /x/oauth/\nAllow: /ajax/bootloader-endpoint/\nAllow: /ajax/pagelet/generic.php/PagePostsSectionPagelet\nAllow: /careers/\nAllow: /safetycheck/\n\nUser-agent: seznambot\nDisallow: /*/plugins/*\nDisallow: /?*next=\nDisallow: /a/bz?\nDisallow: /ajax/\nDisallow: /album.php\nDisallow: /business/*&categories\nDisallow: /business/*?categories\nDisallow: /business/help/search*&query=\nDisallow: /business/help/search*?query=\nDisallow: /business/news*&tags\nDisallow: /business/news*?tags\nDisallow: /business/search*&q=\nDisallow: /business/search*?q=\nDisallow: /checkpoint/\nDisallow: /contact_importer/\nDisallow: /dialog/\nDisallow: /fbml/ajax/dialog/\nDisallow: /feeds/\nDisallow: /file_download.php\nDisallow: /job_application/\nDisallow: /l.php\nDisallow: /login.php*&next=\nDisallow: /login.php/?next=\nDisallow: /login.php?next=\nDisallow: /login/*&next=\nDisallow: /login/?next=\nDisallow: /login/device-based/regular/login/*&next=\nDisallow: /login/device-based/regular/login/?next=\nDisallow: /meta-ai-answers/\nDisallow: /moments_app/\nDisallow: /p.php\nDisallow: /photos.php\nDisallow: /plugins/\nDisallow: /share.php\nDisallow: /share/\nDisallow: /sharer.php\nDisallow: /sharer/\nDisallow: /tr/\nDisallow: /tr?\nDisallow: /ufi/reaction/profile/browser/\nDisallow: /x/oauth/\nAllow: /ajax/bootloader-endpoint/\nAllow: /ajax/pagelet/generic.php/PagePostsSectionPagelet\nAllow: /careers/\nAllow: /safetycheck/\n\nUser-agent: Slurp\nDisallow: /*/plugins/*\nDisallow: /?*next=\nDisallow: /a/bz?\nDisallow: /ajax/\nDisallow: /album.php\nDisallow: /business/*&categories\nDisallow: /business/*?categories\nDisallow: /business/help/search*&query=\nDisallow: /business/help/search*?query=\nDisallow: /business/news*&tags\nDisallow: /business/news*?tags\nDisallow: /business/search*&q=\nDisallow: /business/search*?q=\nDisallow: /checkpoint/\nDisallow: /contact_importer/\nDisallow: /dialog/\nDisallow: /fbml/ajax/dialog/\nDisallow: /feeds/\nDisallow: /file_download.php\nDisallow: /job_application/\nDisallow: /l.php\nDisallow: /login.php*&next=\nDisallow: /login.php/?next=\nDisallow: /login.php?next=\nDisallow: /login/*&next=\nDisallow: /login/?next=\nDisallow: /login/device-based/regular/login/*&next=\nDisallow: /login/device-based/regular/login/?next=\nDisallow: /meta-ai-answers/\nDisallow: /moments_app/\nDisallow: /p.php\nDisallow: /photos.php\nDisallow: /plugins/\nDisallow: /share.php\nDisallow: /share/\nDisallow: /sharer.php\nDisallow: /sharer/\nDisallow: /tr/\nDisallow: /tr?\nDisallow: /ufi/reaction/profile/browser/\nDisallow: /x/oauth/\nAllow: /ajax/bootloader-endpoint/\nAllow: /ajax/pagelet/generic.php/PagePostsSectionPagelet\nAllow: /careers/\nAllow: /safetycheck/\n\nUser-agent: teoma\nDisallow: /*/plugins/*\nDisallow: /?*next=\nDisallow: /a/bz?\nDisallow: /ajax/\nDisallow: /album.php\nDisallow: /business/*&categories\nDisallow: /business/*?categories\nDisallow: /business/help/search*&query=\nDisallow: /business/help/search*?query=\nDisallow: /business/news*&tags\nDisallow: /business/news*?tags\nDisallow: /business/search*&q=\nDisallow: /business/search*?q=\nDisallow: /checkpoint/\nDisallow: /contact_importer/\nDisallow: /dialog/\nDisallow: /fbml/ajax/dialog/\nDisallow: /feeds/\nDisallow: /file_download.php\nDisallow: /job_application/\nDisallow: /l.php\nDisallow: /login.php*&next=\nDisallow: /login.php/?next=\nDisallow: /login.php?next=\nDisallow: /login/*&next=\nDisallow: /login/?next=\nDisallow: /login/device-based/regular/login/*&next=\nDisallow: /login/device-based/regular/login/?next=\nDisallow: /meta-ai-answers/\nDisallow: /moments_app/\nDisallow: /p.php\nDisallow: /photos.php\nDisallow: /plugins/\nDisallow: /share.php\nDisallow: /share/\nDisallow: /sharer.php\nDisallow: /sharer/\nDisallow: /tr/\nDisallow: /tr?\nDisallow: /ufi/reaction/profile/browser/\nDisallow: /x/oauth/\nAllow: /ajax/bootloader-endpoint/\nAllow: /ajax/pagelet/generic.php/PagePostsSectionPagelet\nAllow: /careers/\nAllow: /safetycheck/\n\nUser-agent: TelegramBot\nDisallow: /*/plugins/*\nDisallow: /?*next=\nDisallow: /a/bz?\nDisallow: /ajax/\nDisallow: /album.php\nDisallow: /business/*&categories\nDisallow: /business/*?categories\nDisallow: /business/help/search*&query=\nDisallow: /business/help/search*?query=\nDisallow: /business/news*&tags\nDisallow: /business/news*?tags\nDisallow: /business/search*&q=\nDisallow: /business/search*?q=\nDisallow: /checkpoint/\nDisallow: /contact_importer/\nDisallow: /dialog/\nDisallow: /fbml/ajax/dialog/\nDisallow: /feeds/\nDisallow: /file_download.php\nDisallow: /job_application/\nDisallow: /l.php\nDisallow: /login.php*&next=\nDisallow: /login.php/?next=\nDisallow: /login.php?next=\nDisallow: /login/*&next=\nDisallow: /login/?next=\nDisallow: /login/device-based/regular/login/*&next=\nDisallow: /login/device-based/regular/login/?next=\nDisallow: /meta-ai-answers/\nDisallow: /moments_app/\nDisallow: /p.php\nDisallow: /photos.php\nDisallow: /plugins/\nDisallow: /share.php\nDisallow: /share/\nDisallow: /sharer.php\nDisallow: /sharer/\nDisallow: /tr/\nDisallow: /tr?\nDisallow: /ufi/reaction/profile/browser/\nDisallow: /x/oauth/\nAllow: /ajax/bootloader-endpoint/\nAllow: /ajax/pagelet/generic.php/PagePostsSectionPagelet\nAllow: /careers/\nAllow: /safetycheck/\n\nUser-agent: Twitterbot\nDisallow: /*/plugins/*\nDisallow: /?*next=\nDisallow: /a/bz?\nDisallow: /ajax/\nDisallow: /album.php\nDisallow: /business/*&categories\nDisallow: /business/*?categories\nDisallow: /business/help/search*&query=\nDisallow: /business/help/search*?query=\nDisallow: /business/news*&tags\nDisallow: /business/news*?tags\nDisallow: /business/search*&q=\nDisallow: /business/search*?q=\nDisallow: /checkpoint/\nDisallow: /contact_importer/\nDisallow: /dialog/\nDisallow: /fbml/ajax/dialog/\nDisallow: /feeds/\nDisallow: /file_download.php\nDisallow: /job_application/\nDisallow: /l.php\nDisallow: /login.php*&next=\nDisallow: /login.php/?next=\nDisallow: /login.php?next=\nDisallow: /login/*&next=\nDisallow: /login/?next=\nDisallow: /login/device-based/regular/login/*&next=\nDisallow: /login/device-based/regular/login/?next=\nDisallow: /meta-ai-answers/\nDisallow: /moments_app/\nDisallow: /p.php\nDisallow: /photos.php\nDisallow: /plugins/\nDisallow: /share.php\nDisallow: /share/\nDisallow: /sharer.php\nDisallow: /sharer/\nDisallow: /tr/\nDisallow: /tr?\nDisallow: /ufi/reaction/profile/browser/\nDisallow: /x/oauth/\nAllow: /ajax/bootloader-endpoint/\nAllow: /ajax/pagelet/generic.php/PagePostsSectionPagelet\nAllow: /careers/\nAllow: /safetycheck/\n\nUser-agent: *\nDisallow: /\n\n\n","agents":{"GPTBot":false,"OAI-SearchBot":false,"ChatGPT-User":false,"ClaudeBot":false,"Claude-SearchBot":false,"PerplexityBot":false,"Google-Extended":false,"Applebot-Extended":false,"CCBot":false,"Bytespider":false,"Meta-ExternalAgent":false,"Amazonbot":false}},"sitemap":null,"files":null,"derived":{"reachable":false,"home_class":"refused_us","robots_served":false,"robots_sitemap":false,"content_signal":false,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":[],"blocks_any_ai":false,"files":{"llms_txt":false,"agents_md":false,"agent_skills":false,"mcp_card":false,"api_catalog":false,"a2a_card":false},"any_capability":false,"ai_refused":[],"ai_thin":[],"rings":{"mcp_declared":false,"mcp_live":false,"mcp_auth":false,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":3,"fh":"m.facebook.com","fp_other_host":[]},"derive_v":5,"frame":"top1k","shard":2},{"v":1,"kind":"crawlcheck-census-record","run":43,"domain":"gstatic.com","rank":4,"measured_at":"2026-11-13T07:59:19.049Z","home":{"browser":{"status":404,"content_type":"text/html","bytes":1561,"sha256":"23926e9185d8d43c02807a838ffb373cc1977726094a4e46807c66ada9dd7660","final_host":"www.gstatic.com","final_path":"/","ms":85,"identity":"Mobile browser","html":true},"gptbot":{"status":404,"content_type":"text/html","bytes":1561,"sha256":"23926e9185d8d43c02807a838ffb373cc1977726094a4e46807c66ada9dd7660","final_host":"www.gstatic.com","final_path":"/","ms":70,"identity":"GPTBot","html":true},"claudebot":{"status":404,"content_type":"text/html","bytes":1561,"sha256":"23926e9185d8d43c02807a838ffb373cc1977726094a4e46807c66ada9dd7660","final_host":"www.gstatic.com","final_path":"/","ms":85,"identity":"ClaudeBot","html":true},"oaisearch":{"status":404,"content_type":"text/html","bytes":1561,"sha256":"23926e9185d8d43c02807a838ffb373cc1977726094a4e46807c66ada9dd7660","final_host":"www.gstatic.com","final_path":"/","ms":83,"identity":"OAI-SearchBot","html":true},"perplexity":{"status":404,"content_type":"text/html","bytes":1561,"sha256":"23926e9185d8d43c02807a838ffb373cc1977726094a4e46807c66ada9dd7660","final_host":"www.gstatic.com","final_path":"/","ms":71,"identity":"PerplexityBot","html":true}},"robots":{"status":200,"content_type":"text/plain","bytes":5359,"sha256":"413ae0b7d04ab2b4fea5bdcfcbec7456fed252ef8fbe7182cd9b1ed52a07524e","final_host":"www.gstatic.com","final_path":"/robots.txt","ms":87,"html":false,"text":"User-agent: *\nAllow: /_/aihub_pod/\nAllow: /_/alkali-learn/\nAllow: /_/apps-viewer/\nAllow: /_/atari/\nAllow: /_/boq/\nAllow: /_/boq-deepmind-visualization/\nAllow: /_/boq-play/\nAllow: /_/boq-search/\nAllow: /_/boutique/\nAllow: /_/buganizer/\nAllow: /_/compare-banking/\nAllow: /_/compare-insurance/\nAllow: /_/fi/\nAllow: /_/fonts/\nAllow: /_/mss/boq-bard-web/\nAllow: /_/mss/boq-chrome-cloudcast/\nAllow: /_/mss/boq-cloud-ai-agentis/\nAllow: /_/mss/boq-cloud-client/\nAllow: /_/mss/boq-cloud-web3-portal/\nAllow: /_/mss/boq-deepmind-visualization/\nAllow: /_/mss/boq-deepmind-alphafold-server/\nAllow: /_/mss/boq-discover-labs/\nAllow: /_/mss/boq-gdp/\nAllow: /_/mss/boq-geo/\nAllow: /_/mss/boq-home-enterprise/\nAllow: /_/mss/boq-makersuite/\nAllow: /_/mss/boq-nbu-monetization/\nAllow: /_/mss/boq-search/\nAllow: /_/mss/boq-search-engagement-post/\nAllow: /_/mss/boq-rsvp/\nAllow: /_/mss/boq-travel/\nAllow: /_/mss/boq-identity/\nAllow: /_/mss/boq-labs-tailwind/\nAllow: /_/pantheon/\nAllow: /_/pantheon-platform/\nAllow: /_/play/\nAllow: /_/pog/\nAllow: /_/cloudsourcetools/\nAllow: /_/devopsconsole/\nAllow: /_/tfhub/\nAllow: /_/transparencyreport/\nAllow: /_/gstore/\nAllow: /_/ain/_/js/\nAllow: /ac/\nAllow: /actions/news/rss/\nAllow: /ads/learn/\nAllow: /ads/local/\nAllow: /ads/gpa/vidgen/\nAllow: /ads/research/\nAllow: /ads/ubaq/\nAllow: /agentis/omnichannel/public/\nAllow: /aistudio/*favicon*.png$\nAllow: /aistudio-static/\nAllow: /ai-sandbox-internal/generated_images/\nAllow: /aihub/tfhub/\nAllow: /aitestkitchen/website/flow/applets/\nAllow: /aitestkitchen/website/flow/social/\nAllow: /alkali/\nAllow: /alphafoldserver/\nAllow: /amphtml/stamp/ucp/post-testing/\nAllow: /android/market/\nAllow: /android/market_images/\nAllow: /antigravity/logos/\nAllow: /atari/images/public/\nAllow: /audio_news/\nAllow: /awmp/data/images/\nAllow: /bfe/apps/website/\nAllow: /boutique/sitemaps/\nAllow: /bricks/image/\nAllow: /compare/\nAllow: /chrome/crlset/\nAllow: /chrome/webstore/images/\nAllow: /chromoting/chromoting_logo_*.png$\nAllow: /chromoting/favicon.ico$\nAllow: /cloud-launcher/sitemap/\nAllow: /cloudsourcetools/images/\nAllow: /commercesearch/images/\nAllow: /communities/sitemap/\nAllow: /copy-conductor/crawled/\nAllow: /covid19/mobility/\nAllow: /creatorprofile/branding/\nAllow: /culturalinstitute/images/\nAllow: /culturalinstitute/sitemaps/\nAllow: /culturalinstitute/stella/\nAllow: /ct/log_list/\nAllow: /delight/\nAllow: /devopsconsole/images/\nAllow: /devrel-devsite/\nAllow: /docs/forms/social/\nAllow: /docs/socialmarkup/\nAllow: /dictionary/static/sitemaps/\nAllow: /droidguard/\nAllow: /earth/social/\nAllow: /earth/00-favicon.ico\nAllow: /enterprise-partner-search/sitemaps/\nAllow: /external_hosted/\nAllow: /fi/web/\nAllow: /flights/\nAllow: /flutter-canvaskit/\nAllow: /foodordering/\nAllow: /frommers/\nAllow: /gb/images/\nAllow: /glass/images/commerce/\nAllow: /gpay/social/\nAllow: /growth-ime/\nAllow: /gsx/\nAllow: /gtp/stir/\nAllow: /guidebooks/\nAllow: /GoogleInternetAuthority/\nAllow: /h5games_gamecenter/images/\nAllow: /healthricherkp/illustrations/\nAllow: /healthricherkp/cropped/\nAllow: /hex/\nAllow: /history/static/\nAllow: /identity/accountsettingsactivityweb/\nAllow: /identity/accountsettingssyaf/\nAllow: /images\nAllow: /inproduct_help/chatsupport/\nAllow: /keep/\nAllow: /lamda/images/\nAllow: /labs-code/\nAllow: /local/donut/\nAllow: /mapspro/\nAllow: /marketing-cms/\nAllow: /moderator/static/ux/\nAllow: /modern-formats-eng/ucp/framework_logging/\nAllow: /modern-formats-eng/ucp/*/client/\nAllow: /modern-formats-eng/ucp/*/csr/\nAllow: /modern-formats-eng/ucp/*/dev/\nAllow: /modern-formats-eng/ucp/static/framework_logging/\nAllow: /modern-formats-eng/ucp/static/*/client/\nAllow: /modern-formats-eng/ucp/static/*/csr/\nAllow: /modern-formats-eng/ucp/static/*/dev/\nAllow: /mtcs/cosigners/\nAllow: /nbubumblebee/\nAllow: /og/\nAllow: /ohttp_gateway/\nAllow: /onebox/media/sports/logos/\nAllow: /onebox/media/sports/photos/soccer/wc2018/\nAllow: /onebox/sports/logos/\nAllow: /onebox/sports/logos/sports_types/png/\nAllow: /pagespeed/insights/ui/logo/\nAllow: /pantheon/\nAllow: /pantheon-platform/\nAllow: /patent-search/frontend/\nAllow: /pay/images/merchant_logos/\nAllow: /pay/images/program_logos/\nAllow: /paygo/\nAllow: /play/store/web/\nAllow: /play/surveys/serp/\nAllow: /podcasts/test-podcast/rss/\nAllow: /politics/e/sharing/\nAllow: /prettyearth/assets/\nAllow: /psb/udc/\nAllow: /rsvp/images/\nAllow: /s2/sitemaps/\nAllow: /search-ar-dev/\nAllow: /search-console/scfe/\nAllow: /share\nAllow: /shopping-content-api/product_images/\nAllow: /shopping-organic/price_validation/\nAllow: /shopping/shoppingpage/casa/\nAllow: /shopstream/\nAllow: /sites/\nAllow: /sitemaps/\nAllow: /social/photosui/images/\nAllow: /social/photosui/sitemap/\nAllow: /stadia/gamers/\nAllow: /store/images/favicons/\nAllow: /subscriptions/\nAllow: /support/content/\nAllow: /system1/images/\nAllow: /travel-frontend/images/\nAllow: /finance/favicon/\nAllow: /travel-vacations/\nAllow: /touringbird/\nAllow: /trends/\nAllow: /trends_tpt/\nAllow: /trustedstores/images/\nAllow: /trustedstores/js/\nAllow: /tv/thumb/\nAllow: /ui/\nAllow: /violet-web/previewer/\nAllow: /vr/*\nAllow: /vsi/*.pdf$\nAllow: /wondermissive/interest/\nAllow: /youtube/fonts/\nAllow: /youtube/img/branding/favicon/\nAllow: /youtube/img/promos/\nAllow: /icons/\nAllow: /js/\nAllow: /ss/\nAllow: /*.js\nAllow: /*.css\nAllow: /*.svg\nAllow: /buyside-e2e-monitor/\nAllow: /cgc/*.ico\nAllow: /cgc/*.png\nAllow: /cgc/*.svg\n\nDisallow: /\n","agents":{"GPTBot":false,"OAI-SearchBot":false,"ChatGPT-User":false,"ClaudeBot":false,"Claude-SearchBot":false,"PerplexityBot":false,"Google-Extended":false,"Applebot-Extended":false,"CCBot":false,"Bytespider":false,"Meta-ExternalAgent":false,"Amazonbot":false}},"sitemap":null,"files":null,"derived":{"reachable":false,"home_class":"other","robots_served":false,"robots_sitemap":false,"content_signal":false,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":[],"blocks_any_ai":false,"files":{"llms_txt":false,"agents_md":false,"agent_skills":false,"mcp_card":false,"api_catalog":false,"a2a_card":false},"any_capability":false,"ai_refused":[],"ai_thin":[],"rings":{"mcp_declared":false,"mcp_live":false,"mcp_auth":false,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":3,"fh":"www.gstatic.com","fp_other_host":[]},"derive_v":5,"frame":"top1k","shard":3},{"v":1,"kind":"crawlcheck-census-record","run":43,"domain":"amazonaws.com","rank":5,"measured_at":"2026-11-13T08:05:49.397Z","home":{"browser":{"status":200,"content_type":"text/html","bytes":506736,"sha256":"8e9ce02410290e8ae2d3fd38f3de61df9d979d4ec9a9537a4696a8b6272d24f4","final_host":"aws.amazon.com","final_path":"/","ms":424,"identity":"Mobile browser","html":true,"fp":"138fbbc185e891da"},"gptbot":{"status":200,"content_type":"text/html","bytes":506736,"sha256":"d67825a59e5778e3dc63d833449a3ee0ec5c8d326deb498a8c741df27a22e4a1","final_host":"aws.amazon.com","final_path":"/","ms":413,"identity":"GPTBot","html":true,"fp":"138fbbc185e891da"},"claudebot":{"status":200,"content_type":"text/html","bytes":506736,"sha256":"d9557e5d1573326db93c1e4e2cc5cbaeb72956e4101eb146df3db53016480a7d","final_host":"aws.amazon.com","final_path":"/","ms":413,"identity":"ClaudeBot","html":true,"fp":"138fbbc185e891da"},"oaisearch":{"status":200,"content_type":"text/html","bytes":506736,"sha256":"b3a5857aa4e815c1d6472e21b185f012dd5354715ae9dfed7a3cbcf2908437ca","final_host":"aws.amazon.com","final_path":"/","ms":403,"identity":"OAI-SearchBot","html":true,"fp":"138fbbc185e891da"},"perplexity":{"status":200,"content_type":"text/html","bytes":506736,"sha256":"4ac7c5f4a1c6fb747632e65022ce91309ba51919f60dc79647856e7b7219ba79","final_host":"aws.amazon.com","final_path":"/","ms":427,"identity":"PerplexityBot","html":true,"fp":"138fbbc185e891da"}},"robots":{"status":200,"content_type":"text/plain","bytes":13879,"sha256":"a0223b0119cce646eab6f085445bbce4224747c91153e245e4d2031ce56da39e","final_host":"aws.amazon.com","final_path":"/robots.txt","ms":380,"html":false,"text":"User-agent: AdsBot-Google\r\nUser-agent: AdsBot-Google-Mobile\r\nAllow: /pm/\r\nDisallow: /alexaforbusiness/\r\nDisallow: /blogs/\r\nDisallow: /*/blogs/\r\nDisallow: /solutions/case-studies/\r\nDisallow: /*/solutions/case-studies/\r\nAllow: /solutions/case-studies/sustainability-emea/\r\nAllow: /*/solutions/case-studies/sustainability-emea/\r\nAllow: /solutions/case-studies/innovators/innovators-eu/\r\nAllow: /*/solutions/case-studies/innovators/innovators-eu/\r\nAllow: /solutions/case-studies/generative-ai-ekacare/\r\nAllow: /cn/solutions/case-studies/generative-ai-ekacare/\r\nAllow: /solutions/case-studies/generative-ai-mindtickle/\r\nAllow: /cn/solutions/case-studies/generative-ai-mindtickle/\r\nAllow: /solutions/case-studies/generative-ai-invideo/\r\nAllow: /solutions/case-studies/generative-ai-mitsui/\r\nAllow: /solutions/case-studies/generative-ai-national-australia-bank/\r\nAllow: /solutions/case-studies/generative-ai-ntu/\r\nAllow: /solutions/case-studies/generative-ai-qarc/\r\nAllow: /solutions/case-studies/generative-ai-swimming-australia/\r\nAllow: /cn/solutions/case-studies/generative-ai-swimming-australia/\r\nAllow: /solutions/case-studies/generative-ai-thematic/\r\nAllow: /cn/solutions/case-studies/generative-ai-thematic/\r\nAllow: /solutions/case-studies/generative-ai-turing/\r\nAllow: /solutions/case-studies/generative-ai-stockmark/   \r\nAllow: /*/solutions/case-studies/generative-ai-turing/\r\nAllow: /*/solutions/case-studies/generative-ai-stockmark/   \r\nAllow: /solutions/case-studies/generative-ai-starhub/    \r\nAllow: /solutions/case-studies/generative-ai-bolttech/\r\nAllow: /*/solutions/case-studies/generative-ai-starhub/    \r\nAllow: /*/solutions/case-studies/generative-ai-bolttech/\nAllow: /solutions/case-studies/nba-migration-case-study/\r\n\r\nUser-agent: *\r\n# Block old What's New posts to preserve crawl budget for current content\r\nDisallow: /about-aws/whats-new/200\r\nDisallow: /about-aws/whats-new/201\r\nDisallow: /about-aws/whats-new/2020\r\nDisallow: /about-aws/whats-new/2021\r\nDisallow: /about-aws/whats-new/2022\r\nDisallow: /about-aws/whats-new/2023\r\nDisallow: /about-aws/whats-new/2024\n# Block tag\nDisallow: /*/blogs/*/tag/\nDisallow: /blogs/*/tag/\n\r\n# Other\r\nDisallow: */confirmation/\r\nDisallow: */registration-confirmation/\r\nDisallow: /about-aws/launch-a-website-in-minutes-tutorial/\r\nDisallow: /about-aws/media-coverage/2011/08/17/rhat-support-for-sap-solutions-running-on-rhel-on-amazon-ec2/\r\nDisallow: /activate/accelerators\r\nDisallow: /activate/accelerators/*\r\nDisallow: /activate/accelerators/confirmation/\r\nDisallow: /activate/china-portfolio-signup/\r\nDisallow: /activate/cloudkata/\r\nDisallow: /activate/event/\r\nDisallow: /activate/event/*\r\nDisallow: /activate/events/*\r\nDisallow: /activate/events/hact/\r\nDisallow: /activate/hackathons\r\nDisallow: /activate/hackathons/*\r\nDisallow: /activate/harvard-innovation-lab-students-2013/\r\nDisallow: /activate/harvard-innovation-lab-vip-teams-2013/\r\nDisallow: /activate/hsw/confirmation/\r\nDisallow: /activate/loft/\r\nDisallow: /activate/self-starters/confirmation/\r\nDisallow: /activate/wdm/*\r\nDisallow: /alexa/*\r\nDisallow: /alexa-top-sites/*\r\nDisallow: /alexa-web-search/*\r\nDisallow: /apac/aws-credits-confirmation/\r\nDisallow: /apac/aws-credits-request-confirmation/\r\nDisallow: /articles/item/\r\nDisallow: /awis/*\r\nDisallow: /aws-summit-2014/enterprise-london-db/\r\nDisallow: /aws-summit-2014/london-db/\r\nDisallow: /blogs/7faebe5debfdd358a46f2452a54eec94fce49269/\r\nDisallow: /campaigns/infor-hub-xt5r5iwdrgxbnuz/\r\nDisallow: /careers/aus-nz-event/\r\nDisallow: /careers/aus-nz-event/\r\nDisallow: /careers/aws-ar-event/\r\nDisallow: /careers/aws-aus-event/\r\nDisallow: /careers/aws-australia-event/\r\nDisallow: /careers/aws-bigdata-hiring/\r\nDisallow: /careers/aws-brisbane-event/\r\nDisallow: /careers/aws-db-event/\r\nDisallow: /careers/aws-ks-event/\r\nDisallow: /careers/aws-london-event/\r\nDisallow: /careers/aws-mexicocity-event/\r\nDisallow: /careers/aws-miami-event/\r\nDisallow: /careers/aws-ny-event/\r\nDisallow: /careers/aws-pittsburgh-event/\r\nDisallow: /careers/aws-portland-event/\r\nDisallow: /careers/aws-sea-event/\r\nDisallow: /careers/aws-singapore-event/\r\nDisallow: /careers/aws-sydney-event/\r\nDisallow: /careers/chattanooga-mobile-event/\r\nDisallow: /careers/database-hiring/\r\nDisallow: /careers/devtools-australia-event/\r\nDisallow: /careers/devtools-australia-event/\r\nDisallow: /careers/ebs-cincinnati-event/\r\nDisallow: /careers/ec2-australia-event/\r\nDisallow: /careers/ec2-chicago-event/\r\nDisallow: /careers/ec2-hiring-events/\r\nDisallow: /careers/ec2-ottawa-event/\r\nDisallow: /careers/ec2-vancouver-event/\r\nDisallow: /careers/indianapolis-event/\r\nDisallow: /careers/mexico-edge-hiring-2014/\r\nDisallow: /careers/milwaukee-edge-hiring-2014/\r\nDisallow: /careers/montreal-edge-hiring/\r\nDisallow: /careers/Phoenix-Hiring-Event/\r\nDisallow: /careers/pittsburgh-platform-hiring-2014/\r\nDisallow: /careers/raleigh-mobile-event/\r\nDisallow: /careers/raleigh-storage-hiring-2014/\r\nDisallow: /careers/santiago-event/\r\nDisallow: /careers/Sydney-Hiring-Event/\r\nDisallow: /careers/texas-event/\r\nDisallow: /careers/toronto-event/\r\nDisallow: /careers/toronto-storage-hiring-2014/\r\nDisallow: /Cloud_Start_01.LandingPage.html\r\nDisallow: /Cloud_Start_01-Landing-Page-EDO.html\r\nDisallow: /Cloud_Start_02.ConfirmationPage.html\r\nDisallow: /code/*\r\nDisallow: /compliance/contact-us-confirmation/\r\nDisallow: /compliance/hipaa-eligible-services-reference/\r\nDisallow: /compliance/how-to-open-aws-compliance-reports/\r\nDisallow: /compliance/public-sector-contact-confirmation/\r\nDisallow: /compliance/security-assurance-faq\r\nDisallow: /compliance/security-assurance-faq-1\r\nDisallow: /compliance/security-assurance-faq-10\r\nDisallow: /compliance/security-assurance-faq-2\r\nDisallow: /compliance/security-assurance-faq-3\r\nDisallow: /compliance/security-assurance-faq-4\r\nDisallow: /compliance/security-assurance-faq-5\r\nDisallow: /compliance/security-assurance-faq-6\r\nDisallow: /compliance/security-assurance-faq-7\r\nDisallow: /compliance/security-assurance-faq-8\r\nDisallow: /compliance/security-assurance-faq-9\r\nDisallow: /compliance/security-assurance-faq-complete\r\nDisallow: /compliance/security-assurance-US-Public-Sector-checklist\r\nDisallow: /compliance/security-assurance-US-Public-Sector-checklist-1\r\nDisallow: /compliance/security-assurance-US-Public-Sector-checklist-10\r\nDisallow: /compliance/security-assurance-US-Public-Sector-checklist-2\r\nDisallow: /compliance/security-assurance-US-Public-Sector-checklist-3\r\nDisallow: /compliance/security-assurance-US-Public-Sector-checklist-4\r\nDisallow: /compliance/security-assurance-US-Public-Sector-checklist-5\r\nDisallow: /compliance/security-assurance-US-Public-Sector-checklist-6\r\nDisallow: /compliance/security-assurance-US-Public-Sector-checklist-7\r\nDisallow: /compliance/security-assurance-US-Public-Sector-checklist-8\r\nDisallow: /compliance/security-assurance-US-Public-Sector-checklist-9\r\nDisallow: /compliance/security-assurance-US-Public-Sector-checklist-complete\r\nDisallow: /compliance/third-party-access/\r\nDisallow: /contact-us/ami-submission-conf/\r\nDisallow: /contact-us/article-submission-conf/\r\nDisallow: /contact-us/case-contact-confirmation/\r\nDisallow: /contact-us/customer-app-submission-conf/\r\nDisallow: /contact-us/developertool-submission-conf/\r\nDisallow: /contact-us/ec2-email-limit-request-confirmation/\r\nDisallow: /contact-us/elasticache-node-limit-request-confirmation/\r\nDisallow: /contact-us/marketing-unsubscribe-confirmation/\r\nDisallow: /contact-us/publicdataset-conf/\r\nDisallow: /contact-us/sales-contact-confirmation/\r\nDisallow: /contact-us/samplecode-submission-conf/\r\nDisallow: /csds/*\r\nDisallow: /customerapps/*\r\nDisallow: /datasets/\r\nDisallow: /developertools/2555753788650372\r\nDisallow: /documentation/alexa-top-sites/\r\nDisallow: /documentation/awis/\r\nDisallow: /email-preferences\r\nDisallow: /email-preferences/\r\nDisallow: /events/applesummit\r\nDisallow: /events/cio-advisory-council/\r\nDisallow: /events/finserv-summit-dublin\r\nDisallow: /events/finserv-summit-dublin/confirmation\r\nDisallow: /events/finserv-summit-singapore\r\nDisallow: /events/finserv-summit-singapore/confirmation\r\nDisallow: /events/fsi-cloud-symposium-nyc.html\r\nDisallow: /federal/events/earthobservationday2015/\r\nDisallow: /federal/events/earthobservationday2015/ConfirmationPage/\r\nDisallow: /forms/activate-support\r\nDisallow: /free/cloud-computing-free-tier/\r\nDisallow: /free/databases-free-tier/\r\nDisallow: /get-cloud/\r\nDisallow: /getting-started/functional-groups/\r\nDisallow: /getting-started/functional-groups/*\r\nDisallow: /government-education/nonprofits/aws-serve/\r\nDisallow: /government-education/nonprofits/aws-serve-member/\r\nDisallow: /government-education/nonprofits/credit-program/\r\nDisallow: /government-education/nonprofits/credit-program/confirmation\r\nDisallow: /government-education/nonprofits/credit-program/terms\r\nDisallow: /government-education/nonprofits/grant-program/\r\nDisallow: /government-education/nonprofits/grant-program/confirmation/\r\nDisallow: /government-education/nonprofits/grant-program/terms/\r\nDisallow: /government-education/nonprofits/leadership-forum-poc\r\nDisallow: /government-education/nonprofits/leadership-forum-poc/confirmation\r\nDisallow: /government-education/public-sector-cloud-start/public-sector-cloud-start/\r\nDisallow: /government-education/public-sector-cloud-start/public-sector-cloud-start1/\r\nDisallow: /hackathons/reinvent-2013/\r\nDisallow: /legal/3874403/\r\nDisallow: /lms/contact-us/\r\nDisallow: /loft/\r\nDisallow: /marketplace/reviews/reviews-list/*/write-comment/\r\nDisallow: /marketplace/reviews/write-review/\r\nDisallow: /marketplace/search*\r\nDisallow: /microsoft/bizspark/\r\nDisallow: /microsoft/bizspark/signon/thankyou/\r\nDisallow: /nav/\r\nDisallow: /nav/side-nav/\r\nDisallow: /nav/side-nav/products-services/\r\nDisallow: /nav/tabs/\r\nDisallow: /nav/tabs/account/\r\nDisallow: /nav/tabs/aws/\r\nDisallow: /nav/tabs/community/\r\nDisallow: /nav/tabs/developers/\r\nDisallow: /nav/tabs/products/\r\nDisallow: /nav/tabs/support/\r\nDisallow: /optin/\r\nDisallow: /optout/\r\nDisallow: /partners/contact-us/partners-confirmation/\r\nDisallow: /partners/overview/partner-newsletter/confirmation/\r\nDisallow: /partners/overview/partner-terms-and-conditions/\r\nDisallow: /partners/webinars/request/\r\nDisallow: /ping\r\nDisallow: /pm/\r\nDisallow: /ar/pm/\r\nDisallow: /cn/pm/\r\nDisallow: /de/pm/\r\nDisallow: /es/pm/\r\nDisallow: /fr/pm/\r\nDisallow: /id/pm/\r\nDisallow: /it/pm/\r\nDisallow: /jp/pm/\r\nDisallow: /ko/pm/\r\nDisallow: /pt/pm/\r\nDisallow: /ru/pm/\r\nDisallow: /th/pm/\r\nDisallow: /tr/pm/\r\nDisallow: /tw/pm/\r\nDisallow: /vi/pm/\r\nAllow: /pm/amplify/\r\nAllow: /pm/lambda/\r\nAllow: /pm/sagemaker/\r\nAllow: /pm/textract/\r\nAllow: /pm/transcribe/\r\nDisallow: /premiumsupport/iam/\r\nDisallow: /premiumsupport/knowledge-center/feedback-no/\r\nDisallow: /premiumsupport/knowledge-center/feedback-yes/\r\nDisallow: /pricing/awis/\r\nDisallow: /public-data-sets/applicants/\r\nDisallow: /public-data-sets/terms/\r\nDisallow: /publicdatasets-confirmation/\r\nDisallow: /quickstart/promo\r\nDisallow: /quickstart/promo/confirmation\r\nDisallow: /reinvent/video-case-studies\r\nDisallow: /releasenotes/item/\r\nDisallow: /research-credits/data-science-bowl\r\nDisallow: /ru/blogs/497c2c2d305b7dfa351d17bd3b4398dc37b0e0a4/\r\nDisallow: /s/*\r\nDisallow: /s/dm/house/editorial/startups/ha_ed_activate_loft_launch_sf_1/\r\nDisallow: /sap/saphana/pilot/\r\nDisallow: /sap/saphana/pilot/offer/\r\nDisallow: /sap/saphana/pilot/offer/confirmation\r\nDisallow: /search\r\nDisallow: /search/\r\nAllow: /api/dirs/items/search\r\nDisallow: /security/aws-pgp-public-key\r\nDisallow: /security/ec2-pkgs-public-key\r\nDisallow: /security/introducing-s2n-new-open-source-TLS-implementation\r\nDisallow: /s-moved/dm/house/editorial/startups/ha_ed_activate_loft_launch_sf_1/\r\nDisallow: /solutions/case-studies/imdb/\r\nDisallow: /solutions/featured-partners/\r\nDisallow: /solutions/global-solution-providers/microsoft/bizspark/\r\nDisallow: /solutions/global-solution-providers/microsoft/bizspark/faq/\r\nDisallow: /solutions/global-solution-providers/microsoft/bizspark/signon/\r\nDisallow: /solutions/global-solution-providers/microsoft/bizspark/signon/\r\nDisallow: /solutions/global-solution-providers/microsoft/bizspark/signon/\r\nDisallow: /solutions/global-solution-providers/microsoft/bizspark/signon/thankyou/\r\nDisallow: /solutions/global-solution-providers/microsoft/bizspark/signon/thankyou/\r\nDisallow: /solutions/solution-providers/program/reseller/adguidelines/\r\nDisallow: /solutions/solution-providers/program/reseller/adguidelines/\r\nDisallow: /solutions/solution-providers/program/reseller/authorized/\r\nDisallow: /solutions/solution-providers/program/reseller/authorized/\r\nDisallow: /solutions/solution-providers/program/reseller/customerterms/\r\nDisallow: /solutions/solution-providers/program/reseller/customerterms/\r\nDisallow: /specialhandling-payments/\r\nDisallow: /start-ups/challenge/\r\nDisallow: /start-ups/loft/hack/\r\nDisallow: /start-ups/loft/hacknight/\r\nDisallow: /start-ups/loft/sf2014/\r\nDisallow: /start-ups/loft/sfff100l3ft\r\nDisallow: /start-ups/loft/survey/\r\nDisallow: /start-ups/loft/welcome\r\nDisallow: /unable-to-process-request/\r\nDisallow: /use-cases/global-solution-providers/microsoft/bizspark/\r\nDisallow: /use-cases/global-solution-providers/microsoft/bizspark/getting-started/\r\nDisallow: /use-cases/global-solution-providers/microsoft/bizspark/signon/\r\nDisallow: /use-cases/global-solution-providers/microsoft/bizspark/signon/thankyou/\r\nDisallow: /use-cases/global-solution-providers/microsoft/bizspark/thankyou/\r\nDisallow: /vmware/about\r\nDisallow: /windows/resources/whitepapers/\r\nDisallow: /windows/server2003/access/\r\nDisallow: /z_dx-marketing-team-prototypes/*\r\nDisallow: /community/*\r\nDisallow: /community/*\r\nDisallow: /community/heroes/\r\nDisallow: /*/community/heroes/\r\nDisallow: /*/search\r\nDisallow: /*/search/\r\nSitemap: https://aws.amazon.com/sitemaps/index/\r\nSitemap: https://d36a099zv34xie.cloudfront.net/psf-sitemap.xml\r\nSitemap: https://aws.amazon.com/marketplace/sitemap","agents":{"GPTBot":true,"OAI-SearchBot":true,"ChatGPT-User":true,"ClaudeBot":true,"Claude-SearchBot":true,"PerplexityBot":true,"Google-Extended":true,"Applebot-Extended":true,"CCBot":true,"Bytespider":true,"Meta-ExternalAgent":true,"Amazonbot":true}},"sitemap":{"status":200,"content_type":"application/xml","bytes":2669,"sha256":"0a390975ecbe729624500fd248cee68c04ace1ba68a7f23a6a0429a6c6544ab7","final_host":"aws.amazon.com","final_path":"/sitemaps/index","ms":42,"url":"https://aws.amazon.com/sitemaps/index/","declared":true,"xml":true},"files":{"llms_txt":{"status":404,"content_type":"text/html","bytes":299216,"sha256":"54deb70ea4bae323c8b3ea2e47200164c2a2dcd685b28a7007ed9f67fe6962e5","final_host":"aws.amazon.com","final_path":"/llms.txt","ms":123,"path":"/llms.txt","html":true,"json":null,"present":false},"agents_md":{"status":404,"content_type":"text/html","bytes":299216,"sha256":"8492c7c36f23540ee0e9774917aac6658a997a2c2c150f5bbb1093337873c788","final_host":"aws.amazon.com","final_path":"/agents.md","ms":179,"path":"/agents.md","html":true,"json":null,"present":false},"agent_skills":{"status":404,"content_type":"text/html","bytes":299216,"sha256":"0c9d6f8314251fe5aab9ca830ab6844f51ce042a91c042aebc0a879cfb7b0c2a","final_host":"aws.amazon.com","final_path":"/.well-known/agent-skills/index.json","ms":155,"path":"/.well-known/agent-skills/index.json","html":true,"json":null,"present":false},"mcp_card":{"status":404,"content_type":"text/html","bytes":299216,"sha256":"778fdf6000cdc695794a6a705b904f3d11e678f2d6ea2f6833a3e53f8edb4b0f","final_host":"aws.amazon.com","final_path":"/.well-known/mcp/server-card.json","ms":159,"path":"/.well-known/mcp/server-card.json","html":true,"json":null,"present":false},"api_catalog":{"status":404,"content_type":"text/html","bytes":299216,"sha256":"03c0e1ea61d5304d87da33f102df74d225f5a891ae2c4fbe37df549437b936b2","final_host":"aws.amazon.com","final_path":"/.well-known/api-catalog","ms":158,"path":"/.well-known/api-catalog","html":true,"json":null,"present":false},"a2a_card":{"status":404,"content_type":"text/html","bytes":299216,"sha256":"52a76a98cb4a7d6d39e5b44aa4ab02bac6d09320055a32c1373fee3b7d285777","final_host":"aws.amazon.com","final_path":"/.well-known/agent-card.json","ms":184,"path":"/.well-known/agent-card.json","html":true,"json":null,"present":false}},"rings":{"v":1,"candidates":[],"mcp":[],"a2a":null,"webmcp":{"model_context":false,"bridge":false,"mcp_json":false,"link_rel":false,"any":false},"handshakes":0,"get_path":{"status":404,"content_type":"text/html","ms":194,"error":null},"get_host":{"status":530,"content_type":"text/plain","ms":16,"error":null}},"sf":[-1,0,0,0,0,0,5,0,0],"derived":{"reachable":true,"home_class":"answers","robots_served":true,"robots_sitemap":true,"content_signal":false,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":[],"blocks_any_ai":false,"files":{"llms_txt":false,"agents_md":false,"agent_skills":false,"mcp_card":false,"api_catalog":false,"a2a_card":false},"any_capability":false,"ai_refused":[],"ai_thin":[],"aipref":0,"aipref_measured":true,"sf":[-1,0,0,0,0,0,5,0,0],"rings":{"mcp_declared":false,"mcp_live":false,"mcp_auth":false,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":3,"fh":"aws.amazon.com","fp_other_host":[],"fp":"138fbbc185e891da","fp_bits":{"gptbot":0,"claudebot":0,"oaisearch":0,"perplexity":0}},"derive_v":5,"frame":"top1k","shard":4},{"v":1,"kind":"crawlcheck-census-record","run":43,"domain":"googleapis.com","rank":6,"measured_at":"2026-11-13T08:07:00.657Z","home":{"browser":{"status":404,"content_type":"text/html","bytes":1561,"sha256":"23926e9185d8d43c02807a838ffb373cc1977726094a4e46807c66ada9dd7660","final_host":"googleapis.com","final_path":"/","ms":148,"identity":"Mobile browser","html":true},"gptbot":{"status":404,"content_type":"text/html","bytes":1561,"sha256":"23926e9185d8d43c02807a838ffb373cc1977726094a4e46807c66ada9dd7660","final_host":"googleapis.com","final_path":"/","ms":147,"identity":"GPTBot","html":true},"claudebot":{"status":404,"content_type":"text/html","bytes":1561,"sha256":"23926e9185d8d43c02807a838ffb373cc1977726094a4e46807c66ada9dd7660","final_host":"googleapis.com","final_path":"/","ms":146,"identity":"ClaudeBot","html":true},"oaisearch":{"status":404,"content_type":"text/html","bytes":1561,"sha256":"23926e9185d8d43c02807a838ffb373cc1977726094a4e46807c66ada9dd7660","final_host":"googleapis.com","final_path":"/","ms":147,"identity":"OAI-SearchBot","html":true},"perplexity":{"status":404,"content_type":"text/html","bytes":1561,"sha256":"23926e9185d8d43c02807a838ffb373cc1977726094a4e46807c66ada9dd7660","final_host":"googleapis.com","final_path":"/","ms":145,"identity":"PerplexityBot","html":true}},"robots":{"status":404,"content_type":"text/html","bytes":1571,"sha256":"6adee83a3eba87d30fdca3997d27097697f2767805f222a0f78c3f04392004d2","final_host":"googleapis.com","final_path":"/robots.txt","ms":141,"html":true,"text":null,"agents":null},"sitemap":null,"files":null,"derived":{"reachable":false,"home_class":"other","robots_served":false,"robots_sitemap":false,"content_signal":false,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":[],"blocks_any_ai":false,"files":{"llms_txt":false,"agents_md":false,"agent_skills":false,"mcp_card":false,"api_catalog":false,"a2a_card":false},"any_capability":false,"ai_refused":[],"ai_thin":[],"rings":{"mcp_declared":false,"mcp_live":false,"mcp_auth":false,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":3,"fh":"googleapis.com","fp_other_host":[]},"derive_v":5,"frame":"top1k","shard":5},{"v":1,"kind":"crawlcheck-census-record","run":43,"domain":"akamai.net","rank":7,"measured_at":"2026-11-13T08:07:15.177Z","home":{"browser":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"akamai.net","final_path":"/","ms":10,"identity":"Mobile browser","html":false},"gptbot":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"akamai.net","final_path":"/","ms":10,"identity":"GPTBot","html":false},"claudebot":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"akamai.net","final_path":"/","ms":10,"identity":"ClaudeBot","html":false},"oaisearch":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"akamai.net","final_path":"/","ms":10,"identity":"OAI-SearchBot","html":false},"perplexity":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"akamai.net","final_path":"/","ms":11,"identity":"PerplexityBot","html":false}},"robots":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"akamai.net","final_path":"/robots.txt","ms":10,"html":false,"text":null,"agents":null},"sitemap":null,"files":null,"derived":{"reachable":false,"home_class":"no_site","robots_served":false,"robots_sitemap":false,"content_signal":false,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":[],"blocks_any_ai":false,"files":{"llms_txt":false,"agents_md":false,"agent_skills":false,"mcp_card":false,"api_catalog":false,"a2a_card":false},"any_capability":false,"ai_refused":[],"ai_thin":[],"rings":{"mcp_declared":false,"mcp_live":false,"mcp_auth":false,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":3,"fh":"akamai.net","fp_other_host":[]},"derive_v":5,"frame":"top1k","shard":6},{"v":1,"kind":"crawlcheck-census-record","run":43,"domain":"microsoft.com","rank":8,"measured_at":"2026-11-13T08:08:20.375Z","home":{"browser":{"status":403,"content_type":"text/html","bytes":4409,"sha256":"249acb37167de3ed7574b26d546934dd4b6954aed8ecc222b64a78a92efc9c39","final_host":"www.microsoft.com","final_path":"/en-gb","ms":144,"identity":"Mobile browser","html":true},"gptbot":{"status":403,"content_type":"text/html","bytes":4409,"sha256":"11b377c1f15a3da566a6a14e3f0a8aae3b0f0dc828fd1ec1207c509422b7d58a","final_host":"www.microsoft.com","final_path":"/en-gb","ms":144,"identity":"GPTBot","html":true},"claudebot":{"status":403,"content_type":"text/html","bytes":4409,"sha256":"71b615eeeb3cecc0254bf2251373851a67c3f98f9b26b024062adc6f06d8ca36","final_host":"www.microsoft.com","final_path":"/","ms":120,"identity":"ClaudeBot","html":true},"oaisearch":{"status":403,"content_type":"text/html","bytes":4409,"sha256":"15678dcf56d0dd0d6cc5a4cbc0b9099f481d4c2f924cef25194840c19a8a722d","final_host":"www.microsoft.com","final_path":"/","ms":131,"identity":"OAI-SearchBot","html":true},"perplexity":{"status":200,"content_type":"text/html","bytes":212668,"sha256":"19fe55fb837097b3b43366192aa7b9e57cd15284263d49ee84600c74876cf4f3","final_host":"www.microsoft.com","final_path":"/en-gb","ms":233,"identity":"PerplexityBot","html":true,"fp":"4d77861d8ed8fc76"}},"robots":{"status":200,"content_type":"text/plain","bytes":9126,"sha256":"79a2a24e3e2c00deaba2fc1922b9113f5508e74133911eadea4ebb00b5de9768","final_host":"www.microsoft.com","final_path":"/robots.txt","ms":161,"html":false,"text":"# Robots.txt file for www.microsoft.com v2.11\r\n\r\nUser-agent: *\r\nDisallow: /*?*ep_*\r\nDisallow: /*&ep_*\r\nDisallow: /store/XboxComMsCom3PAdsOptOutCookieSync.html\r\nDisallow: /store/XboxComMsComCartMuidSync.html\r\nDisallow: /*/store/locations/store-locations-sitemap.xml\r\nDisallow: /en-us/windows/si/matrix.html\r\nDisallow: /en-us/windows/si/matrix.html\r\nDisallow: /*/security/search-results.aspx?\r\nDisallow: /*/music/*/search/\r\nDisallow: /*/store/buynow?\r\nDisallow: /*/store/cart\r\nDisallow: /store/buy/cartcount\r\nDisallow: /*/search/\r\nDisallow: /*/music/*/Search/\r\nDisallow: /*/Search/\r\nDisallow: /*/newsearch/\r\nDisallow: *action=catalogsearch&\r\nDisallow: /*/store/d/groove-music-pass/cfq7ttc0k5dq/0001\r\nDisallow: /*/store/best-rated/*?*&*&*\r\nDisallow: /*/store/best-selling/*?*&*&*\r\nDisallow: /*/store/coming-soon/*?*&*&*\r\nDisallow: /*/store/deals/*?*&*&*\r\nDisallow: /*/store/game-demos/*?*&*&*\r\nDisallow: /*/store/game-previews/*?*&*&*\r\nDisallow: /*/store/most-played/*?*&*&*\r\nDisallow: /*/store/most-popular/*?*&*&*\r\nDisallow: /*/store/new/*?*&*&*\r\nDisallow: /*/store/new-and-rising/*?*&*&*\r\nDisallow: /*/store/top-free/*?*&*&*\r\nDisallow: /*/store/top-paid/*?*&*&*\r\nDisallow: /*/store/collections/helpmechoosepc?*&*&*\r\nDisallow: /*/store/collections/helpmechoosepc/?*&*&*\r\nDisallow: /*/store/movies-and-tv/*?*&*&*\r\nDisallow: /cascadeauth/microsoft-365/account/silentauth\r\nDisallow: /*/microsoft-365/site-search\r\nDisallow: /news/search\r\nDisallow: /video/search\r\nDisallow: */qa-folder\r\nAllow: /*/store/*/search/\r\nAllow: /*/store/*/layout/\r\nAllow: /*/store/music/groove-music-pass/*\r\nAllow: *action=catalogsearch&catalog_mode=grid&page=2$\r\nAllow: *action=catalogsearch&catalog_mode=grid&page=3$\r\nAllow: *action=catalogsearch&catalog_mode=grid&page=4$\r\nAllow: *action=catalogsearch&catalog_mode=grid&page=5$\r\nAllow: *action=catalogsearch&catalog_mode=grid&page=6$\r\nAllow: *action=catalogsearch&catalog_mode=grid&page=7$\r\nAllow: *action=catalogsearch&catalog_mode=grid&page=8$\r\nAllow: *action=catalogsearch&catalog_mode=list&page=2$\r\nAllow: *action=catalogsearch&catalog_mode=list&page=3$\r\nAllow: *action=catalogsearch&catalog_mode=list&page=4$\r\nAllow: *action=catalogsearch&catalog_mode=list&page=5$\r\nAllow: *action=catalogsearch&catalog_mode=list&page=6$\r\nAllow: *action=catalogsearch&catalog_mode=list&page=7$\r\nAllow: *action=catalogsearch&catalog_mode=list&page=8$\r\nDisallow: *action=accessorysearch&product=*&*\r\nAllow: *action=accessorysearch&product=*$\r\nDisallow: *action=accessorysearch&\r\nAllow: *action=accessorysearch&page=2$\r\nAllow: *action=accessorysearch&page=3$\r\nAllow: *action=accessorysearch&page=4$\r\nAllow: *action=accessorysearch&page=5$\r\nAllow: *action=accessorysearch&page=6$\r\nAllow: *action=accessorysearch&page=7$\r\nAllow: *action=accessorysearch&page=8$\r\nDisallow: *action=productcompareaction&\r\nDisallow: *action=productLinkAction&\r\nDisallow: *action=overlay&\r\nDisallow: *action=quickSearch&\r\nDisallow: *action=writeReview\r\nDisallow: *rep=hc\r\nDisallow: *fe=true\r\nDisallow: *?intc=\r\nDisallow: *&solved=\r\nDisallow: /music/*/wal/\r\nDisallow: */wal/\r\nDisallow: */layout/\r\nDisallow: */base-en/\r\nDisallow: *action=siteSearch\r\nDisallow: *action=productSupportSearch\r\nDisallow: *=imgmanager\r\nDisallow: */unsubscribe/\r\nDisallow: *ActivityUID=\r\nDisallow: /feeds/TechNet/fr-fr/screenshot/screenshot%20surface.jpg\r\nDisallow: /imaginecup/*\r\nDisallow: /*/download/confirmation.aspx?\r\nDisallow: /*HpOptOut=true$\r\nDisallow: /*TOCLinksForCrawlers*\r\nDisallow: /*mac/help.mspx?\r\nDisallow: /*mactopia/help.mspx?\r\nDisallow: */mediaapi/\r\nDisallow: /blacklisted*\r\nDisallow: /canada/Library/mnp/2/aspx/\r\nDisallow: /communities/bin.aspx?\r\nDisallow: /communities/blogs/PortalResults.mspx?\r\nDisallow: /communities/eventdetails.mspx?\r\nDisallow: /communities/rss.aspx*\r\nDisallow: /*/download/confirmation.aspx?\r\nDisallow: /*/download/registration-suggested.aspx?\r\nDisallow: /*/download/results.aspx?\r\nDisallow: /*/download/Browse.aspx?\r\nDisallow: /*/download/browse.aspx?\r\nDisallow: /*/download/info.aspx?\r\nDisallow: /*/download/thankyou.aspx\r\nDisallow: /*/download/thankyou.aspx?\r\nDisallow: /france/formation/centres/planning.asp?\r\nDisallow: /france/ie/default.asp?\r\nDisallow: /france/mnp_utility.mspx?\r\nDisallow: /genuine/\r\nDisallow: /Germany/kleinunternehmen/euga/detail.mspx?\r\nDisallow: /Germany/kleinunternehmen/euga/results.mspx?\r\nDisallow: /germany/library/images/mnp/\r\nDisallow: /germany/video/de/de/related*\r\nDisallow: /hpc/*/supported-applications.aspx?\r\nDisallow: /ie/ie40/\r\nDisallow: /info/customerror.htm*\r\nDisallow: /info/smart404.asp*\r\nDisallow: /intlkb/\r\nDisallow: /isapi/\r\nDisallow: /Japan/DirectX/default.asp?\r\nDisallow: /japan/directx/default.asp?\r\nDisallow: /japan/enable/textview.asp?\r\nDisallow: /japan/products/library/search.asp?\r\nDisallow: /japan/showcase/print/default.aspx?\r\nDisallow: /japan/terminology/query.asp?\r\nDisallow: /*mnp_utility.mspx?\r\nDisallow: /rus/licensing/Unilateral.aspx/*\r\nDisallow: /spain/empresas/\r\nDisallow: /spain/medianaempresa/\r\nDisallow: /windows/compatibility/windows-vista/\r\nDisallow: /windows/compatibility/windows-7/\r\nDisallow: /windows/compatibility/windows-7/*\r\nDisallow: /windows/404.aspx?*\r\nDisallow: /windows/campaign/meet-start.aspx\r\nDisallow: /windows/campaign/meet-apps.aspx\r\nDisallow: /windows/campaign/features-built-in-apps.aspx\r\nDisallow: /ru-ru/events/platforma/materials/default.aspx?speaker*\r\nDisallow: /de-de/corporate/rechtliche-hinweise/impressum_de.aspx\r\nDisallow: /en/microsoft-365\r\nDisallow: /en/microsoft-365/\r\nAllow: /en/microsoft-365/word\r\nAllow: /en/microsoft-365/outlook/web-email-login-for-outlook\r\nAllow: /en/microsoft-365/free-office-online-for-the-web\r\nAllow: /en/microsoft-365/planner/microsoft-planner\r\nAllow: /en/microsoft-365/yammer/yammer-overview\r\nAllow: /en/microsoft-365/publisher\r\nAllow: /en/microsoft-365/products-apps-services\r\nAllow: /en/microsoft-365/onenote/digital-note-taking-app\r\nAllow: /en/microsoft-365/powerpoint\r\nAllow: /en/microsoft-365/excel\r\nAllow: /en/microsoft-365/microsoft-office\r\nAllow: /en/microsoft-365/business/microsoft-365-administration\r\nAllow: /en/microsoft-365-copilot/microsoft-copilot-studio\r\nAllow: /en/microsoft-365/project/project-management-software\r\nAllow: /en/microsoft-365/project/microsoft-project-enterprise-plans-and-pricing\r\nAllow: /en/microsoft-365/onedrive/online-cloud-storage\r\nAllow: /en/microsoft-365/business\r\nAllow: /en/microsoft-365/clipchamp\r\nAllow: /en/microsoft-viva/engage\r\nDisallow: /en/microsoft-365/\r\nDisallow: /es/microsoft-365\r\nDisallow: /es/microsoft-365/\r\nDisallow: /fr/microsoft-365\r\nDisallow: /fr/microsoft-365/\r\nDisallow: /en/security\r\nDisallow: /en/security/\r\nDisallow: /es/security\r\nDisallow: /es/security/\r\nDisallow: /fr/security\r\nDisallow: /fr/security/\r\nDisallow: /en/microsoft-teams\r\nDisallow: /en/microsoft-teams/\r\nDisallow: /es/microsoft-teams\r\nDisallow: /es/microsoft-teams/\r\nDisallow: /fr/microsoft-teams\r\nDisallow: /fr/microsoft-teams/\r\nDisallow: /en/microsoft-viva\r\nDisallow: /en/microsoft-viva/\r\nDisallow: /es/microsoft-viva\r\nDisallow: /es/microsoft-viva/\r\nDisallow: /fr/microsoft-viva\r\nDisallow: /fr/microsoft-viva/\r\nDisallow: */isv/*?\r\nDisallow: /wp-admin/post.php \r\nAllow: /wp-admin/admin-ajax.php \r\nAllow: /wp-content/uploads/ \r\nDisallow: /blog/feed/ \r\nDisallow: /wp-login.php \r\nDisallow: /xmlrpc.php \r\nDisallow: /*?p= \r\nDisallow: /*?*p=* \r\nDisallow: /cdn-cgi/* \r\nDisallow: /page/*/?s= \r\nDisallow: /login/ \r\nDisallow: /*/embed/ \r\nDisallow: /cgi-bin/ \r\nAllow: /?display=wide \r\nDisallow: /readme.html \r\nDisallow: /*noItems= \r\nDisallow: /?s=* \r\nDisallow: /?sort-by=*\r\n\r\nSitemap: https://www.microsoft.com/store/sitemap-index.xml\r\nSitemap: https://www.microsoft.com/en-us/surface/surface.index.xml\r\nSitemap: https://www.microsoft.com/en-us/windows/windows-index.xml\r\nSitemap: https://www.microsoft.com/sitemaps/ai/sitemapindex.xml\r\nSitemap: https://www.microsoft.com/sitemaps/dynamics-365/sitemapindex.xml\r\nSitemap: https://www.microsoft.com/sitemaps/isv/sitemapindex.xml\r\nSitemap: https://www.microsoft.com/sitemaps/microsoft-365/sitemapindex.xml\r\nSitemap: https://www.microsoft.com/sitemaps/microsoft-copilot/sitemapindex.xml\r\nSitemap: https://www.microsoft.com/sitemaps/microsoft-fabric/sitemapindex.xml\r\nSitemap: https://www.microsoft.com/sitemaps/microsoft-teams/sitemapindex.xml\r\nSitemap: https://www.microsoft.com/sitemaps/power-platform/sitemapindex.xml\r\nSitemap: https://www.microsoft.com/sitemaps/microsoft-viva/sitemapindex.xml\r\nSitemap: https://www.microsoft.com/sitemaps/security/sitemapindex.xml\r\nSitemap: https://www.microsoft.com/sitemaps/trust-center/sitemapindex.xml\r\nSitemap: https://www.microsoft.com/sitemaps/windows-365/sitemapindex.xml\r\nSitemap: https://www.microsoft.com/sitemaps/customers/sitemapindex.xml\r\nSitemap: https://www.microsoft.com/sitemaps/education/sitemapindex.xml\r\nSitemap: https://www.microsoft.com/sitemaps/industry/sitemapindex.xml\r\nSitemap: https://www.microsoft.com/sitemaps/microsoft-cloud/sitemapindex.xml\r\nSitemap: https://www.microsoft.com/sitemaps/sustainability/sitemapindex.xml\r\nSitemap: https://www.microsoft.com/en-us/microsoft-365-life-hacks/sitemap.xml\r\nSitemap: https://www.microsoft.com/windows/business/sitemap.xml","agents":{"GPTBot":true,"OAI-SearchBot":true,"ChatGPT-User":true,"ClaudeBot":true,"Claude-SearchBot":true,"PerplexityBot":true,"Google-Extended":true,"Applebot-Extended":true,"CCBot":true,"Bytespider":true,"Meta-ExternalAgent":true,"Amazonbot":true}},"sitemap":null,"files":null,"derived":{"reachable":false,"home_class":"refused_us","robots_served":false,"robots_sitemap":false,"content_signal":false,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":[],"blocks_any_ai":false,"files":{"llms_txt":false,"agents_md":false,"agent_skills":false,"mcp_card":false,"api_catalog":false,"a2a_card":false},"any_capability":false,"ai_refused":[],"ai_thin":[],"rings":{"mcp_declared":false,"mcp_live":false,"mcp_auth":false,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":3,"fh":"www.microsoft.com","fp_other_host":[]},"derive_v":5,"frame":"top1k","shard":7},{"v":1,"kind":"crawlcheck-census-record","run":43,"domain":"youtube.com","rank":9,"measured_at":"2026-11-13T07:56:55.238Z","home":{"browser":{"status":200,"content_type":"text/html","bytes":380660,"sha256":"0d4b9f95c07b2d801b9153a21333886a34587a1a42973b8f80986f3415238884","final_host":"m.youtube.com","final_path":"/","ms":491,"identity":"Mobile browser","html":true},"gptbot":{"status":200,"content_type":"text/html","bytes":1015689,"sha256":"5c616fe8019c3aa7b51f3384a522b91c0ae8f6974611f9d57b4011fb9e46cd03","final_host":"www.youtube.com","final_path":"/","ms":305,"identity":"GPTBot","html":true},"claudebot":{"status":200,"content_type":"text/html","bytes":1010402,"sha256":"d0f91d78deae9fd8a7e4ce47fa4dce0510b095cc2edc296a1de882ea7959d501","final_host":"www.youtube.com","final_path":"/","ms":286,"identity":"ClaudeBot","html":true},"oaisearch":{"status":200,"content_type":"text/html","bytes":922917,"sha256":"11bca3fe1519737cb6c9c60ab5ebf66cec88047e4789b920d3c712778cd44dfc","final_host":"www.youtube.com","final_path":"/","ms":298,"identity":"OAI-SearchBot","html":true},"perplexity":{"status":200,"content_type":"text/html","bytes":974077,"sha256":"581050e57b463e651b47b5e1886049671f88508c55be7625233040a86213776d","final_host":"www.youtube.com","final_path":"/","ms":301,"identity":"PerplexityBot","html":true}},"robots":{"status":200,"content_type":"text/plain","bytes":792,"sha256":"96f8640ca357626e7d9a894057fef8b5746eb838c29ed68fc176af58c9348c6f","final_host":"www.youtube.com","final_path":"/robots.txt","ms":106,"html":false,"text":"# robots.txt file for YouTube\n# Created in the distant future (the year 2000) after\n# the robotic uprising of the mid 90's which wiped out all humans.\n\nUser-agent: Mediapartners-Google*\nDisallow:\n\nUser-agent: *\nDisallow: /api/\nDisallow: /channel_picker\nDisallow: /comment\nDisallow: /feeds/videos.xml\nDisallow: /file_download\nDisallow: /get_video\nDisallow: /get_video_info\nDisallow: /get_midroll_info\nDisallow: /live_chat\nDisallow: /login\nDisallow: /qr\nDisallow: /results\nDisallow: /signup\nDisallow: /t/terms\nDisallow: /timedtext_video\nDisallow: /verify_age\nDisallow: /watch_ajax\nDisallow: /watch_fragments_ajax\nDisallow: /watch_popup\nDisallow: /watch_queue_ajax\nDisallow: /youtubei/\n\nSitemap: https://www.youtube.com/sitemaps/sitemap.xml\nSitemap: https://www.youtube.com/product/sitemap.xml\n\n","agents":{"GPTBot":true,"OAI-SearchBot":true,"ChatGPT-User":true,"ClaudeBot":true,"Claude-SearchBot":true,"PerplexityBot":true,"Google-Extended":true,"Applebot-Extended":true,"CCBot":true,"Bytespider":true,"Meta-ExternalAgent":true,"Amazonbot":true}},"sitemap":{"status":200,"content_type":"text/xml","bytes":2045,"sha256":"89bea4222871c7b633fc750e5f1b2fd503ab80ecb9c6cf7aae7de3cf3bbddfbb","final_host":"www.youtube.com","final_path":"/sitemaps/sitemap.xml","ms":54,"url":"https://www.youtube.com/sitemaps/sitemap.xml","declared":true,"xml":true},"files":{"llms_txt":{"status":404,"content_type":"text/html","bytes":758,"sha256":"4efebf71d118dfbccb3b5d0087cec213f39174920671bbed10ddda0ce97b03d7","final_host":"www.youtube.com","final_path":"/llms.txt","ms":315,"path":"/llms.txt","html":true,"json":null,"present":false},"agents_md":{"status":404,"content_type":"text/html","bytes":758,"sha256":"a655442feed812fdc6c6bf91904187a34ffd0f69cfb2667dc8af4902130b44b2","final_host":"www.youtube.com","final_path":"/agents.md","ms":333,"path":"/agents.md","html":true,"json":null,"present":false},"agent_skills":{"status":404,"content_type":"text/html","bytes":758,"sha256":"4d42e376898408e1b351096c769b765e3ab6dbf6589f4567f35eb452f36a1ea0","final_host":"www.youtube.com","final_path":"/.well-known/agent-skills/index.json","ms":278,"path":"/.well-known/agent-skills/index.json","html":true,"json":null,"present":false},"mcp_card":{"status":404,"content_type":"text/html","bytes":758,"sha256":"4d2e760c1851df2be860054440d9bea68faf64e7f60ac3032968d0e3542bdd51","final_host":"www.youtube.com","final_path":"/.well-known/mcp/server-card.json","ms":344,"path":"/.well-known/mcp/server-card.json","html":true,"json":null,"present":false},"api_catalog":{"status":404,"content_type":"text/html","bytes":758,"sha256":"cc28bc7cfba6ac07f44a2b1b124925c08f1b9ed5cd640ffdbc76a5a427c3d7b6","final_host":"www.youtube.com","final_path":"/.well-known/api-catalog","ms":334,"path":"/.well-known/api-catalog","html":true,"json":null,"present":false},"a2a_card":{"status":404,"content_type":"text/html","bytes":758,"sha256":"274f9d67e4ab418a9f7e8effd32878d2ef949810ee840c5b4596f8a482a7acf7","final_host":"www.youtube.com","final_path":"/.well-known/agent-card.json","ms":325,"path":"/.well-known/agent-card.json","html":true,"json":null,"present":false}},"rings":{"v":1,"candidates":[],"mcp":[],"a2a":null,"webmcp":{"model_context":false,"bridge":false,"mcp_json":false,"link_rel":false,"any":false},"handshakes":0,"get_path":{"status":404,"content_type":"text/html","ms":242,"error":null},"get_host":{"status":530,"content_type":"text/plain","ms":81,"error":null}},"sf":[4,0,0,0,0,0,1,0,1],"derived":{"reachable":true,"home_class":"answers","robots_served":true,"robots_sitemap":true,"content_signal":false,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":[],"blocks_any_ai":false,"files":{"llms_txt":false,"agents_md":false,"agent_skills":false,"mcp_card":false,"api_catalog":false,"a2a_card":false},"any_capability":false,"ai_refused":[],"ai_thin":[],"aipref":0,"aipref_measured":true,"sf":[4,0,0,0,0,0,1,0,1],"rings":{"mcp_declared":false,"mcp_live":false,"mcp_auth":false,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":3,"fh":"m.youtube.com","fp_other_host":[]},"derive_v":5,"frame":"top1k","shard":0},{"v":1,"kind":"crawlcheck-census-record","run":43,"domain":"apple.com","rank":10,"measured_at":"2026-11-13T07:57:55.549Z","home":{"browser":{"status":200,"content_type":"text/html","bytes":256049,"sha256":"5f260dd1e71a056e8ee1397e5e3fb63f3aaa3c90e1a312ec51b7b27054902de7","final_host":"www.apple.com","final_path":"/","ms":105,"identity":"Mobile browser","html":true,"fp":"55de96c41fca2d5e"},"gptbot":{"status":200,"content_type":"text/html","bytes":256049,"sha256":"5f260dd1e71a056e8ee1397e5e3fb63f3aaa3c90e1a312ec51b7b27054902de7","final_host":"www.apple.com","final_path":"/","ms":39,"identity":"GPTBot","html":true,"fp":"55de96c41fca2d5e"},"claudebot":{"status":200,"content_type":"text/html","bytes":256049,"sha256":"5f260dd1e71a056e8ee1397e5e3fb63f3aaa3c90e1a312ec51b7b27054902de7","final_host":"www.apple.com","final_path":"/","ms":56,"identity":"ClaudeBot","html":true,"fp":"55de96c41fca2d5e"},"oaisearch":{"status":200,"content_type":"text/html","bytes":256049,"sha256":"5f260dd1e71a056e8ee1397e5e3fb63f3aaa3c90e1a312ec51b7b27054902de7","final_host":"www.apple.com","final_path":"/","ms":11,"identity":"OAI-SearchBot","html":true,"fp":"55de96c41fca2d5e"},"perplexity":{"status":200,"content_type":"text/html","bytes":256049,"sha256":"5f260dd1e71a056e8ee1397e5e3fb63f3aaa3c90e1a312ec51b7b27054902de7","final_host":"www.apple.com","final_path":"/","ms":13,"identity":"PerplexityBot","html":true,"fp":"55de96c41fca2d5e"}},"robots":{"status":200,"content_type":"text/plain","bytes":1018,"sha256":"3115f45c98cfa19341978dcab5dfbc0fd3e0ff4eb3ab1fe8192c3fbf7227dee7","final_host":"www.apple.com","final_path":"/robots.txt","ms":8,"html":false,"text":"# robots.txt for https://www.apple.com/\n\nUser-agent: *\nDisallow: /*shop/browse/overlay/*\nDisallow: /*shop/iphone/payments/overlay/*\nDisallow: /cn/*/aow/*\nDisallow: /tmall*\nAllow: /ac/globalnav/2.0/*/images/ac-globalnav/globalnav/search/* \t\n\nUser-agent: Baiduspider\nDisallow: /*\nAllow: /cn/*\nAllow: /cn-edu/*\nAllow: /cn-k12/*\n\nUser-agent: HaoSouSpider\nDisallow: /*\nAllow: /cn/*\nAllow: /cn-edu/*\nAllow: /cn-k12/*\n\nUser-agent: Sogou web spider\nDisallow: /*\nAllow: /cn/*\nAllow: /cn-k12/*\n\nUser-agent: Sogou inst spider\nDisallow: /*\nAllow: /cn/*\nAllow: /cn-k12/*\n\nUser-agent: Sogou spider2\nDisallow: /*\nAllow: /cn/*\nAllow: /cn-k12/*\n\n#DaumWebMasterTool:6ff40e0f09564a9d4e3bc80e7cbde7c196677a2f5f610eb9c029836774f9a8e9:D9a8IEFAzmC/NWsQ3tB0Wg==\n\nSitemap: https://www.apple.com/shop/sitemap.xml\nSitemap: https://www.apple.com/autopush/sitemap/sitemap-index.xml\nSitemap: https://www.apple.com/newsroom/sitemap.xml\nSitemap: https://www.apple.com/retail/sitemap/sitemap.xml\nSitemap: https://www.apple.com/today/sitemap-index.xml\n","agents":{"GPTBot":true,"OAI-SearchBot":true,"ChatGPT-User":true,"ClaudeBot":true,"Claude-SearchBot":true,"PerplexityBot":true,"Google-Extended":true,"Applebot-Extended":true,"CCBot":true,"Bytespider":true,"Meta-ExternalAgent":true,"Amazonbot":true}},"sitemap":{"status":200,"content_type":"text/xml","bytes":3776,"sha256":"6143254373f03e6b21fad78999cd596f2e91f01925f97b588d1e337c36a558e5","final_host":"www.apple.com","final_path":"/shop/sitemap.xml","ms":248,"url":"https://www.apple.com/shop/sitemap.xml","declared":true,"xml":true},"files":{"llms_txt":{"status":404,"content_type":"text/html","bytes":109401,"sha256":"8653dc6569ae0b1a61e1c2d8dca414f9d12fd03c176d2fdc7d8634a03c0b3cd4","final_host":"www.apple.com","final_path":"/llms.txt","ms":400,"path":"/llms.txt","html":true,"json":null,"present":false},"agents_md":{"status":404,"content_type":"text/html","bytes":109401,"sha256":"8653dc6569ae0b1a61e1c2d8dca414f9d12fd03c176d2fdc7d8634a03c0b3cd4","final_host":"www.apple.com","final_path":"/agents.md","ms":318,"path":"/agents.md","html":true,"json":null,"present":false},"agent_skills":{"status":404,"content_type":"text/html","bytes":196,"sha256":"80c3fe2ae1062abf56456f52518bd670f9ec3917b7f85e152b347ac6b6faf880","final_host":"www.apple.com","final_path":"/.well-known/agent-skills/index.json","ms":253,"path":"/.well-known/agent-skills/index.json","html":true,"json":null,"present":false},"mcp_card":{"status":404,"content_type":"text/html","bytes":196,"sha256":"80c3fe2ae1062abf56456f52518bd670f9ec3917b7f85e152b347ac6b6faf880","final_host":"www.apple.com","final_path":"/.well-known/mcp/server-card.json","ms":260,"path":"/.well-known/mcp/server-card.json","html":true,"json":null,"present":false},"api_catalog":{"status":404,"content_type":"text/html","bytes":196,"sha256":"80c3fe2ae1062abf56456f52518bd670f9ec3917b7f85e152b347ac6b6faf880","final_host":"www.apple.com","final_path":"/.well-known/api-catalog","ms":333,"path":"/.well-known/api-catalog","html":true,"json":null,"present":false},"a2a_card":{"status":404,"content_type":"text/html","bytes":196,"sha256":"80c3fe2ae1062abf56456f52518bd670f9ec3917b7f85e152b347ac6b6faf880","final_host":"www.apple.com","final_path":"/.well-known/agent-card.json","ms":280,"path":"/.well-known/agent-card.json","html":true,"json":null,"present":false}},"rings":{"v":1,"candidates":[],"mcp":[],"a2a":null,"webmcp":{"model_context":false,"bridge":false,"mcp_json":false,"link_rel":false,"any":false},"handshakes":0,"get_path":{"status":404,"content_type":"text/html","ms":677,"error":null},"get_host":{"status":530,"content_type":"text/plain","ms":7,"error":null}},"sf":[4,0,0,0,0,0,0,0,0],"derived":{"reachable":true,"home_class":"answers","robots_served":true,"robots_sitemap":true,"content_signal":false,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":[],"blocks_any_ai":false,"files":{"llms_txt":false,"agents_md":false,"agent_skills":false,"mcp_card":false,"api_catalog":false,"a2a_card":false},"any_capability":false,"ai_refused":[],"ai_thin":[],"aipref":0,"aipref_measured":true,"sf":[4,0,0,0,0,0,0,0,0],"rings":{"mcp_declared":false,"mcp_live":false,"mcp_auth":false,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":3,"fh":"www.apple.com","fp_other_host":[],"fp":"55de96c41fca2d5e","fp_bits":{"gptbot":0,"claudebot":0,"oaisearch":0,"perplexity":0}},"derive_v":5,"frame":"top1k","shard":1},{"v":1,"kind":"crawlcheck-census-record","run":43,"domain":"instagram.com","rank":11,"measured_at":"2026-11-13T07:58:58.649Z","home":{"browser":{"status":null,"error":"timeout","ms":8000,"identity":"Mobile browser","html":null},"gptbot":{"status":null,"error":"timeout","ms":8000,"identity":"GPTBot","html":null},"claudebot":{"status":null,"error":"timeout","ms":8000,"identity":"ClaudeBot","html":null},"oaisearch":{"status":null,"error":"timeout","ms":8000,"identity":"OAI-SearchBot","html":null},"perplexity":{"status":null,"error":"timeout","ms":8000,"identity":"PerplexityBot","html":null}},"robots":{"status":200,"content_type":"text/plain","bytes":6394,"sha256":"3a62d0ddf2d87aa7cef3d1b7facf0bc49c10f7f4580755c0230fd2f89a679da5","final_host":"www.instagram.com","final_path":"/robots.txt","ms":253,"html":false,"text":"# Notice: Collection of data on Instagram through automated means is\n# prohibited unless you have express written permission from Instagram\n# and may only be conducted for the limited purpose contained in said\n# permission.\n# All authorized user-agents listed on this page must comply with Meta’s\n# Automated Data Collection Terms available at:\n# https://www.facebook.com/legal/automated_data_collection_terms\nUser-agent: __usage_ai-training\nDisallow: /\n\nUser-agent: Amazonbot\nDisallow: /\n\nUser-agent: Applebot-Extended\nDisallow: /\n\nUser-agent: Brightbot\nDisallow: /\n\nUser-agent: ClaudeBot\nDisallow: /\n\nUser-agent: Google-Extended\nDisallow: /\n\nUser-agent: GPTBot\nDisallow: /\n\nUser-agent: PerplexityBot\nDisallow: /\n\nUser-agent: PetalBot\nDisallow: /\n\nUser-agent: Scrapy\nDisallow: /\n\nUser-agent: uptimerobot\nDisallow: /\n\nUser-agent: viberbot\nDisallow: /\n\nUser-agent: YaK\nDisallow: /\n\nUser-agent: Yandex\nDisallow: /\n\nUser-agent: Yeti\nDisallow: /\n\nUser-agent: Applebot\nDisallow: /accounts/login/*?next=\nDisallow: /ajax/\nDisallow: /client_error/\nDisallow: /direct/\nDisallow: /logging/\nDisallow: /publicapi/\nDisallow: /qp/batch_fetch_web/\nDisallow: /query/\nAllow: /places/c/\n\nUser-agent: Bingbot\nDisallow: /*/c/\nDisallow: /*/comments/\nDisallow: /*/liked_by/\nDisallow: /accounts/login/*?next=\nDisallow: /ajax/\nDisallow: /client_error/\nDisallow: /direct/\nDisallow: /logging/\nDisallow: /publicapi/\nDisallow: /qp/batch_fetch_web/\nDisallow: /query/\nAllow: /places/c/\n\nUser-agent: deepcrawl\nDisallow: /accounts/login/*?next=\nDisallow: /ajax/\nDisallow: /client_error/\nDisallow: /direct/\nDisallow: /logging/\nDisallow: /publicapi/\nDisallow: /qp/batch_fetch_web/\nDisallow: /query/\nAllow: /places/c/\n\nUser-agent: Discordbot\nDisallow: /accounts/login/*?next=\nDisallow: /ajax/\nDisallow: /client_error/\nDisallow: /direct/\nDisallow: /logging/\nDisallow: /publicapi/\nDisallow: /qp/batch_fetch_web/\nDisallow: /query/\nAllow: /places/c/\n\nUser-agent: DuckDuckBot\nDisallow: /accounts/login/*?next=\nDisallow: /ajax/\nDisallow: /client_error/\nDisallow: /direct/\nDisallow: /logging/\nDisallow: /publicapi/\nDisallow: /qp/batch_fetch_web/\nDisallow: /query/\nAllow: /places/c/\n\nUser-agent: facebookexternalhit\nDisallow: /accounts/login/*?next=\nDisallow: /ajax/\nDisallow: /client_error/\nDisallow: /direct/\nDisallow: /logging/\nDisallow: /publicapi/\nDisallow: /qp/batch_fetch_web/\nDisallow: /query/\nAllow: /places/c/\n\nUser-agent: Googlebot\nDisallow: /*/c/\nDisallow: /*/comments/\nDisallow: /*/liked_by/\nDisallow: /accounts/login/*?next=\nDisallow: /ajax/\nDisallow: /client_error/\nDisallow: /direct/\nDisallow: /logging/\nDisallow: /publicapi/\nDisallow: /qp/batch_fetch_web/\nDisallow: /query/\nAllow: /places/c/\n\nUser-agent: Googlebot-Image\nDisallow: /accounts/login/*?next=\nDisallow: /ajax/\nDisallow: /client_error/\nDisallow: /direct/\nDisallow: /logging/\nDisallow: /publicapi/\nDisallow: /qp/batch_fetch_web/\nDisallow: /query/\nAllow: /places/c/\n\nUser-agent: Googlebot-Video\nDisallow: /accounts/login/*?next=\nDisallow: /ajax/\nDisallow: /client_error/\nDisallow: /direct/\nDisallow: /logging/\nDisallow: /publicapi/\nDisallow: /qp/batch_fetch_web/\nDisallow: /query/\nAllow: /places/c/\n\nUser-agent: Googlebot-News\nDisallow: /accounts/login/*?next=\nDisallow: /ajax/\nDisallow: /client_error/\nDisallow: /direct/\nDisallow: /logging/\nDisallow: /publicapi/\nDisallow: /qp/batch_fetch_web/\nDisallow: /query/\nAllow: /places/c/\n\nUser-agent: Google-InspectionTool\nDisallow: /accounts/login/*?next=\nDisallow: /ajax/\nDisallow: /client_error/\nDisallow: /direct/\nDisallow: /logging/\nDisallow: /publicapi/\nDisallow: /qp/batch_fetch_web/\nDisallow: /query/\nAllow: /places/c/\n\nUser-agent: lumar\nDisallow: /accounts/login/*?next=\nDisallow: /ajax/\nDisallow: /client_error/\nDisallow: /direct/\nDisallow: /logging/\nDisallow: /publicapi/\nDisallow: /qp/batch_fetch_web/\nDisallow: /query/\nAllow: /places/c/\n\nUser-agent: LinkedInBot\nDisallow: /accounts/login/*?next=\nDisallow: /ajax/\nDisallow: /client_error/\nDisallow: /direct/\nDisallow: /logging/\nDisallow: /publicapi/\nDisallow: /qp/batch_fetch_web/\nDisallow: /query/\nAllow: /places/c/\n\nUser-agent: msnbot\nDisallow: /accounts/login/*?next=\nDisallow: /ajax/\nDisallow: /client_error/\nDisallow: /direct/\nDisallow: /logging/\nDisallow: /publicapi/\nDisallow: /qp/batch_fetch_web/\nDisallow: /query/\nAllow: /places/c/\n\nUser-agent: Pinterestbot\nDisallow: /accounts/login/*?next=\nDisallow: /ajax/\nDisallow: /client_error/\nDisallow: /direct/\nDisallow: /logging/\nDisallow: /publicapi/\nDisallow: /qp/batch_fetch_web/\nDisallow: /query/\nAllow: /places/c/\n\nUser-agent: Screaming Frog SEO Spider\nDisallow: /accounts/login/*?next=\nDisallow: /ajax/\nDisallow: /client_error/\nDisallow: /direct/\nDisallow: /logging/\nDisallow: /publicapi/\nDisallow: /qp/batch_fetch_web/\nDisallow: /query/\nAllow: /places/c/\n\nUser-agent: seznambot\nDisallow: /accounts/login/*?next=\nDisallow: /ajax/\nDisallow: /client_error/\nDisallow: /direct/\nDisallow: /logging/\nDisallow: /publicapi/\nDisallow: /qp/batch_fetch_web/\nDisallow: /query/\nAllow: /places/c/\n\nUser-agent: Slurp\nDisallow: /accounts/login/*?next=\nDisallow: /ajax/\nDisallow: /client_error/\nDisallow: /direct/\nDisallow: /logging/\nDisallow: /publicapi/\nDisallow: /qp/batch_fetch_web/\nDisallow: /query/\nAllow: /places/c/\n\nUser-agent: teoma\nDisallow: /accounts/login/*?next=\nDisallow: /ajax/\nDisallow: /client_error/\nDisallow: /direct/\nDisallow: /logging/\nDisallow: /publicapi/\nDisallow: /qp/batch_fetch_web/\nDisallow: /query/\nAllow: /places/c/\n\nUser-agent: TelegramBot\nDisallow: /accounts/login/*?next=\nDisallow: /ajax/\nDisallow: /client_error/\nDisallow: /direct/\nDisallow: /logging/\nDisallow: /publicapi/\nDisallow: /qp/batch_fetch_web/\nDisallow: /query/\nAllow: /places/c/\n\nUser-agent: Twitterbot\nDisallow: /accounts/login/*?next=\nDisallow: /ajax/\nDisallow: /client_error/\nDisallow: /direct/\nDisallow: /logging/\nDisallow: /publicapi/\nDisallow: /qp/batch_fetch_web/\nDisallow: /query/\nAllow: /places/c/\n\nUser-agent: *\nDisallow: /\n\nSitemap: https://www.instagram.com/sitemap/ig_places_sitemap.xml.gz\nSitemap: https://www.instagram.com/sitemap/ig_seo_profile_sitemap.xml.gz\nSitemap: https://www.instagram.com/sitemap/ig_seo_profile_sitemap_non_media_eligible.xml.gz\nSitemap: https://www.instagram.com/sitemap/dim_ig_logged_out_web_keyword_ent.xml.gz\nSitemap: https://www.instagram.com/sitemap/ig_seo_profile_sitemap_v2.xml.gz\nSitemap: https://www.instagram.com/sitemap/ig_seo_historical_reels_mvp_video_rows.xml.gz\n\n","agents":{"GPTBot":false,"OAI-SearchBot":false,"ChatGPT-User":false,"ClaudeBot":false,"Claude-SearchBot":false,"PerplexityBot":false,"Google-Extended":false,"Applebot-Extended":false,"CCBot":false,"Bytespider":false,"Meta-ExternalAgent":false,"Amazonbot":false}},"sitemap":null,"files":null,"derived":{"reachable":false,"home_class":"no_site","robots_served":false,"robots_sitemap":false,"content_signal":false,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":[],"blocks_any_ai":false,"files":{"llms_txt":false,"agents_md":false,"agent_skills":false,"mcp_card":false,"api_catalog":false,"a2a_card":false},"any_capability":false,"ai_refused":[],"ai_thin":[],"rings":{"mcp_declared":false,"mcp_live":false,"mcp_auth":false,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":3,"fh":null,"fp_other_host":[]},"derive_v":5,"frame":"top1k","shard":2},{"v":1,"kind":"crawlcheck-census-record","run":43,"domain":"mail.ru","rank":12,"measured_at":"2026-11-13T07:59:19.049Z","home":{"browser":{"status":200,"content_type":"text/html","bytes":424700,"sha256":"d180d80f76f395f2710cadb0ca2b1dbe6ce803b8c029d0a05066a3b9be697062","final_host":"mail.ru","final_path":"/","ms":2271,"identity":"Mobile browser","html":true,"fp":"2d2b8f6d5b56a162"},"gptbot":{"status":200,"content_type":"text/html","bytes":936,"sha256":"15f6512ee7e04cc840afe5568c481d910ee70e8046bbb9a3a56dc5ad7f855543","final_host":"account.mail.ru","final_path":"/login","ms":2876,"identity":"GPTBot","html":true},"claudebot":{"status":200,"content_type":"text/html","bytes":936,"sha256":"9a0804011fabe138965f4568ba4020c6ccaab4634f280d786320e37ecb4f0e29","final_host":"account.mail.ru","final_path":"/login","ms":3190,"identity":"ClaudeBot","html":true},"oaisearch":{"status":200,"content_type":"text/html","bytes":936,"sha256":"616156346dfa3917b123e16cd143ec5ea11358eeecdb1bb1afa7ac013ee8b547","final_host":"account.mail.ru","final_path":"/login","ms":2800,"identity":"OAI-SearchBot","html":true},"perplexity":{"status":200,"content_type":"text/html","bytes":936,"sha256":"6c89829ee14f5f8ec9163d38e3716b546da7bf367a6ce22e45f14e0c39c7e1b3","final_host":"account.mail.ru","final_path":"/login","ms":2832,"identity":"PerplexityBot","html":true}},"robots":{"status":200,"content_type":"text/plain","bytes":433,"sha256":"d3fd6768e203ab4008918459ce8d1ccc9e31d703cc01c9765902bcfc894822bf","final_host":"mail.ru","final_path":"/robots.txt","ms":1925,"html":false,"text":"User-agent: *\nAllow: /*.css\nAllow: /*.jpg\nAllow: /*.gif\nAllow: /*.png\nAllow: /v/j/*.js\nAllow: /$\nDisallow: /\n\nUser-agent: Googlebot\nAllow: /*.css\nAllow: /*.jpg\nAllow: /*.gif\nAllow: /*.png\nAllow: /v/j/*.js\nDisallow: *?\nDisallow: *search\nDisallow: *auth\nDisallow: *api\n\nUser-agent: Yandex\nAllow: /*.css\nAllow: /*.jpg\nAllow: /*.gif\nAllow: /*.png\nAllow: /v/j/*.js\nAllow: /$\nDisallow: /\n\nUser-agent: Twitterbot\nDisallow: /\nAllow: /?logo=\n","agents":{"GPTBot":true,"OAI-SearchBot":true,"ChatGPT-User":true,"ClaudeBot":true,"Claude-SearchBot":true,"PerplexityBot":true,"Google-Extended":true,"Applebot-Extended":true,"CCBot":true,"Bytespider":true,"Meta-ExternalAgent":true,"Amazonbot":true}},"sitemap":{"status":200,"content_type":"text/html","bytes":936,"sha256":"eff10e3998b3efb2dbdaf3721c36cf3b033a8fdc5b12b0ce387491642265abe8","final_host":"account.mail.ru","final_path":"/login","ms":762,"url":"https://mail.ru/sitemap.xml","declared":false,"xml":false},"files":{"llms_txt":{"status":200,"content_type":"text/html","bytes":936,"sha256":"d8492fbdf72b0a956dc53f98654de15306c1f4fdd0197fc2774f09bedd58151a","final_host":"account.mail.ru","final_path":"/login","ms":1407,"path":"/llms.txt","html":true,"json":null,"present":false},"agents_md":{"status":200,"content_type":"text/html","bytes":936,"sha256":"ef93ed07b2d582654ba8c1c0353fce3ef849397f53fe380f3f4d126dda3cbeb9","final_host":"account.mail.ru","final_path":"/login","ms":1730,"path":"/agents.md","html":true,"json":null,"present":false},"agent_skills":{"status":200,"content_type":"text/html","bytes":936,"sha256":"90f28bf582df8cb3213ce2e541044083893d0d5e5eca536ade6b81c97a907a95","final_host":"account.mail.ru","final_path":"/login","ms":880,"path":"/.well-known/agent-skills/index.json","html":true,"json":null,"present":false},"mcp_card":{"status":200,"content_type":"text/html","bytes":936,"sha256":"9a77c1f9c17978ba1e1ed5d666c61a7ff1d2b9ea9f78f2488365c9897906b2cc","final_host":"account.mail.ru","final_path":"/login","ms":890,"path":"/.well-known/mcp/server-card.json","html":true,"json":null,"present":false},"api_catalog":{"status":200,"content_type":"text/html","bytes":936,"sha256":"f64fac0d94ad493cecd68d9796ba6bd0b9cfc7385b0849d00e18f34d7e04679f","final_host":"account.mail.ru","final_path":"/login","ms":831,"path":"/.well-known/api-catalog","html":true,"json":null,"present":false},"a2a_card":{"status":200,"content_type":"text/html","bytes":936,"sha256":"994823eecb4c731f355a910b483107d966fc1698ed2a95def1561caed1ce1456","final_host":"account.mail.ru","final_path":"/login","ms":792,"path":"/.well-known/agent-card.json","html":true,"json":null,"present":false}},"rings":{"v":1,"candidates":[],"mcp":[],"a2a":null,"webmcp":{"model_context":false,"bridge":false,"mcp_json":false,"link_rel":false,"any":false},"handshakes":0,"get_path":{"status":200,"content_type":"text/html","ms":783,"error":null},"get_host":{"status":200,"content_type":"text/html","ms":2276,"error":null}},"sf":[4,0,0,0,0,0,2,0,0],"derived":{"reachable":true,"home_class":"answers","robots_served":true,"robots_sitemap":false,"content_signal":false,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":[],"blocks_any_ai":false,"files":{"llms_txt":false,"agents_md":false,"agent_skills":false,"mcp_card":false,"api_catalog":false,"a2a_card":false},"any_capability":false,"ai_refused":[],"ai_thin":["gptbot","claudebot","oaisearch","perplexity"],"aipref":0,"aipref_measured":true,"sf":[4,0,0,0,0,0,2,0,0],"rings":{"mcp_declared":false,"mcp_live":false,"mcp_auth":false,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":3,"fh":"mail.ru","fp_other_host":["gptbot","claudebot","oaisearch","perplexity"],"fp":"2d2b8f6d5b56a162","fp_bits":{}},"derive_v":5,"frame":"top1k","shard":3},{"v":1,"kind":"crawlcheck-census-record","run":43,"domain":"fbcdn.net","rank":13,"measured_at":"2026-11-13T08:05:49.397Z","home":{"browser":{"status":400,"content_type":"text/html","bytes":3676,"sha256":"47a0876684b8d81868b9c7cc798dec8d7e7cf97de5113c0c97c4cd76efe611ce","final_host":"m.facebook.com","final_path":"/","ms":581,"identity":"Mobile browser","html":true},"gptbot":{"status":200,"content_type":"text/html","bytes":366802,"sha256":"2ed45ea4d35a9d3e09e772472105c4beafdafbae70dd81ffe1eb0be5a4a9a55d","final_host":"www.facebook.com","final_path":"/","ms":429,"identity":"GPTBot","html":true,"fp":"1eb5bc9249c88fab"},"claudebot":{"status":200,"content_type":"text/html","bytes":366873,"sha256":"3ae336ee6226b728363fb452bdad00d51f4e203579240481352bc98a49fb0dfc","final_host":"www.facebook.com","final_path":"/","ms":545,"identity":"ClaudeBot","html":true,"fp":"1eb5bc9249c88fab"},"oaisearch":{"status":200,"content_type":"text/html","bytes":366947,"sha256":"a33d6e5beafa6aebcebefe4012a2737ea7290021fbae2e45939ae01a69937dc7","final_host":"www.facebook.com","final_path":"/","ms":545,"identity":"OAI-SearchBot","html":true,"fp":"1eb5bc9249c88fab"},"perplexity":{"status":200,"content_type":"text/html","bytes":367712,"sha256":"c8fe249d7e930b87c50e73ebb42a827d3ccec8744ca7da1cf4141a050c25ccb6","final_host":"www.facebook.com","final_path":"/","ms":323,"identity":"PerplexityBot","html":true,"fp":"1eb5bc9249c88fab"}},"robots":{"status":200,"content_type":"text/plain","bytes":3400,"sha256":"17d601824568141914b8889d631e92c35aa17dec0be3e567c762cb670202d53f","final_host":"fbcdn.net","final_path":"/robots.txt","ms":120,"html":false,"text":"# Notice: Collection of data on Facebook through automated means is\n# prohibited unless you have express written permission from Facebook\n# and may only be conducted for the limited purpose contained in said\n# permission.\n# All authorized user-agents listed on this page must comply with Meta’s\n# Automated Data Collection Terms available at:\n# https://www.facebook.com/legal/automated_data_collection_terms\nUser-agent: __usage_ai-training\nDisallow: /\n\nUser-agent: Amazonbot\nDisallow: /\n\nUser-agent: Applebot-Extended\nDisallow: /\n\nUser-agent: Brightbot\nDisallow: /\n\nUser-agent: ClaudeBot\nDisallow: /\n\nUser-agent: Google-Extended\nDisallow: /\n\nUser-agent: GPTBot\nDisallow: /\n\nUser-agent: PerplexityBot\nDisallow: /\n\nUser-agent: PetalBot\nDisallow: /\n\nUser-agent: Scrapy\nDisallow: /\n\nUser-agent: uptimerobot\nDisallow: /\n\nUser-agent: viberbot\nDisallow: /\n\nUser-agent: YaK\nDisallow: /\n\nUser-agent: Yandex\nDisallow: /\n\nUser-agent: Yeti\nDisallow: /\n\nUser-agent: Applebot\nDisallow: /\nAllow: /map_tile.php\nAllow: /security/hsts-pixel.gif\nAllow: /static_map.php\n\nUser-agent: Bingbot\nDisallow: /\nAllow: /map_tile.php\nAllow: /security/hsts-pixel.gif\nAllow: /static_map.php\n\nUser-agent: deepcrawl\nDisallow: /\nAllow: /map_tile.php\nAllow: /security/hsts-pixel.gif\nAllow: /static_map.php\n\nUser-agent: Discordbot\nDisallow: /\nAllow: /map_tile.php\nAllow: /security/hsts-pixel.gif\nAllow: /static_map.php\n\nUser-agent: DuckDuckBot\nDisallow: /\nAllow: /map_tile.php\nAllow: /security/hsts-pixel.gif\nAllow: /static_map.php\n\nUser-agent: facebookexternalhit\nDisallow: /\nAllow: /map_tile.php\nAllow: /security/hsts-pixel.gif\nAllow: /static_map.php\n\nUser-agent: Googlebot\nDisallow: /\nAllow: /map_tile.php\nAllow: /security/hsts-pixel.gif\nAllow: /static_map.php\n\nUser-agent: Googlebot-Image\nDisallow: /\nDisallow: /map_tile.php\nDisallow: /static_map.php\nAllow: /map_tile.php\nAllow: /security/hsts-pixel.gif\nAllow: /static_map.php\n\nUser-agent: Googlebot-Video\nDisallow: /\nAllow: /map_tile.php\nAllow: /security/hsts-pixel.gif\nAllow: /static_map.php\n\nUser-agent: Googlebot-News\nDisallow: /\nAllow: /map_tile.php\nAllow: /security/hsts-pixel.gif\nAllow: /static_map.php\n\nUser-agent: Google-InspectionTool\nDisallow: /\nAllow: /map_tile.php\nAllow: /security/hsts-pixel.gif\nAllow: /static_map.php\n\nUser-agent: lumar\nDisallow: /\nAllow: /map_tile.php\nAllow: /security/hsts-pixel.gif\nAllow: /static_map.php\n\nUser-agent: LinkedInBot\nDisallow: /\nAllow: /map_tile.php\nAllow: /security/hsts-pixel.gif\nAllow: /static_map.php\n\nUser-agent: msnbot\nDisallow: /\nAllow: /map_tile.php\nAllow: /security/hsts-pixel.gif\nAllow: /static_map.php\n\nUser-agent: Pinterestbot\nDisallow: /\nAllow: /map_tile.php\nAllow: /security/hsts-pixel.gif\nAllow: /static_map.php\n\nUser-agent: Screaming Frog SEO Spider\nDisallow: /\nAllow: /map_tile.php\nAllow: /security/hsts-pixel.gif\nAllow: /static_map.php\n\nUser-agent: seznambot\nDisallow: /\nAllow: /map_tile.php\nAllow: /security/hsts-pixel.gif\nAllow: /static_map.php\n\nUser-agent: Slurp\nDisallow: /\nAllow: /map_tile.php\nAllow: /security/hsts-pixel.gif\nAllow: /static_map.php\n\nUser-agent: teoma\nDisallow: /\nAllow: /map_tile.php\nAllow: /security/hsts-pixel.gif\nAllow: /static_map.php\n\nUser-agent: TelegramBot\nDisallow: /\nAllow: /map_tile.php\nAllow: /security/hsts-pixel.gif\nAllow: /static_map.php\n\nUser-agent: Twitterbot\nDisallow: /\nAllow: /map_tile.php\nAllow: /security/hsts-pixel.gif\nAllow: /static_map.php\n\nUser-agent: *\nDisallow: /\n\n\n","agents":{"GPTBot":false,"OAI-SearchBot":false,"ChatGPT-User":false,"ClaudeBot":false,"Claude-SearchBot":false,"PerplexityBot":false,"Google-Extended":false,"Applebot-Extended":false,"CCBot":false,"Bytespider":false,"Meta-ExternalAgent":false,"Amazonbot":false}},"sitemap":null,"files":null,"derived":{"reachable":false,"home_class":"refused_us","robots_served":false,"robots_sitemap":false,"content_signal":false,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":[],"blocks_any_ai":false,"files":{"llms_txt":false,"agents_md":false,"agent_skills":false,"mcp_card":false,"api_catalog":false,"a2a_card":false},"any_capability":false,"ai_refused":[],"ai_thin":[],"rings":{"mcp_declared":false,"mcp_live":false,"mcp_auth":false,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":3,"fh":"m.facebook.com","fp_other_host":[]},"derive_v":5,"frame":"top1k","shard":4},{"v":1,"kind":"crawlcheck-census-record","run":43,"domain":"ezviz7.com","rank":14,"measured_at":"2026-11-13T08:07:00.657Z","home":{"browser":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"ezviz7.com","final_path":"/","ms":611,"identity":"Mobile browser","html":false},"gptbot":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"ezviz7.com","final_path":"/","ms":607,"identity":"GPTBot","html":false},"claudebot":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"ezviz7.com","final_path":"/","ms":606,"identity":"ClaudeBot","html":false},"oaisearch":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"ezviz7.com","final_path":"/","ms":605,"identity":"OAI-SearchBot","html":false},"perplexity":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"ezviz7.com","final_path":"/","ms":605,"identity":"PerplexityBot","html":false}},"robots":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"ezviz7.com","final_path":"/robots.txt","ms":604,"html":false,"text":null,"agents":null},"sitemap":null,"files":null,"derived":{"reachable":false,"home_class":"no_site","robots_served":false,"robots_sitemap":false,"content_signal":false,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":[],"blocks_any_ai":false,"files":{"llms_txt":false,"agents_md":false,"agent_skills":false,"mcp_card":false,"api_catalog":false,"a2a_card":false},"any_capability":false,"ai_refused":[],"ai_thin":[],"rings":{"mcp_declared":false,"mcp_live":false,"mcp_auth":false,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":3,"fh":"ezviz7.com","fp_other_host":[]},"derive_v":5,"frame":"top1k","shard":5},{"v":1,"kind":"crawlcheck-census-record","run":43,"domain":"twitter.com","rank":15,"measured_at":"2026-11-13T08:07:15.177Z","home":{"browser":{"status":200,"content_type":"text/html","bytes":46087,"sha256":"70a08266859851ac645fef75395295e504732804b91a4e49e3b98cd10f386c84","final_host":"x.com","final_path":"/","ms":106,"identity":"Mobile browser","html":true,"fp":"8b48d3564ad225ec"},"gptbot":{"status":402,"content_type":"application/json","bytes":55,"sha256":"35e642f0e1bd426b9d8b0ee0284aefe801873dc13c18819fbeeaf2bf47136664","final_host":"x.com","final_path":"/","ms":61,"identity":"GPTBot","html":false},"claudebot":{"status":403,"content_type":null,"bytes":0,"sha256":"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855","final_host":"x.com","final_path":"/","ms":87,"identity":"ClaudeBot","html":null},"oaisearch":{"status":402,"content_type":"application/json","bytes":55,"sha256":"35e642f0e1bd426b9d8b0ee0284aefe801873dc13c18819fbeeaf2bf47136664","final_host":"x.com","final_path":"/","ms":61,"identity":"OAI-SearchBot","html":false},"perplexity":{"status":402,"content_type":"application/json","bytes":55,"sha256":"35e642f0e1bd426b9d8b0ee0284aefe801873dc13c18819fbeeaf2bf47136664","final_host":"x.com","final_path":"/","ms":45,"identity":"PerplexityBot","html":false}},"robots":{"status":200,"content_type":"text/plain","bytes":3039,"sha256":"b54dd4fefb8786b6441dc95ab04db1dbe4c6b9fcb5e7bb60d8381ab6e40989ff","final_host":"twitter.com","final_path":"/robots.txt","ms":31,"html":false,"text":"# Google / Bing Search Engine Robots\n# ==================================\n# Shared group so Googlebot and Bingbot always get the same rules.\nUser-agent: Googlebot\nUser-agent: Bingbot\n\nAllow: /*?s=\nAllow: /*?t=\nAllow: /*?ref_src=\nAllow: /hashtag/*?src=\nAllow: /search?q=%23\nAllow: /i/api/\n# Same length as the /*/likes and /*/media rules; RFC 9309 breaks the tie in\n# favor of Allow, so /i/api/ fetches stay crawlable.\nAllow: /i/api/*\nDisallow: /*?lang=en-ss\nAllow: /*?lang=\nDisallow: /search/realtime\nDisallow: /search/users\nDisallow: /search/*/grid\n\nDisallow: /*/analytics\nDisallow: /*/followers\nDisallow: /*/following\nDisallow: /*/verified_followers\n\nDisallow: /account/deactivated\nDisallow: /settings/deactivated\n\n# Only `*` and `$` are metacharacters here (RFC 9309); regex classes match\n# literally. /photo is anchored so /status/*/photo/1 media pages stay open.\nDisallow: /*/likes\nDisallow: /*/likes?\nDisallow: /*/retweets\nDisallow: /*/retweets?\nDisallow: /*/media\nDisallow: /*/media?\nDisallow: /*/photo$\nDisallow: /*/photo?\nAllow: /*?\n\nUser-agent: facebookexternalhit\n\nAllow: /*?lang=\nAllow: /*?s=\nAllow: /*&s=\nAllow: /*?t=\nAllow: /*?ref_src=\nAllow: /hashtag/*?src=\nAllow: /search?*cashtagRestId=\nAllow: /i/api/\nAllow: /i/api/*\nDisallow: /search?q=\nDisallow: /search/realtime\nDisallow: /search/users\nDisallow: /search/*/grid\n\nDisallow: /*?\nDisallow: /*/followers\nDisallow: /*/following\nDisallow: /*/verified_followers\n\nDisallow: /account/deactivated\nDisallow: /settings/deactivated\n\nDisallow: /*/likes\nDisallow: /*/likes?\nDisallow: /*/retweets\nDisallow: /*/retweets?\nDisallow: /*/media\nDisallow: /*/media?\nDisallow: /*/photo$\nDisallow: /*/photo?\n\nUser-Agent: Google-Extended\nDisallow: *\n\nUser-Agent: FacebookBot\nDisallow: *\n\nUser-agent: Discordbot\nDisallow: *\n\n# Meta AI / product crawlers (product tokens; versions like /1.1 are ignored\n# under RFC 9309 token matching).\nUser-agent: meta-webindexer\nDisallow: *\n\nUser-agent: meta-externalagent\nDisallow: *\n\nUser-agent: meta-externalads\nDisallow: *\n\nUser-agent: meta-externalfetcher\nDisallow: *\n\n# Shared AI training preference (IETF draft-kelley-aipref-training-preference-signal),\n# honored by Microsoft, Google, OpenAI, and Anthropic. This is a usage policy\n# token, not a crawler: it does not change crawl access, and `User-agent: *`\n# never matches it.\nUser-agent: __usage_ai-training\nDisallow: /\n\n# Every bot that might possibly read and respect this file\n# ========================================================\nUser-agent: *\nDisallow: /\n\n\n# WHAT-4882 - Keep notification-email links (/i/u) out of search results.\n# Named crawlers stay un-blocked on purpose: they must crawl /i/u to see its\n# X-Robots-Tag noindex (the robots.txt Noindex directive died in 2019).\nDisallow: /i/u\n\n# Wait 1 second between successive requests. See ONBOARD-2698 for details.\nCrawl-delay: 1\n\n# Independent of user agent. Links in the sitemap are full URLs using https://\n# and need to match the protocol of the sitemap.\nSitemap: https://x.com/sitemap.xml\nSitemap: https://x.com/sitemaps/finance0\n","agents":{"GPTBot":false,"OAI-SearchBot":false,"ChatGPT-User":false,"ClaudeBot":false,"Claude-SearchBot":false,"PerplexityBot":false,"Google-Extended":false,"Applebot-Extended":false,"CCBot":false,"Bytespider":false,"Meta-ExternalAgent":false,"Amazonbot":false}},"sitemap":{"status":404,"content_type":"text/html","bytes":46082,"sha256":"2d5749b149031056f03ccb950a3ad5585d158856ae2db6ff794de91b425ef4b8","final_host":"x.com","final_path":"/sitemap.xml","ms":75,"url":"https://x.com/sitemap.xml","declared":true,"xml":false},"files":{"llms_txt":{"status":404,"content_type":"text/html","bytes":46063,"sha256":"7a6f25d0a84646a9f5f7f0a316336148d4d53b8ae37e07c3b0e4d9e002e75865","final_host":"x.com","final_path":"/llms.txt","ms":113,"path":"/llms.txt","html":true,"json":null,"present":false},"agents_md":{"status":404,"content_type":"text/html","bytes":46070,"sha256":"0690916fa4c05959b98ace9c98395e574aecfd7bf562ee2e1059459a134c99c0","final_host":"x.com","final_path":"/agents.md","ms":125,"path":"/agents.md","html":true,"json":null,"present":false},"agent_skills":{"status":404,"content_type":"text/html","bytes":46251,"sha256":"2d9dd919e01c5c943eb89a83e169340b4ec697fe07d3ee94ac75b4c5c6bec2ed","final_host":"x.com","final_path":"/.well-known/agent-skills/index.json","ms":120,"path":"/.well-known/agent-skills/index.json","html":true,"json":null,"present":false},"mcp_card":{"status":404,"content_type":"text/html","bytes":46234,"sha256":"e6eeb137a83208f031292ea7d75e11b61af175fbb6c59394031ba80d7daaf47a","final_host":"x.com","final_path":"/.well-known/mcp/server-card.json","ms":111,"path":"/.well-known/mcp/server-card.json","html":true,"json":null,"present":false},"api_catalog":{"status":404,"content_type":"text/html","bytes":46167,"sha256":"21b95f97b59362b2e50d8279a61d7711deb9a172f7b1efb0ce2d4653fe200ecb","final_host":"x.com","final_path":"/.well-known/api-catalog","ms":104,"path":"/.well-known/api-catalog","html":true,"json":null,"present":false},"a2a_card":{"status":404,"content_type":"text/html","bytes":46192,"sha256":"0cf6a72de835014fcba624a5f7040226996226dbfd8060d1c6adba70773a5d5e","final_host":"x.com","final_path":"/.well-known/agent-card.json","ms":105,"path":"/.well-known/agent-card.json","html":true,"json":null,"present":false}},"rings":{"v":1,"candidates":[],"mcp":[],"a2a":null,"webmcp":{"model_context":false,"bridge":false,"mcp_json":false,"link_rel":false,"any":false},"handshakes":0,"get_path":{"status":200,"content_type":"text/html","ms":509,"error":null},"get_host":{"status":530,"content_type":"text/plain","ms":14,"error":null}},"sf":[4,0,0,0,0,0,1,0,1],"derived":{"reachable":true,"home_class":"answers","robots_served":true,"robots_sitemap":true,"content_signal":false,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":["GPTBot","OAI-SearchBot","ChatGPT-User","ClaudeBot","Claude-SearchBot","PerplexityBot","Google-Extended","Applebot-Extended","CCBot","Bytespider","Meta-ExternalAgent","Amazonbot"],"blocks_any_ai":true,"files":{"llms_txt":false,"agents_md":false,"agent_skills":false,"mcp_card":false,"api_catalog":false,"a2a_card":false},"any_capability":false,"ai_refused":["gptbot","claudebot","oaisearch","perplexity"],"ai_thin":[],"aipref":2,"aipref_measured":true,"sf":[4,0,0,0,0,0,1,0,1],"rings":{"mcp_declared":false,"mcp_live":false,"mcp_auth":false,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":3,"fh":"x.com","fp_other_host":[],"fp":"8b48d3564ad225ec","fp_bits":{}},"derive_v":5,"frame":"top1k","shard":6},{"v":1,"kind":"crawlcheck-census-record","run":43,"domain":"domaincontrol.com","rank":16,"measured_at":"2026-11-13T08:08:20.375Z","home":{"browser":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"domaincontrol.com","final_path":"/","ms":55,"identity":"Mobile browser","html":false},"gptbot":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"domaincontrol.com","final_path":"/","ms":44,"identity":"GPTBot","html":false},"claudebot":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"domaincontrol.com","final_path":"/","ms":31,"identity":"ClaudeBot","html":false},"oaisearch":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"domaincontrol.com","final_path":"/","ms":31,"identity":"OAI-SearchBot","html":false},"perplexity":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"domaincontrol.com","final_path":"/","ms":14,"identity":"PerplexityBot","html":false}},"robots":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"domaincontrol.com","final_path":"/robots.txt","ms":1,"html":false,"text":null,"agents":null},"sitemap":null,"files":null,"derived":{"reachable":false,"home_class":"no_site","robots_served":false,"robots_sitemap":false,"content_signal":false,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":[],"blocks_any_ai":false,"files":{"llms_txt":false,"agents_md":false,"agent_skills":false,"mcp_card":false,"api_catalog":false,"a2a_card":false},"any_capability":false,"ai_refused":[],"ai_thin":[],"rings":{"mcp_declared":false,"mcp_live":false,"mcp_auth":false,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":3,"fh":"domaincontrol.com","fp_other_host":[]},"derive_v":5,"frame":"top1k","shard":7},{"v":1,"kind":"crawlcheck-census-record","run":43,"domain":"dzen.ru","rank":17,"measured_at":"2026-11-13T07:56:55.238Z","home":{"browser":{"status":200,"content_type":"text/html","bytes":3046,"sha256":"e5f361400aad4dba1ebff666763d4909b00cb3d31220ce31d3c700d26411ba16","final_host":"sso.passport.yandex.ru","final_path":"/push","ms":1109,"identity":"Mobile browser","html":true},"gptbot":{"status":200,"content_type":"text/html","bytes":2886,"sha256":"b0fabd30fd2d6c498e03908dadf8da0a8a3a880d33575ff0df87e818158ae155","final_host":"sso.passport.yandex.ru","final_path":"/push","ms":1070,"identity":"GPTBot","html":true},"claudebot":{"status":200,"content_type":"text/html","bytes":2883,"sha256":"96de1d47bfcf6996cc76fb947436c3837bb0e2b98573350475a7947e4a72a6d2","final_host":"sso.passport.yandex.ru","final_path":"/push","ms":1048,"identity":"ClaudeBot","html":true},"oaisearch":{"status":200,"content_type":"text/html","bytes":2886,"sha256":"de39b1ee73dd3ea50bdd0e69d42e84389998d432536dbd41bb9d35059b063539","final_host":"sso.passport.yandex.ru","final_path":"/push","ms":1072,"identity":"OAI-SearchBot","html":true},"perplexity":{"status":200,"content_type":"text/html","bytes":2886,"sha256":"8b520173ecf82e9b9ff922d94e05597536577aaa01fbcdbcadd42cb17a489bf0","final_host":"sso.passport.yandex.ru","final_path":"/push","ms":923,"identity":"PerplexityBot","html":true}},"robots":{"status":200,"content_type":"text/plain","bytes":9794,"sha256":"a6e7d16a66dcbbfe8d64ec966726922ac7736fd28f3a4b5bede7897992c92086","final_host":"dzen.ru","final_path":"/robots.txt","ms":329,"html":false,"text":"User-agent: Yandex\nAllow: /\nAllow: /.well-known/assetlinks.json\nAllow: /*?utm_sourse=55\n\nDisallow: /about*?*\nDisallow: /away\nDisallow: /top$\nDisallow: /search\nDisallow: /money\nDisallow: /onboarding\nDisallow: /news/top\nDisallow: /news/smi\nDisallow: /news/search\nDisallow: /t/\nDisallow: /user/\nDisallow: /embed/\nDisallow: /*/url\nDisallow: /video/games/tags/\nDisallow: /*sso_failed=\nDisallow: /*erid=\nDisallow: /*?utm*\nDisallow: /games\nDisallow: /subscriptions-manager?channel=\n\nDisallow: /articles?\nDisallow: /video?\nDisallow: /discussions\n\nClean-param: &_csrf\nClean-param: &action\nClean-param: &amp\nClean-param: &ampplace\nClean-param: &amupr_id\nClean-param: &appsearch_header\nClean-param: &autoplay\nClean-param: &backoffice\nClean-param: &branding\nClean-param: &briefEditorPublicationId\nClean-param: &bx_sender_conversion_id\nClean-param: &clid\nClean-param: &collection\nClean-param: &comment-request\nClean-param: &comments_data\nClean-param: &content\nClean-param: &contour\nClean-param: &country_code\nClean-param: &device_id\nClean-param: &encoded_pulse_user_id\nClean-param: &encoded_pulse_user_info\nClean-param: &epik\nClean-param: &etext=\nClean-param: &experiment\nClean-param: &f\nClean-param: &family\nClean-param: &fan\nClean-param: &favid\nClean-param: &feed_exp\nClean-param: &feed_filter_source\nClean-param: &firstchars\nClean-param: &flight_id\nClean-param: &from\nClean-param: &from_block\nClean-param: &from_cloud\nClean-param: &from_floors_promo\nClean-param: &from_page\nClean-param: &from_parent_id\nClean-param: &from_parent_type\nClean-param: &from_site\nClean-param: &ga_client_id\nClean-param: &hide_interest_header\nClean-param: &id\nClean-param: &integration\nClean-param: &interview_id\nClean-param: &invite\nClean-param: &item_id\nClean-param: &lang\nClean-param: &link\nClean-param: &loop\nClean-param: &lr\nClean-param: &modal\nClean-param: &mt_click_id\nClean-param: &mt_link_id\nClean-param: &mute\nClean-param: &no_ad\nClean-param: &noredir\nClean-param: &nw\nClean-param: &p\nClean-param: &page\nClean-param: &parent_rid\nClean-param: &persistent_id\nClean-param: &place\nClean-param: &play_on_visible\nClean-param: &pulse_user_id\nClean-param: &query\nClean-param: &ref\nClean-param: &referrer\nClean-param: &referrer_clid\nClean-param: &referrer_place\nClean-param: &repostId\nClean-param: &rid\nClean-param: &rubric\nClean-param: &secdata\nClean-param: &service\nClean-param: &share_to\nClean-param: &sid\nClean-param: &sortby\nClean-param: &source\nClean-param: &sso_failed\nClean-param: &stid\nClean-param: &story\nClean-param: &t\nClean-param: &text\nClean-param: &to\nClean-param: &token\nClean-param: &train\nClean-param: &tst\nClean-param: &tt\nClean-param: &tv\nClean-param: &utr\nClean-param: &uuid\nClean-param: &vertical\nClean-param: &via\nClean-param: &vid\nClean-param: &video_id\nClean-param: &wan\nClean-param: &web_uid\nClean-param: &web_yandexuid\nClean-param: &wizard\nClean-param: &yandexClientId\n\nUser-agent: Googlebot\nAllow: /\nAllow: /.well-known/assetlinks.json\nAllow: /list/*\n\nAllow: /a/*_csrf\nAllow: /a/*action\nAllow: /a/*amp\nAllow: /a/*ampplace\nAllow: /a/*amupr_id\nAllow: /a/*appsearch_header=\nAllow: /a/*autoplay=\nAllow: /a/*backoffice=\nAllow: /a/*branding=\nAllow: /a/*briefEditorPublicationId=\nAllow: /a/*bx_sender_conversion_id=\nAllow: /a/*cachebust=\nAllow: /a/*clid=\nAllow: /a/*collection=\nAllow: /a/*country_code=\nAllow: /a/*erid=\nAllow: /a/*experiment=\nAllow: /a/*feed_exp=\nAllow: /a/*from_cloud=\nAllow: /a/*from_page=\nAllow: /a/*invite=\nAllow: /a/*loop=\nAllow: /a/*pulse_user_id=\nAllow: /a/*sso_failed=\nAllow: /a/*t=\nAllow: /a/*to=\nAllow: /a/*token=\nAllow: /a/*utm_referrer=\nAllow: /a/*utm_source=\nAllow: /a/*?etext=\n\nAllow: /b/*_csrf\nAllow: /b/*action\nAllow: /b/*amp\nAllow: /b/*ampplace\nAllow: /b/*amupr_id\nAllow: /b/*appsearch_header=\nAllow: /b/*autoplay=\nAllow: /b/*backoffice=\nAllow: /b/*branding=\nAllow: /b/*briefEditorPublicationId=\nAllow: /b/*bx_sender_conversion_id=\nAllow: /b/*cachebust=\nAllow: /b/*clid=\nAllow: /b/*collection=\nAllow: /b/*country_code=\nAllow: /b/*erid=\nAllow: /b/*experiment=\nAllow: /b/*feed_exp=\nAllow: /b/*from_cloud=\nAllow: /b/*from_page=\nAllow: /b/*invite=\nAllow: /b/*loop=\nAllow: /b/*pulse_user_id=\nAllow: /b/*sso_failed=\nAllow: /b/*t=\nAllow: /b/*to=\nAllow: /b/*token=\nAllow: /b/*utm_referrer=\nAllow: /b/*utm_source=\nAllow: /b/*?etext=\n\nAllow: /list/*_csrf\nAllow: /list/*action\nAllow: /list/*amp\nAllow: /list/*ampplace\nAllow: /list/*amupr_id\nAllow: /list/*appsearch_header=\nAllow: /list/*autoplay=\nAllow: /list/*backoffice=\nAllow: /list/*branding=\nAllow: /list/*briefEditorPublicationId=\nAllow: /list/*bx_sender_conversion_id=\nAllow: /list/*cachebust=\nAllow: /list/*clid=\nAllow: /list/*collection=\nAllow: /list/*country_code=\nAllow: /list/*erid=\nAllow: /list/*experiment=\nAllow: /list/*feed_exp=\nAllow: /list/*from_cloud=\nAllow: /list/*from_page=\nAllow: /list/*invite=\nAllow: /list/*loop=\nAllow: /list/*pulse_user_id=\nAllow: /list/*sso_failed=\nAllow: /list/*t=\nAllow: /list/*to=\nAllow: /list/*token=\nAllow: /list/*utm_referrer=\nAllow: /list/*utm_source=\nAllow: /list/*?etext=\n\nAllow: /video/watch/*_csrf\nAllow: /video/watch/*action\nAllow: /video/watch/*amp\nAllow: /video/watch/*ampplace\nAllow: /video/watch/*amupr_id\nAllow: /video/watch/*appsearch_header=\nAllow: /video/watch/*autoplay=\nAllow: /video/watch/*backoffice=\nAllow: /video/watch/*branding=\nAllow: /video/watch/*briefEditorPublicationId=\nAllow: /video/watch/*bx_sender_conversion_id=\nAllow: /video/watch/*cachebust=\nAllow: /video/watch/*clid=\nAllow: /video/watch/*collection=\nAllow: /video/watch/*country_code=\nAllow: /video/watch/*erid=\nAllow: /video/watch/*experiment=\nAllow: /video/watch/*feed_exp=\nAllow: /video/watch/*from_cloud=\nAllow: /video/watch/*from_page=\nAllow: /video/watch/*invite=\nAllow: /video/watch/*loop=\nAllow: /video/watch/*pulse_user_id=\nAllow: /video/watch/*sso_failed=\nAllow: /video/watch/*t=\nAllow: /video/watch/*to=\nAllow: /video/watch/*token=\nAllow: /video/watch/*utm_referrer=\nAllow: /video/watch/*utm_source=\nAllow: /video/watch/*?etext=\n\nAllow: /shorts/*_csrf\nAllow: /shorts/*action\nAllow: /shorts/*amp\nAllow: /shorts/*ampplace\nAllow: /shorts/*amupr_id\nAllow: /shorts/*appsearch_header=\nAllow: /shorts/*autoplay=\nAllow: /shorts/*backoffice=\nAllow: /shorts/*branding=\nAllow: /shorts/*briefEditorPublicationId=\nAllow: /shorts/*bx_sender_conversion_id=\nAllow: /shorts/*cachebust=\nAllow: /shorts/*clid=\nAllow: /shorts/*collection=\nAllow: /shorts/*country_code=\nAllow: /shorts/*erid=\nAllow: /shorts/*experiment=\nAllow: /shorts/*feed_exp=\nAllow: /shorts/*from_cloud=\nAllow: /shorts/*from_page=\nAllow: /shorts/*invite=\nAllow: /shorts/*loop=\nAllow: /shorts/*pulse_user_id=\nAllow: /shorts/*sso_failed=\nAllow: /shorts/*t=\nAllow: /shorts/*to=\nAllow: /shorts/*token=\nAllow: /shorts/*utm_referrer=\nAllow: /shorts/*utm_source=\nAllow: /shorts/*?etext=\n\nDisallow: /about*?*\nDisallow: /away\nDisallow: /top$\nDisallow: /search\nDisallow: /money\nDisallow: /onboarding\nDisallow: /news/top\nDisallow: /news/smi\nDisallow: /news/search\nDisallow: /t/\nDisallow: /user/\nDisallow: /embed/\nDisallow: /*/url\nDisallow: /video/games/tags/\nDisallow: /*_csrf\nDisallow: /*action\nDisallow: /*amp\nDisallow: /*ampplace\nDisallow: /*amupr_id\nDisallow: /*appsearch_header=\nDisallow: /*autoplay=\nDisallow: /*backoffice=\nDisallow: /*branding=\nDisallow: /*briefEditorPublicationId=\nDisallow: /*bx_sender_conversion_id=\nDisallow: /*cachebust=\nDisallow: /*clid=\nDisallow: /*collection=\nDisallow: /*country_code=\nDisallow: /*erid=\nDisallow: /*experiment=\nDisallow: /*feed_exp=\nDisallow: /*from_cloud=\nDisallow: /*from_page=\nDisallow: /*invite=\nDisallow: /*loop=\nDisallow: /*pulse_user_id=\nDisallow: /*sso_failed=\nDisallow: /*t=\nDisallow: /*to=\nDisallow: /*token=\nDisallow: /*utm_referrer=\nDisallow: /*utm_source=\nDisallow: /games\nDisallow: /*?etext=\nDisallow: /subscriptions-manager?channel=\n\nDisallow: /articles?\nDisallow: /video?\n\nUser-Agent: *\nAllow: /\nAllow: /.well-known/assetlinks.json\nDisallow: /about*?*\nDisallow: /away\nDisallow: /embed/\nDisallow: /games\nDisallow: /money\nDisallow: /news/search\nDisallow: /news/smi\nDisallow: /news/top\nDisallow: /onboarding\nDisallow: /search\nDisallow: /subscriptions-manager?channel=\nDisallow: /t/\nDisallow: /top$\nDisallow: /user/\nDisallow: /*/url\nDisallow: /video/games/tags/\nDisallow: /*erid=\nDisallow: /*?etext=\nDisallow: /*sso_failed=\nDisallow: /*?sso_failed=\nDisallow: /*to=\n\nSitemap: https://dzen.ru/sitemaps/brief_v2/sitemap.xml\nSitemap: https://dzen.ru/sitemaps/channels_v2/sitemap.xml\nSitemap: https://dzen.ru/sitemaps/gif_long_v2/sitemap.xml\nSitemap: https://dzen.ru/sitemaps/gif_short_v2/sitemap.xml\nSitemap: https://dzen.ru/sitemaps/image_v2/sitemap.xml\nSitemap: https://dzen.ru/sitemaps/native_v2/sitemap.xml\nSitemap: https://dzen.ru/sitemaps/slug-factory/sitemap.xml\nSitemap: https://dzen.ru/sitemaps/suite_v2/sitemap.xml\nSitemap: https://dzen.ru/sitemaps/tv/sitemap.xml\nSitemap: https://dzen.ru/sitemaps/news-archive/sitemap.xml\nSitemap: https://dzen.ru/sitemaps/news-region/sitemap.xml\nSitemap: https://dzen.ru/sitemaps/news-rubric/sitemap.xml\nSitemap: https://dzen.ru/sitemaps/meta-channel_v2/sitemap.xml\n\nSitemap: https://dzen.ru/sitemaps/brief_v2/sitemap.xml.gz\nSitemap: https://dzen.ru/sitemaps/channels_v2/sitemap.xml.gz\nSitemap: https://dzen.ru/sitemaps/gif_long_v2/sitemap.xml.gz\nSitemap: https://dzen.ru/sitemaps/gif_short_v2/sitemap.xml.gz\nSitemap: https://dzen.ru/sitemaps/image_v2/sitemap.xml.gz\nSitemap: https://dzen.ru/sitemaps/native_v2/sitemap.xml.gz\nSitemap: https://dzen.ru/sitemaps/slug-factory/sitemap.xml.gz\nSitemap: https://dzen.ru/sitemaps/suite_v2/sitemap.xml.gz\nSitemap: https://dzen.ru/sitemaps/tv/sitemap.xml.gz\nSitemap: https://dzen.ru/sitemaps/news-archive/sitemap.xml.gz\nSitemap: https://dzen.ru/sitemaps/news-region/sitemap.xml.gz\nSitemap: https://dzen.ru/sitemaps/news-rubric/sitemap.xml.gz\nSitemap: https://dzen.ru/sitemaps/meta-channel_v2/sitemap.xml.gz","agents":{"GPTBot":true,"OAI-SearchBot":true,"ChatGPT-User":true,"ClaudeBot":true,"Claude-SearchBot":true,"PerplexityBot":true,"Google-Extended":true,"Applebot-Extended":true,"CCBot":true,"Bytespider":true,"Meta-ExternalAgent":true,"Amazonbot":true}},"sitemap":{"status":200,"content_type":"application/xml","bytes":32999,"sha256":"53b5f5bf3b102edebc96266ca9410de8e4e628c0ce2682f659d762d0fd1098c3","final_host":"dzen.ru","final_path":"/sitemaps/brief_v2/sitemap.xml","ms":114,"url":"https://dzen.ru/sitemaps/brief_v2/sitemap.xml","declared":true,"xml":true},"files":{"llms_txt":{"status":200,"content_type":"text/html","bytes":2893,"sha256":"1e264df5fff1ab3954280e069856faee223767fcd4e4ced07b5c29cc8b01d29a","final_host":"sso.passport.yandex.ru","final_path":"/push","ms":183,"path":"/llms.txt","html":true,"json":null,"present":false},"agents_md":{"status":200,"content_type":"text/html","bytes":2892,"sha256":"92f0068520cd6a2eb38c229f707849db3e5349eb9b8c92f1dcdff5cee8a43c02","final_host":"sso.passport.yandex.ru","final_path":"/push","ms":316,"path":"/agents.md","html":true,"json":null,"present":false},"agent_skills":{"status":200,"content_type":"text/html","bytes":2923,"sha256":"56f37c86084c1d7f47b37aed6a58e0f452c15f980364b5159bec0e347bc167f7","final_host":"sso.passport.yandex.ru","final_path":"/push","ms":181,"path":"/.well-known/agent-skills/index.json","html":true,"json":null,"present":false},"mcp_card":{"status":200,"content_type":"text/html","bytes":2923,"sha256":"1ef2b0597383e933697a68d6950938fcb10db339746a55ba15308ae1b143b0e5","final_host":"sso.passport.yandex.ru","final_path":"/push","ms":310,"path":"/.well-known/mcp/server-card.json","html":true,"json":null,"present":false},"api_catalog":{"status":200,"content_type":"text/html","bytes":2914,"sha256":"c67d91ee5e96e77f88cebb12adb929de427003c13c9484acb5792b866c345cc8","final_host":"sso.passport.yandex.ru","final_path":"/push","ms":344,"path":"/.well-known/api-catalog","html":true,"json":null,"present":false},"a2a_card":{"status":200,"content_type":"text/html","bytes":2913,"sha256":"387d22869ad0a3bc33cda089111f58abdd25bb387864475282e8306138331057","final_host":"sso.passport.yandex.ru","final_path":"/push","ms":311,"path":"/.well-known/agent-card.json","html":true,"json":null,"present":false}},"rings":{"v":1,"candidates":[],"mcp":[],"a2a":null,"webmcp":{"model_context":false,"bridge":false,"mcp_json":false,"link_rel":false,"any":false},"handshakes":0,"get_path":{"status":200,"content_type":"text/html","ms":191,"error":null},"get_host":{"status":530,"content_type":"text/plain","ms":215,"error":null}},"sf":[4,0,0,0,0,0,0,0,0],"derived":{"reachable":true,"home_class":"answers","robots_served":true,"robots_sitemap":true,"content_signal":false,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":[],"blocks_any_ai":false,"files":{"llms_txt":false,"agents_md":false,"agent_skills":false,"mcp_card":false,"api_catalog":false,"a2a_card":false},"any_capability":false,"ai_refused":[],"ai_thin":[],"aipref":0,"aipref_measured":true,"sf":[4,0,0,0,0,0,0,0,0],"rings":{"mcp_declared":false,"mcp_live":false,"mcp_auth":false,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":3,"fh":"sso.passport.yandex.ru","fp_other_host":[]},"derive_v":5,"frame":"top1k","shard":0},{"v":1,"kind":"crawlcheck-census-record","run":43,"domain":"linkedin.com","rank":18,"measured_at":"2026-11-13T07:57:55.549Z","home":{"browser":{"status":200,"content_type":"text/html","bytes":506144,"sha256":"29fa8d6f27544b94ea845106b26f5d4877221e7477a5887df7f9913c33f1d3c5","final_host":"www.linkedin.com","final_path":"/","ms":685,"identity":"Mobile browser","html":true,"fp":"e0d414696e54024d"},"gptbot":{"status":999,"content_type":"text/html","bytes":2109,"sha256":"388be16d000f70286349134384cbd80f7b8eeac84e265029d6a2ab0ab4128ad9","final_host":"linkedin.com","final_path":"/","ms":223,"identity":"GPTBot","html":true},"claudebot":{"status":403,"content_type":"text/html","bytes":5502,"sha256":"422d40b7cf933bdac6617ede585f4fb504c3a5d081f0316b1088a983c20484b3","final_host":"www.linkedin.com","final_path":"/","ms":241,"identity":"ClaudeBot","html":true},"oaisearch":{"status":200,"content_type":"text/html","bytes":141144,"sha256":"8c36c23c9c419f15561626b823867b7e4b6e871adb4fcbc5eb6754281e190ad1","final_host":"www.linkedin.com","final_path":"/","ms":465,"identity":"OAI-SearchBot","html":true,"fp":"e072000f7ba10acd"},"perplexity":{"status":403,"content_type":"text/html","bytes":5695,"sha256":"f689c2728e645d7708f9baf1e45fb2017b631dcd2b7c9a9d90dbd82f06f9bfe7","final_host":"www.linkedin.com","final_path":"/","ms":197,"identity":"PerplexityBot","html":true}},"robots":{"status":200,"content_type":"text/html","bytes":21343,"sha256":"d28c5c2e5b79b3e7120a5a37a20d63303e4db8d17fc1be3f6aa8e115ca583e4e","final_host":"linkedin.com","final_path":"/robots.txt","ms":179,"html":true,"text":null,"agents":null},"sitemap":{"status":200,"content_type":"text/html","bytes":21349,"sha256":"c4d16a14f56b32c3d156d01450609f930bee7fe792c5afb76dd433f7c478b9b5","final_host":"linkedin.com","final_path":"/sitemap.xml","ms":29,"url":"https://linkedin.com/sitemap.xml","declared":false,"xml":false},"files":{"llms_txt":{"status":200,"content_type":"text/html","bytes":21340,"sha256":"7e5b60a3e7fcacc794c1019c02fef2e7d08c7f22b243bfab053fa09728b5c90e","final_host":"linkedin.com","final_path":"/llms.txt","ms":71,"path":"/llms.txt","html":true,"json":null,"present":false},"agents_md":{"status":200,"content_type":"text/html","bytes":21342,"sha256":"87e5b8f2c83cac2de78758a640aa5f0a8a7e48bcb91ae29e0ab9a4be473c8acb","final_host":"linkedin.com","final_path":"/agents.md","ms":68,"path":"/agents.md","html":true,"json":null,"present":false},"agent_skills":{"status":200,"content_type":"text/html","bytes":21413,"sha256":"1cf7cd444084154d5720d30c631593bd2bdcc4fe77bbc4863dc402427976fbd3","final_host":"linkedin.com","final_path":"/.well-known/agent-skills/index.json","ms":72,"path":"/.well-known/agent-skills/index.json","html":true,"json":null,"present":false},"mcp_card":{"status":200,"content_type":"text/html","bytes":21404,"sha256":"9710ea80dccc8f50ce7851bb1cf00566343992b8a010ca29864b814951209276","final_host":"linkedin.com","final_path":"/.well-known/mcp/server-card.json","ms":34,"path":"/.well-known/mcp/server-card.json","html":true,"json":null,"present":false},"api_catalog":{"status":200,"content_type":"text/html","bytes":21380,"sha256":"b74f2b92dd9d652e92063a437567c31c8a1fd6b0cbc483c8791812e07ab08a5f","final_host":"linkedin.com","final_path":"/.well-known/api-catalog","ms":33,"path":"/.well-known/api-catalog","html":true,"json":null,"present":false},"a2a_card":{"status":200,"content_type":"text/html","bytes":21390,"sha256":"11ddca8fd31a359515fea7551e6aa5b1757a011fe5b7371f7bbafde5adfe85e1","final_host":"linkedin.com","final_path":"/.well-known/agent-card.json","ms":30,"path":"/.well-known/agent-card.json","html":true,"json":null,"present":false}},"rings":{"v":1,"candidates":[],"mcp":[],"a2a":null,"webmcp":{"model_context":false,"bridge":false,"mcp_json":false,"link_rel":false,"any":false},"handshakes":0,"get_path":{"status":200,"content_type":"text/html","ms":68,"error":null},"get_host":{"status":404,"content_type":"text/html","ms":344,"error":null}},"sf":[4,0,0,0,0,0,1,0,0],"derived":{"reachable":true,"home_class":"answers","robots_served":false,"robots_sitemap":false,"content_signal":false,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":[],"blocks_any_ai":false,"files":{"llms_txt":false,"agents_md":false,"agent_skills":false,"mcp_card":false,"api_catalog":false,"a2a_card":false},"any_capability":false,"ai_refused":["gptbot","claudebot","perplexity"],"ai_thin":["oaisearch"],"sf":[4,0,0,0,0,0,1,0,0],"rings":{"mcp_declared":false,"mcp_live":false,"mcp_auth":false,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":3,"fh":"www.linkedin.com","fp_other_host":[],"fp":"e0d414696e54024d","fp_bits":{"oaisearch":21}},"derive_v":5,"frame":"top1k","shard":1},{"v":1,"kind":"crawlcheck-census-record","run":43,"domain":"googlevideo.com","rank":19,"measured_at":"2026-11-13T07:58:58.649Z","home":{"browser":{"status":526,"content_type":"text/plain","bytes":16,"sha256":"4c543fe8a4d739c6f622b04303ac9e9d9fafbf00da950fc916ad826da2c0cdac","final_host":"googlevideo.com","final_path":"/","ms":63,"identity":"Mobile browser","html":false},"gptbot":{"status":526,"content_type":"text/plain","bytes":16,"sha256":"4c543fe8a4d739c6f622b04303ac9e9d9fafbf00da950fc916ad826da2c0cdac","final_host":"googlevideo.com","final_path":"/","ms":80,"identity":"GPTBot","html":false},"claudebot":{"status":526,"content_type":"text/plain","bytes":16,"sha256":"4c543fe8a4d739c6f622b04303ac9e9d9fafbf00da950fc916ad826da2c0cdac","final_host":"googlevideo.com","final_path":"/","ms":36,"identity":"ClaudeBot","html":false},"oaisearch":{"status":526,"content_type":"text/plain","bytes":16,"sha256":"4c543fe8a4d739c6f622b04303ac9e9d9fafbf00da950fc916ad826da2c0cdac","final_host":"googlevideo.com","final_path":"/","ms":32,"identity":"OAI-SearchBot","html":false},"perplexity":{"status":526,"content_type":"text/plain","bytes":16,"sha256":"4c543fe8a4d739c6f622b04303ac9e9d9fafbf00da950fc916ad826da2c0cdac","final_host":"googlevideo.com","final_path":"/","ms":35,"identity":"PerplexityBot","html":false}},"robots":{"status":526,"content_type":"text/plain","bytes":16,"sha256":"4c543fe8a4d739c6f622b04303ac9e9d9fafbf00da950fc916ad826da2c0cdac","final_host":"googlevideo.com","final_path":"/robots.txt","ms":44,"html":false,"text":null,"agents":null},"sitemap":null,"files":null,"derived":{"reachable":false,"home_class":"no_site","robots_served":false,"robots_sitemap":false,"content_signal":false,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":[],"blocks_any_ai":false,"files":{"llms_txt":false,"agents_md":false,"agent_skills":false,"mcp_card":false,"api_catalog":false,"a2a_card":false},"any_capability":false,"ai_refused":[],"ai_thin":[],"rings":{"mcp_declared":false,"mcp_live":false,"mcp_auth":false,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":3,"fh":"googlevideo.com","fp_other_host":[]},"derive_v":5,"frame":"top1k","shard":2},{"v":1,"kind":"crawlcheck-census-record","run":43,"domain":"akamaiedge.net","rank":20,"measured_at":"2026-11-13T07:59:19.049Z","home":{"browser":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"akamaiedge.net","final_path":"/","ms":7,"identity":"Mobile browser","html":false},"gptbot":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"akamaiedge.net","final_path":"/","ms":1,"identity":"GPTBot","html":false},"claudebot":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"akamaiedge.net","final_path":"/","ms":1,"identity":"ClaudeBot","html":false},"oaisearch":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"akamaiedge.net","final_path":"/","ms":1,"identity":"OAI-SearchBot","html":false},"perplexity":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"akamaiedge.net","final_path":"/","ms":0,"identity":"PerplexityBot","html":false}},"robots":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"akamaiedge.net","final_path":"/robots.txt","ms":1,"html":false,"text":null,"agents":null},"sitemap":null,"files":null,"derived":{"reachable":false,"home_class":"no_site","robots_served":false,"robots_sitemap":false,"content_signal":false,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":[],"blocks_any_ai":false,"files":{"llms_txt":false,"agents_md":false,"agent_skills":false,"mcp_card":false,"api_catalog":false,"a2a_card":false},"any_capability":false,"ai_refused":[],"ai_thin":[],"rings":{"mcp_declared":false,"mcp_live":false,"mcp_auth":false,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":3,"fh":"akamaiedge.net","fp_other_host":[]},"derive_v":5,"frame":"top1k","shard":3},{"v":1,"kind":"crawlcheck-census-record","run":43,"domain":"googletagmanager.com","rank":21,"measured_at":"2026-11-13T08:05:49.397Z","home":{"browser":{"status":404,"content_type":"text/html","bytes":1561,"sha256":"23926e9185d8d43c02807a838ffb373cc1977726094a4e46807c66ada9dd7660","final_host":"googletagmanager.com","final_path":"/","ms":117,"identity":"Mobile browser","html":true},"gptbot":{"status":404,"content_type":"text/html","bytes":1561,"sha256":"23926e9185d8d43c02807a838ffb373cc1977726094a4e46807c66ada9dd7660","final_host":"googletagmanager.com","final_path":"/","ms":33,"identity":"GPTBot","html":true},"claudebot":{"status":404,"content_type":"text/html","bytes":1561,"sha256":"23926e9185d8d43c02807a838ffb373cc1977726094a4e46807c66ada9dd7660","final_host":"googletagmanager.com","final_path":"/","ms":29,"identity":"ClaudeBot","html":true},"oaisearch":{"status":404,"content_type":"text/html","bytes":1561,"sha256":"23926e9185d8d43c02807a838ffb373cc1977726094a4e46807c66ada9dd7660","final_host":"googletagmanager.com","final_path":"/","ms":22,"identity":"OAI-SearchBot","html":true},"perplexity":{"status":404,"content_type":"text/html","bytes":1561,"sha256":"23926e9185d8d43c02807a838ffb373cc1977726094a4e46807c66ada9dd7660","final_host":"googletagmanager.com","final_path":"/","ms":73,"identity":"PerplexityBot","html":true}},"robots":{"status":404,"content_type":"text/html","bytes":1571,"sha256":"6adee83a3eba87d30fdca3997d27097697f2767805f222a0f78c3f04392004d2","final_host":"googletagmanager.com","final_path":"/robots.txt","ms":85,"html":true,"text":null,"agents":null},"sitemap":null,"files":null,"derived":{"reachable":false,"home_class":"other","robots_served":false,"robots_sitemap":false,"content_signal":false,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":[],"blocks_any_ai":false,"files":{"llms_txt":false,"agents_md":false,"agent_skills":false,"mcp_card":false,"api_catalog":false,"a2a_card":false},"any_capability":false,"ai_refused":[],"ai_thin":[],"rings":{"mcp_declared":false,"mcp_live":false,"mcp_auth":false,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":3,"fh":"googletagmanager.com","fp_other_host":[]},"derive_v":5,"frame":"top1k","shard":4},{"v":1,"kind":"crawlcheck-census-record","run":43,"domain":"office.com","rank":22,"measured_at":"2026-11-13T08:07:00.657Z","home":{"browser":{"status":200,"content_type":"text/html","bytes":310751,"sha256":"2dd0e68930f85c59602b053111647575aacaf27708127e674fd4e753b3712d9f","final_host":"m365.cloud.microsoft","final_path":"/","ms":347,"identity":"Mobile browser","html":true,"fp":"ca43656b27cb4a20"},"gptbot":{"status":200,"content_type":"text/html","bytes":331673,"sha256":"bbb12b79f69554be2326685b8c0bc07bb95e32e5850de37e0ae9484bb4f9bb50","final_host":"m365.cloud.microsoft","final_path":"/","ms":302,"identity":"GPTBot","html":true,"fp":"dbc3656ba3c34260"},"claudebot":{"status":200,"content_type":"text/html","bytes":331673,"sha256":"bbb12b79f69554be2326685b8c0bc07bb95e32e5850de37e0ae9484bb4f9bb50","final_host":"m365.cloud.microsoft","final_path":"/","ms":302,"identity":"ClaudeBot","html":true,"fp":"dbc3656ba3c34260"},"oaisearch":{"status":200,"content_type":"text/html","bytes":331673,"sha256":"bbb12b79f69554be2326685b8c0bc07bb95e32e5850de37e0ae9484bb4f9bb50","final_host":"m365.cloud.microsoft","final_path":"/","ms":302,"identity":"OAI-SearchBot","html":true,"fp":"dbc3656ba3c34260"},"perplexity":{"status":200,"content_type":"text/html","bytes":331673,"sha256":"bbb12b79f69554be2326685b8c0bc07bb95e32e5850de37e0ae9484bb4f9bb50","final_host":"m365.cloud.microsoft","final_path":"/","ms":302,"identity":"PerplexityBot","html":true,"fp":"dbc3656ba3c34260"}},"robots":{"status":200,"content_type":"text/html","bytes":60362,"sha256":"b5a4f1052be4d5b9456166c001f7f06dbf28c310b0cfb12a63cdcc16560c48d1","final_host":"login.microsoftonline.com","final_path":"/common/oauth2/v2.0/authorize","ms":331,"html":true,"text":null,"agents":null},"sitemap":{"status":200,"content_type":"text/html","bytes":60432,"sha256":"3afbec13d32826dd5f6ee1b0eefd6b4c0fc8af0da57bfe8a4376f9d920b15db8","final_host":"login.microsoftonline.com","final_path":"/common/oauth2/v2.0/authorize","ms":138,"url":"https://office.com/sitemap.xml","declared":false,"xml":false},"files":{"llms_txt":{"status":200,"content_type":"text/html","bytes":60391,"sha256":"5c38015a95c26b16da70ba708f4ae78c2f3afe351c36afef4ca2aa6eae855b9e","final_host":"login.microsoftonline.com","final_path":"/common/oauth2/v2.0/authorize","ms":154,"path":"/llms.txt","html":true,"json":null,"present":false},"agents_md":{"status":200,"content_type":"text/html","bytes":60351,"sha256":"c1c4c1bbe673407258077f38cba5b5947ddbd39a5514870197b3bb9ccfe05eb1","final_host":"login.microsoftonline.com","final_path":"/common/oauth2/v2.0/authorize","ms":150,"path":"/agents.md","html":true,"json":null,"present":false},"agent_skills":{"status":200,"content_type":"text/html","bytes":60627,"sha256":"698e77b0704b251d848b741969505465ac76ea1a710b33b7068e9da8882d0d4b","final_host":"login.microsoftonline.com","final_path":"/common/oauth2/v2.0/authorize","ms":94,"path":"/.well-known/agent-skills/index.json","html":true,"json":null,"present":false},"mcp_card":{"status":200,"content_type":"text/html","bytes":60797,"sha256":"50a1508e988f5c7d0f051363298a87ffd66d996b46aa0fb98cc2c60d8d9f0dc3","final_host":"login.microsoftonline.com","final_path":"/common/oauth2/v2.0/authorize","ms":153,"path":"/.well-known/mcp/server-card.json","html":true,"json":null,"present":false},"api_catalog":{"status":200,"content_type":"text/html","bytes":60662,"sha256":"addb80c1292798fad0aa94e6c86259763e0dd17b9aa8771328197cfc68bbe406","final_host":"login.microsoftonline.com","final_path":"/common/oauth2/v2.0/authorize","ms":143,"path":"/.well-known/api-catalog","html":true,"json":null,"present":false},"a2a_card":{"status":200,"content_type":"text/html","bytes":60748,"sha256":"958d6f9b54acd72004d20a5970eff62ebd0577f1b2ac9b40e1c11d2ca6a2258b","final_host":"login.microsoftonline.com","final_path":"/common/oauth2/v2.0/authorize","ms":109,"path":"/.well-known/agent-card.json","html":true,"json":null,"present":false}},"rings":{"v":1,"candidates":[],"mcp":[],"a2a":null,"webmcp":{"model_context":false,"bridge":false,"mcp_json":false,"link_rel":false,"any":false},"handshakes":0,"get_path":{"status":200,"content_type":"text/html","ms":137,"error":null},"get_host":{"status":530,"content_type":"text/plain","ms":9,"error":null}},"sf":[4,0,0,0,0,0,2,16,0],"derived":{"reachable":true,"home_class":"answers","robots_served":false,"robots_sitemap":false,"content_signal":false,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":[],"blocks_any_ai":false,"files":{"llms_txt":false,"agents_md":false,"agent_skills":false,"mcp_card":false,"api_catalog":false,"a2a_card":false},"any_capability":false,"ai_refused":[],"ai_thin":[],"sf":[4,0,0,0,0,0,2,16,0],"rings":{"mcp_declared":false,"mcp_live":false,"mcp_auth":false,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":3,"fh":"m365.cloud.microsoft","fp_other_host":[],"fp":"ca43656b27cb4a20","fp_bits":{"gptbot":8,"claudebot":8,"oaisearch":8,"perplexity":8}},"derive_v":5,"frame":"top1k","shard":5},{"v":1,"kind":"crawlcheck-census-record","run":43,"domain":"live.com","rank":23,"measured_at":"2026-11-13T08:07:15.177Z","home":{"browser":{"status":521,"content_type":"text/plain","bytes":16,"sha256":"4558ae09802e147610ebdb88e4472fe625c897af9a18b04d47dfcdb6b563a138","final_host":"outlook.live.com","final_path":"/owa","ms":180,"identity":"Mobile browser","html":false},"gptbot":{"status":null,"error":"timeout","ms":8001,"identity":"GPTBot","html":null},"claudebot":{"status":null,"error":"timeout","ms":8000,"identity":"ClaudeBot","html":null},"oaisearch":{"status":null,"error":"timeout","ms":8000,"identity":"OAI-SearchBot","html":null},"perplexity":{"status":null,"error":"timeout","ms":8000,"identity":"PerplexityBot","html":null}},"robots":{"status":null,"error":"timeout","ms":8000,"html":null,"text":null,"agents":null},"sitemap":null,"files":null,"derived":{"reachable":false,"home_class":"no_site","robots_served":false,"robots_sitemap":false,"content_signal":false,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":[],"blocks_any_ai":false,"files":{"llms_txt":false,"agents_md":false,"agent_skills":false,"mcp_card":false,"api_catalog":false,"a2a_card":false},"any_capability":false,"ai_refused":[],"ai_thin":[],"rings":{"mcp_declared":false,"mcp_live":false,"mcp_auth":false,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":3,"fh":"outlook.live.com","fp_other_host":[]},"derive_v":5,"frame":"top1k","shard":6},{"v":1,"kind":"crawlcheck-census-record","run":43,"domain":"akadns.net","rank":24,"measured_at":"2026-11-13T08:08:20.375Z","home":{"browser":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"akadns.net","final_path":"/","ms":5,"identity":"Mobile browser","html":false},"gptbot":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"akadns.net","final_path":"/","ms":5,"identity":"GPTBot","html":false},"claudebot":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"akadns.net","final_path":"/","ms":5,"identity":"ClaudeBot","html":false},"oaisearch":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"akadns.net","final_path":"/","ms":5,"identity":"OAI-SearchBot","html":false},"perplexity":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"akadns.net","final_path":"/","ms":4,"identity":"PerplexityBot","html":false}},"robots":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"akadns.net","final_path":"/robots.txt","ms":6,"html":false,"text":null,"agents":null},"sitemap":null,"files":null,"derived":{"reachable":false,"home_class":"no_site","robots_served":false,"robots_sitemap":false,"content_signal":false,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":[],"blocks_any_ai":false,"files":{"llms_txt":false,"agents_md":false,"agent_skills":false,"mcp_card":false,"api_catalog":false,"a2a_card":false},"any_capability":false,"ai_refused":[],"ai_thin":[],"rings":{"mcp_declared":false,"mcp_live":false,"mcp_auth":false,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":3,"fh":"akadns.net","fp_other_host":[]},"derive_v":5,"frame":"top1k","shard":7},{"v":1,"kind":"crawlcheck-census-record","run":43,"domain":"hicloudcam.com","rank":25,"measured_at":"2026-11-13T07:56:55.238Z","home":{"browser":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"hicloudcam.com","final_path":"/","ms":614,"identity":"Mobile browser","html":false},"gptbot":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"hicloudcam.com","final_path":"/","ms":43,"identity":"GPTBot","html":false},"claudebot":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"hicloudcam.com","final_path":"/","ms":42,"identity":"ClaudeBot","html":false},"oaisearch":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"hicloudcam.com","final_path":"/","ms":35,"identity":"OAI-SearchBot","html":false},"perplexity":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"hicloudcam.com","final_path":"/","ms":24,"identity":"PerplexityBot","html":false}},"robots":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"hicloudcam.com","final_path":"/robots.txt","ms":18,"html":false,"text":null,"agents":null},"sitemap":null,"files":null,"derived":{"reachable":false,"home_class":"no_site","robots_served":false,"robots_sitemap":false,"content_signal":false,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":[],"blocks_any_ai":false,"files":{"llms_txt":false,"agents_md":false,"agent_skills":false,"mcp_card":false,"api_catalog":false,"a2a_card":false},"any_capability":false,"ai_refused":[],"ai_thin":[],"rings":{"mcp_declared":false,"mcp_live":false,"mcp_auth":false,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":3,"fh":"hicloudcam.com","fp_other_host":[]},"derive_v":5,"frame":"top1k","shard":0},{"v":1,"kind":"crawlcheck-census-record","run":43,"domain":"amazon.com","rank":26,"measured_at":"2026-11-13T07:57:55.549Z","home":{"browser":{"status":202,"content_type":"text/html","bytes":2012,"sha256":"64960fb6f659003c901b850a4f992156eca56d013c219c43a02e428d412eb5c9","final_host":"www.amazon.com","final_path":"/","ms":754,"identity":"Mobile browser","html":true},"gptbot":{"status":202,"content_type":"text/html","bytes":2012,"sha256":"49780fc5ae345034f9fe84a224c2d9bb61ce9b3c84be97f0526f6636fd142a30","final_host":"www.amazon.com","final_path":"/","ms":784,"identity":"GPTBot","html":true},"claudebot":{"status":202,"content_type":"text/html","bytes":2012,"sha256":"48e50951121c74a48a91a2d475fa7e22fbb2cf4150eff7e783b3af66b4a5aed7","final_host":"www.amazon.com","final_path":"/","ms":749,"identity":"ClaudeBot","html":true},"oaisearch":{"status":202,"content_type":"text/html","bytes":2012,"sha256":"4b249932604425befdf81e7ac9b0ecd014009887d334db51f4252a6ba3d8cc87","final_host":"www.amazon.com","final_path":"/","ms":758,"identity":"OAI-SearchBot","html":true},"perplexity":{"status":202,"content_type":"text/html","bytes":2012,"sha256":"95e8749edab4ce5d3202942543f15d5cd906bbe5033f46c6644fe00497d94b84","final_host":"www.amazon.com","final_path":"/","ms":744,"identity":"PerplexityBot","html":true}},"robots":{"status":200,"content_type":"text/plain","bytes":7947,"sha256":"74fa27e28cd9d80bdd155ffecf3a033ea2e195a80c83d6fa46acb20665c3cb4d","final_host":"www.amazon.com","final_path":"/robots.txt","ms":1075,"html":false,"text":"User-agent: *\nDisallow: /exec/obidos/account-access-login\nDisallow: /exec/obidos/change-style\nDisallow: /exec/obidos/flex-sign-in\nDisallow: /exec/obidos/handle-buy-box\nDisallow: /exec/obidos/tg/cm/member/\nDisallow: /gp/aw/help/id=sss\nDisallow: /gp/cart\nDisallow: /gp/flex\nDisallow: /gp/product/e-mail-friend\nDisallow: /gp/product/product-availability\nDisallow: /gp/product/rate-this-item\nDisallow: /gp/sign-in\nDisallow: /gp/reader\nDisallow: /gp/sitbv3/reader\nDisallow: /gp/richpub/syltguides/create\nDisallow: /gp/gfix\nDisallow: /gp/associations/wizard.html\nDisallow: /gp/dmusic/order\nDisallow: /gp/legacy-handle-buy-box.html\nDisallow: /gp/aws/ssop\nDisallow: /gp/yourstore\nDisallow: /gp/gift-central/organizer/add-wishlist\nDisallow: /gp/vote\nDisallow: /gp/voting/\nDisallow: /gp/music/wma-pop-up\nDisallow: /gp/customer-images\nDisallow: /gp/richpub/listmania/createpipeline\nDisallow: /gp/content-form\nDisallow: /gp/pdp/invitation/invite\nDisallow: /gp/customer-reviews/common/du\nDisallow: /gp/customer-reviews/write-a-review.html\nDisallow: /gp/associations/wizard.html\nDisallow: /gp/music/clipserve\nDisallow: /gp/customer-media/upload\nDisallow: /gp/history\nDisallow: /gp/item-dispatch\nDisallow: /gp/dmusic/order/handle-buy-box.html\nDisallow: /gp/recsradio\nDisallow: /gp/slredirect\nDisallow: /dp/shipping/\nDisallow: /dp/twister-update/\nDisallow: /dp/manual-submit/\nDisallow: /dp/e-mail-friend/\nDisallow: /dp/product-availability/\nDisallow: /dp/rate-this-item/\nDisallow: /gp/registry/wishlist/*/reserve\nDisallow: /gp/structured-ratings/actions/get-experience.html\nDisallow: /gp/twitter/\nDisallow: /ap/signin\nDisallow: /gp/registry/wishlist/\nDisallow: /wishlist/\nAllow: /wishlist/universal*\nAllow: /wishlist/vendor-button*\nAllow: /wishlist/get-button*\nDisallow: /gp/wishlist/\nAllow: /gp/wishlist/universal*\nAllow: /gp/wishlist/vendor-button*\nAllow: /gp/wishlist/ipad-install*\nDisallow: /registry/wishlist/\nDisallow: /review/common/du\nDisallow: /gp/registry/search.html\nDisallow: /product-reviews/B0069IY63Y\nDisallow: /gp/orc/rml/\nDisallow: */gcrnsts\nDisallow: /gp/gc/widget\nDisallow: /gp/dmusic/mp3/player\nDisallow: /gp/entity-alert/external\nDisallow: /gp/customer-reviews/dynamic/sims-box\nDisallow: /review/dynamic/sims-box\nDisallow: /gp/redirect.html\nDisallow: /gp/twister/ajaxv2\nDisallow: /ss/twister/ajax\nDisallow: /b?*node=7454917011\nDisallow: /b?*node=7454927011\nDisallow: /b?*node=7454939011\nDisallow: /b?*node=7454898011\nDisallow: /gp/customer-media/actions/delete/\nDisallow: /gp/customer-media/actions/edit-caption/\nDisallow: /gp/dmusic/\nAllow: /gp/dmusic/promotions/PrimeMusic\nAllow: /gp/dmusic/promotions/AmazonMusicUnlimited\nDisallow: /gp/offer-listing/\nDisallow: /b?*node=9052533011\nDisallow: /lm/R1XIHQVKXSKBNJ\nDisallow: /lm/R3HQ5WJSZK6QSO\nDisallow: /surprise/\nDisallow: /local/ajax/\nDisallow: /gp/socialmedia/giveaways\nDisallow: /gp/b2b-rd\nDisallow: /gp/aw/so.html\nDisallow: /reviews/iframe\nDisallow: /gp/switch-language\nDisallow: /ga/p/\nDisallow: /gp/profile/\nDisallow: /giveaway/host/setup/\nDisallow: /ss/customer-reviews/lighthouse/\nDisallow: /ospublishing/story/*\nDisallow: /gp/aw/ol/\nDisallow: /gp/promotion/\nDisallow: /hz/leaderboard/top-reviewers/\nDisallow: /creatorhub\nDisallow: /creatorhub/*\nDisallow: /slp/s$\nDisallow: /-/\nAllow: /-/es/\nAllow: /-/en$\nAllow: /-/zh_TW/\nAllow: /-/zh_TW$\nAllow: /-/he/\nAllow: /-/he$\nDisallow: /hz/help/contact/*/message/$\nDisallow: /gp/aw/shoppingAids/\nDisallow: /rss/people/*/reviews\nDisallow: /gp/pdp/rss/*/reviews\nDisallow: /gp/cdp/member-reviews/\nDisallow: /gp/aw/cr/\nAllow: /gp/offer-listing/B000\nAllow: /gp/offer-listing/9000\nDisallow: /gp/aag\nAllow: /gp/aag/main?*seller=ABVFEJU8LS620\nDisallow: /gp/pdp/profile/\nDisallow: /gp/help/customer/express/c2c/\nDisallow: /slp/*/b$\nDisallow: /hz/contact-us/ajax/initiate-trusted-contact/\nDisallow: /hp/video/\nDisallow: /gp/video/api\nDisallow: /gp/video/settings\nDisallow: /gp/video/library\nDisallow: /gp/video/watchlist\nDisallow: /gp/video/mystuff\nDisallow: /gp/video/profiles\nDisallow: /gp/video/search\nDisallow: /gp/video/auth\nDisallow: /gp/video/salp/i\nDisallow: /icdp/\nDisallow: /shoptheshow/*?cat=\nDisallow: /shoptheshow/*&cat=\n\nUser-agent: EtaoSpider\nDisallow: /\n\nUser-agent: GPTBot\nDisallow: /\n\n\nUser-agent: CCBot\nDisallow: /\n\nUser-Agent: PerplexityBot\nDisallow: /\n\nUser-agent: Google-Extended\nDisallow: /\n\nUser-agent: GoogleAgent-Mariner\nDisallow: /\n\nUser-agent: GoogleAgent-Shopping\nDisallow: /\n\nUser-agent: ClaudeBot\nDisallow: /\n\nUser-agent: meta-externalagent\nDisallow: /\n\nUser-agent: Bytespider\nDisallow: /\n\nUser-agent: Scrapy\nDisallow: /\n\nUser-agent: PetalBot\nDisallow: /\n\nUser-agent: Devin\nDisallow: /\n\nUser-agent: omgili\nDisallow: /\n\nUser-agent: AI2Bot\nDisallow: /\n\nUser-agent: Gemini-Deep-Research\nDisallow: /\n\nUser-agent: PanguBot\nDisallow: /\n\nUser-agent: MistralAI-User\nDisallow: /\n\nUser-agent: Diffbot\nDisallow: /\n\nUser-agent: Sidetrade indexer bot\nDisallow: /\n\nUser-agent: Google-NotebookLM\nDisallow: /\n\nUser-agent: DuckAssistBot\nDisallow: /\n\nUser-agent: Copilot\nDisallow: /\n\nUser-agent: Ai2Bot-Dolma\nDisallow: /\n\nUser-agent: Google-CloudVertexBot\nDisallow: /\n\nUser-agent: meta-externalfetcher\nDisallow: /\n\nUser-agent: iaskspider\nDisallow: /\n\nUser-agent: wpbot\nDisallow: /\n\nUser-agent: aiHitBot\nDisallow: /\n\nUser-agent: cohere-ai\nDisallow: /\n\nUser-agent: FriendlyCrawler\nDisallow: /\n\nUser-agent: img2dataset\nDisallow: /\n\nUser-agent: VelenPublicWebCrawler\nDisallow: /\n\nUser-agent: YouBot\nDisallow: /\n\nUser-agent: Brightbot\nDisallow: /\n\nUser-agent: Claude-User\nDisallow: /\n\nUser-agent: ClaudeBot\nDisallow: /\n\nUser-agent: Claude-SearchBot\nDisallow: /\n\nUser-agent: Perplexity-User\nDisallow: /\n\nUser-agent: ISSCyberRiskCrawler\nDisallow: /\n\nUser-agent: CopilotNative\nDisallow: /\n\nUser-agent: CopilotSapphire\nDisallow: /\n\nUser-agent: imgproxy\nDisallow: /\n\nUser-agent: SemrushBot-SWA\nDisallow: /\n\nUser-agent: ChatGPT-User\nDisallow: /\n\nUser-agent: OAI-SearchBot\nDisallow: /\n\nUser-agent: Timpibot\nDisallow: /\n\nUser-agent: Thinkbot\nDisallow: /\n\nUser-agent: AddSearchBot\nDisallow: /\n\nUser-agent: AI2Bot-DeepResearchEval\nDisallow: /\n\nUser-agent: Andibot\nDisallow: /\n\nUser-agent: Anomura\nDisallow: /\n\nUser-agent: atlassian-bot\nDisallow: /\n\nUser-agent: BuddyBot\nDisallow: /\n\nUser-agent: Channel3Bot\nDisallow: /\n\nUser-agent: ChatGLM-Spider\nDisallow: /\n\nUser-agent: Claude-Web\nDisallow: /\n\nUser-agent: Cloudflare-AutoRAG\nDisallow: /\n\nUser-agent: CloudflareBrowserRenderingCrawler\nDisallow: /\n\nUser-agent: cohere-training-data-crawler\nDisallow: /\n\nUser-agent: Cotoyogi\nDisallow: /\n\nUser-agent: Crawl4AI\nDisallow: /\n\nUser-agent: Datenbank Crawler\nDisallow: /\n\nUser-agent: DeepSeekBot\nDisallow: /\n\nUser-agent: Echobot Bot\nDisallow: /\n\nUser-agent: EchoboxBot\nDisallow: /\n\nUser-agent: factset_spyderbot\nDisallow: /\n\nUser-agent: Google-Firebase\nDisallow: /\n\nUser-agent: GoogleOther\nDisallow: /\n\nUser-agent: Grok-DeepSearch\nDisallow: /\n\nUser-agent: GrokBot\nDisallow: /\n\nUser-agent: iAskBot\nDisallow: /\n\nUser-agent: IbouBot\nDisallow: /\n\nUser-agent: ICC-Crawler\nDisallow: /\n\nUser-agent: ImagesiftBot\nDisallow: /\n\nUser-agent: imageSpider\nDisallow: /\n\nUser-agent: Kangaroo Bot\nDisallow: /\n\nUser-agent: KlaviyoAIBot\nDisallow: /\n\nUser-agent: KunatoCrawler\nDisallow: /\n\nUser-agent: laion-huggingface-processor\nDisallow: /\n\nUser-agent: Linguee Bot\nDisallow: /\n\nUser-agent: LinkupBot\nDisallow: /\n\nUser-agent: Manus-User\nDisallow: /\n\nUser-agent: meta-webindexer\nDisallow: /\n\nUser-agent: netEstate Imprint Crawler\nDisallow: /\n\nUser-agent: Panscient\nDisallow: /\n\nUser-agent: panscient.com\nDisallow: /\n\nUser-agent: PhindBot\nDisallow: /\n\nUser-agent: Poggio-Citations\nDisallow: /\n\nUser-agent: QualifiedBot\nDisallow: /\n\nUser-agent: SBIntuitionsBot\nDisallow: /\n\nUser-agent: ShapBot\nDisallow: /\n\nUser-agent: TavilyBot\nDisallow: /\n\nUser-agent: TerraCotta\nDisallow: /\n\nUser-agent: TwinAgent\nDisallow: /\n\nUser-agent: webzio-extended\nDisallow: /\n\nUser-agent: WRTNBot\nDisallow: /\n\nUser-agent: xAI-Grok\nDisallow: /\n\nUser-agent: YaK\nDisallow: /\n\nUser-agent: ZanistaBot\nDisallow: /","agents":{"GPTBot":false,"OAI-SearchBot":false,"ChatGPT-User":false,"ClaudeBot":false,"Claude-SearchBot":false,"PerplexityBot":false,"Google-Extended":false,"Applebot-Extended":true,"CCBot":false,"Bytespider":false,"Meta-ExternalAgent":false,"Amazonbot":true}},"sitemap":{"status":500,"content_type":"text/html","bytes":2672,"sha256":"c669d528a3876410eebef4c55e504ec016707d321f090eae9f3b09b8eaedc67f","final_host":"www.amazon.com","final_path":"/sitemap.xml","ms":1951,"url":"https://amazon.com/sitemap.xml","declared":false,"xml":false},"files":{"llms_txt":{"status":404,"content_type":"text/html","bytes":2296,"sha256":"34c92fb4c6351b94e5cc16f68bb483b0fcda57ee7c43887214005b87d1888d3a","final_host":"www.amazon.com","final_path":"/llms.txt","ms":502,"path":"/llms.txt","html":true,"json":null,"present":false},"agents_md":{"status":404,"content_type":"text/html","bytes":2296,"sha256":"34c92fb4c6351b94e5cc16f68bb483b0fcda57ee7c43887214005b87d1888d3a","final_host":"www.amazon.com","final_path":"/agents.md","ms":604,"path":"/agents.md","html":true,"json":null,"present":false},"agent_skills":{"status":404,"content_type":"text/html","bytes":2296,"sha256":"34c92fb4c6351b94e5cc16f68bb483b0fcda57ee7c43887214005b87d1888d3a","final_host":"www.amazon.com","final_path":"/.well-known/agent-skills/index.json","ms":959,"path":"/.well-known/agent-skills/index.json","html":true,"json":null,"present":false},"mcp_card":{"status":404,"content_type":"text/html","bytes":2296,"sha256":"34c92fb4c6351b94e5cc16f68bb483b0fcda57ee7c43887214005b87d1888d3a","final_host":"www.amazon.com","final_path":"/.well-known/mcp/server-card.json","ms":574,"path":"/.well-known/mcp/server-card.json","html":true,"json":null,"present":false},"api_catalog":{"status":404,"content_type":"text/html","bytes":2296,"sha256":"34c92fb4c6351b94e5cc16f68bb483b0fcda57ee7c43887214005b87d1888d3a","final_host":"www.amazon.com","final_path":"/.well-known/api-catalog","ms":507,"path":"/.well-known/api-catalog","html":true,"json":null,"present":false},"a2a_card":{"status":404,"content_type":"text/html","bytes":2296,"sha256":"34c92fb4c6351b94e5cc16f68bb483b0fcda57ee7c43887214005b87d1888d3a","final_host":"www.amazon.com","final_path":"/.well-known/agent-card.json","ms":486,"path":"/.well-known/agent-card.json","html":true,"json":null,"present":false}},"rings":{"v":1,"candidates":[],"mcp":[],"a2a":null,"webmcp":{"model_context":false,"bridge":false,"mcp_json":false,"link_rel":false,"any":false},"handshakes":0,"get_path":{"status":200,"content_type":null,"ms":468,"error":null},"get_host":{"status":530,"content_type":"text/plain","ms":13,"error":null}},"sf":[4,0,0,0,0,0,1,0,0],"derived":{"reachable":false,"home_class":"other","robots_served":false,"robots_sitemap":false,"content_signal":false,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":[],"blocks_any_ai":false,"files":{"llms_txt":false,"agents_md":false,"agent_skills":false,"mcp_card":false,"api_catalog":false,"a2a_card":false},"any_capability":false,"ai_refused":[],"ai_thin":[],"sf":[4,0,0,0,0,0,1,0,0],"rings":{"mcp_declared":false,"mcp_live":false,"mcp_auth":false,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":3,"fh":"www.amazon.com","fp_other_host":[]},"derive_v":5,"frame":"top1k","shard":1},{"v":1,"kind":"crawlcheck-census-record","run":43,"domain":"gtld-servers.net","rank":27,"measured_at":"2026-11-13T07:58:58.649Z","home":{"browser":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"gtld-servers.net","final_path":"/","ms":32,"identity":"Mobile browser","html":false},"gptbot":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"gtld-servers.net","final_path":"/","ms":1,"identity":"GPTBot","html":false},"claudebot":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"gtld-servers.net","final_path":"/","ms":0,"identity":"ClaudeBot","html":false},"oaisearch":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"gtld-servers.net","final_path":"/","ms":1,"identity":"OAI-SearchBot","html":false},"perplexity":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"gtld-servers.net","final_path":"/","ms":1,"identity":"PerplexityBot","html":false}},"robots":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"gtld-servers.net","final_path":"/robots.txt","ms":1,"html":false,"text":null,"agents":null},"sitemap":null,"files":null,"derived":{"reachable":false,"home_class":"no_site","robots_served":false,"robots_sitemap":false,"content_signal":false,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":[],"blocks_any_ai":false,"files":{"llms_txt":false,"agents_md":false,"agent_skills":false,"mcp_card":false,"api_catalog":false,"a2a_card":false},"any_capability":false,"ai_refused":[],"ai_thin":[],"rings":{"mcp_declared":false,"mcp_live":false,"mcp_auth":false,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":3,"fh":"gtld-servers.net","fp_other_host":[]},"derive_v":5,"frame":"top1k","shard":2},{"v":1,"kind":"crawlcheck-census-record","run":43,"domain":"azure.com","rank":28,"measured_at":"2026-11-13T07:59:19.049Z","home":{"browser":{"status":200,"content_type":"text/html","bytes":463681,"sha256":"ab5a6f1ba20339274ab097617630fd275b24c0b1d699245fe1b9278e30080ae8","final_host":"azure.microsoft.com","final_path":"/en-us","ms":667,"identity":"Mobile browser","html":true,"fp":"da5efe5944800e3c"},"gptbot":{"status":200,"content_type":"text/html","bytes":463681,"sha256":"02ae72a36675e66db6d4a9962ecd6a2d720e5aa4bcf0fe8501cb707a0d5fdd89","final_host":"azure.microsoft.com","final_path":"/en-us","ms":4524,"identity":"GPTBot","html":true,"fp":"da5efe5944800e3c"},"claudebot":{"status":200,"content_type":"text/html","bytes":463681,"sha256":"61149a0a159440d7c616c372f14ece8b56b4bf52eeaafe2a82c053e1a9ed0284","final_host":"azure.microsoft.com","final_path":"/en-us","ms":4331,"identity":"ClaudeBot","html":true,"fp":"da5efe5944800e3c"},"oaisearch":{"status":200,"content_type":"text/html","bytes":463681,"sha256":"a5d8e5f5a5c5d8e469898e39b0ed4424be51f8aa1956e52d5a960512d4885b99","final_host":"azure.microsoft.com","final_path":"/en-us","ms":5794,"identity":"OAI-SearchBot","html":true,"fp":"da5efe5944800e3c"},"perplexity":{"status":200,"content_type":"text/html","bytes":463681,"sha256":"a65e2846a6f42acf5646165d8ff1dfe980f8f436f00cfccdc10b3c56c0426866","final_host":"azure.microsoft.com","final_path":"/en-us","ms":3946,"identity":"PerplexityBot","html":true,"fp":"da5efe5944800e3c"}},"robots":{"status":200,"content_type":"text/html","bytes":463681,"sha256":"7c30f60310c7f6c2ec8d21278e79b8815ea7510431af38bcb5e49f65e6ab480a","final_host":"azure.microsoft.com","final_path":"/en-us","ms":314,"html":true,"text":null,"agents":null},"sitemap":{"status":200,"content_type":"text/html","bytes":463681,"sha256":"77b5c61bc7d80eaf8f0d5707ab060600f14ef7ae2316ad365793c191cf7a4e7f","final_host":"azure.microsoft.com","final_path":"/en-us","ms":458,"url":"https://azure.com/sitemap.xml","declared":false,"xml":false},"files":{"llms_txt":{"status":404,"content_type":"text/html","bytes":94701,"sha256":"2481a98d677136bd8b1cf39c588e18ebff29372604ddcb1cb7220eb9cb3e01ff","final_host":"azure.microsoft.com","final_path":"/llms.txt","ms":278,"path":"/llms.txt","html":true,"json":null,"present":false},"agents_md":{"status":200,"content_type":"text/html","bytes":463681,"sha256":"3087aa7fea2bf290a3de06c7db6e160bcd8f65bd55a3510ba30878b0ad3e0258","final_host":"azure.microsoft.com","final_path":"/en-us","ms":1276,"path":"/agents.md","html":true,"json":null,"present":false},"agent_skills":{"status":200,"content_type":"text/html","bytes":463681,"sha256":"d929b4154fcad0b932c1589dca07f13202ea789d71546f1c0ac1bac961bd0af1","final_host":"azure.microsoft.com","final_path":"/en-us","ms":887,"path":"/.well-known/agent-skills/index.json","html":true,"json":null,"present":false},"mcp_card":{"status":200,"content_type":"text/html","bytes":463681,"sha256":"a9cd27368fe8947a6eee1ab7c9dfad6a33ff0dc36f33d1fa048723cf70ea1fce","final_host":"azure.microsoft.com","final_path":"/en-us","ms":427,"path":"/.well-known/mcp/server-card.json","html":true,"json":null,"present":false},"api_catalog":{"status":200,"content_type":"text/html","bytes":463681,"sha256":"22e8f3496d4a3e0d100183cdb4392266493dbdab142444fdd46c3340b3c01d9f","final_host":"azure.microsoft.com","final_path":"/en-us","ms":877,"path":"/.well-known/api-catalog","html":true,"json":null,"present":false},"a2a_card":{"status":200,"content_type":"text/html","bytes":463681,"sha256":"d4987f3698d159b1b9f91dd723e05f1106a89026836bc6bef0e3a9fceeb7c1aa","final_host":"azure.microsoft.com","final_path":"/en-us","ms":275,"path":"/.well-known/agent-card.json","html":true,"json":null,"present":false}},"rings":{"v":1,"candidates":[],"mcp":[],"a2a":null,"webmcp":{"model_context":false,"bridge":false,"mcp_json":false,"link_rel":false,"any":false},"handshakes":0,"get_path":{"status":200,"content_type":"text/html","ms":310,"error":null},"get_host":{"status":530,"content_type":"text/plain","ms":14,"error":null}},"sf":[4,0,0,0,0,0,4,0,0],"derived":{"reachable":true,"home_class":"answers","robots_served":false,"robots_sitemap":false,"content_signal":false,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":[],"blocks_any_ai":false,"files":{"llms_txt":false,"agents_md":false,"agent_skills":false,"mcp_card":false,"api_catalog":false,"a2a_card":false},"any_capability":false,"ai_refused":[],"ai_thin":[],"sf":[4,0,0,0,0,0,4,0,0],"rings":{"mcp_declared":false,"mcp_live":false,"mcp_auth":false,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":3,"fh":"azure.microsoft.com","fp_other_host":[],"fp":"da5efe5944800e3c","fp_bits":{"gptbot":0,"claudebot":0,"oaisearch":0,"perplexity":0}},"derive_v":5,"frame":"top1k","shard":3},{"v":1,"kind":"crawlcheck-census-record","run":43,"domain":"github.com","rank":29,"measured_at":"2026-11-13T08:05:49.397Z","home":{"browser":{"status":200,"content_type":"text/html","bytes":525536,"sha256":"2dfdd245519f72d3d24b4be53a9c70a091e363f6bab86c6cd89d9163ba780da8","final_host":"github.com","final_path":"/","ms":384,"identity":"Mobile browser","html":true,"fp":"99b771e96063c6b5"},"gptbot":{"status":200,"content_type":"text/html","bytes":525602,"sha256":"eb1de5e95d4f007f2323c829455828b56f538843fa1543431d6d3584765398f3","final_host":"github.com","final_path":"/","ms":123,"identity":"GPTBot","html":true,"fp":"91a751e971636aa5"},"claudebot":{"status":200,"content_type":"text/html","bytes":525293,"sha256":"907fc9a47c33a55753807b54c4b2aa6be08a46c02151f956d11290b5cba46187","final_host":"github.com","final_path":"/","ms":125,"identity":"ClaudeBot","html":true,"fp":"999751e17663ea8d"},"oaisearch":{"status":200,"content_type":"text/html","bytes":525602,"sha256":"a9b976db823a90f771bdc7186e22d1b65dcefe275e2aa59837f216caa0bdfaf2","final_host":"github.com","final_path":"/","ms":9,"identity":"OAI-SearchBot","html":true,"fp":"91a751e971636aa5"},"perplexity":{"status":200,"content_type":"text/html","bytes":525602,"sha256":"0253c0072f851704310985675051631ab3cd7ff7660e33a6a5529022b1f58a9d","final_host":"github.com","final_path":"/","ms":9,"identity":"PerplexityBot","html":true,"fp":"91a751e971636aa5"}},"robots":{"status":200,"content_type":"text/plain","bytes":29821,"sha256":"8b8e46edf4593a7e23fbf207472b2b56601bfae79eff6d9e17f990a6db39d5d3","final_host":"github.com","final_path":"/robots.txt","ms":87,"html":false,"text":"# If you would like to crawl GitHub contact us via https://support.github.com?tags=dotcom-robots\n# We also provide an extensive API: https://docs.github.com\nSitemap: https://github.com/sitemap_marketing.xml\n\nUser-agent: GPTBot\nUser-agent: OAI-SearchBot\nUser-agent: ChatGPT-User\nUser-agent: ClaudeBot\nUser-agent: Claude-SearchBot\nUser-agent: Claude-User\nUser-agent: PerplexityBot\nUser-agent: Perplexity-User\nCrawl-delay: 1\nAllow: /*?tab=achievements&achievement=*\nAllow: /marketplace/*?tab=readme$\nAllow: /marketplace/*?tab=transparency$\nAllow: /enterprise/contact?*\nAllow: /enterprise/contact/data-residency?*\nAllow: /features/codespaces?*\nAllow: /features/copilot/plans?*\nAllow: /security/advanced-security/demo?*\nAllow: /security/contact-sales?*\nAllow: /security/plans?*\nAllow: /topics/*?q=*\nAllow: /topics/*?*&q=*\nAllow: /orgs/*/repositories?q=*\nAllow: /orgs/*/repositories?*&q=*\nDisallow: /account-login\nDisallow: /copilot/\nDisallow: /copilot/c/\nDisallow: /ekansa/Open-Context-Data\nDisallow: /ekansa/opencontext-*\nDisallow: /Explodingstuff/\nDisallow: /gist/\nDisallow: /search/advanced\nDisallow: /search$\nDisallow: /*/*/pulse_committer_data.json$\nDisallow: /*/*/pulse_committer_data.json?*\nDisallow: /*/archive/\nDisallow: /*/blame/\nDisallow: /*/cache/\nDisallow: /*/raw/\nDisallow: /*/tree/\nDisallow: */tarball/\nDisallow: */zipball/\nDisallow: /.git/\nDisallow: */.git/\nDisallow: /*.git$\nDisallow: /*.atom$\nDisallow: /*report-abuse?report=*\nDisallow: /*/*/commits/\nDisallow: /marketplace/*/*?tab=readme$\nDisallow: /marketplace/*/*?tab=transparency$\nDisallow: /marketplace/*?*?tab=readme$\nDisallow: /marketplace/*?*?tab=transparency$\nDisallow: /marketplace/actions?tab=readme$\nDisallow: /marketplace/actions?tab=transparency$\nDisallow: /marketplace/free-trials?tab=readme$\nDisallow: /marketplace/free-trials?tab=transparency$\nDisallow: /marketplace/manage?tab=readme$\nDisallow: /marketplace/manage?tab=transparency$\nDisallow: /marketplace/models?tab=readme$\nDisallow: /marketplace/models?tab=transparency$\nDisallow: /marketplace/new?tab=readme$\nDisallow: /marketplace/new?tab=transparency$\nDisallow: /marketplace/publishers?tab=readme$\nDisallow: /marketplace/publishers?tab=transparency$\nDisallow: /marketplace/recently-added?tab=readme$\nDisallow: /marketplace/recently-added?tab=transparency$\nDisallow: /topics/*/*?q=*\nDisallow: /topics/*/*?*&q=*\nDisallow: /topics/*?*?q=*\nDisallow: /topics/*?*?*&q=*\nDisallow: /orgs/*/*/repositories?q=*\nDisallow: /orgs/*/*/repositories?*&q=*\nDisallow: /orgs/*?*/repositories?q=*\nDisallow: /orgs/*?*/repositories?*&q=*\nDisallow: /topics/*?source=*&q=*\nDisallow: /topics/*?*&source=*&q=*\nDisallow: /topics/*?q=*&source=*\nDisallow: /topics/*?*&q=*&source=*\nDisallow: /topics/*?ref_cta=*&q=*\nDisallow: /topics/*?*&ref_cta=*&q=*\nDisallow: /topics/*?q=*&ref_cta=*\nDisallow: /topics/*?*&q=*&ref_cta=*\nDisallow: /topics/*?plan=*&q=*\nDisallow: /topics/*?*&plan=*&q=*\nDisallow: /topics/*?q=*&plan=*\nDisallow: /topics/*?*&q=*&plan=*\nDisallow: /topics/*?return_to=*&q=*\nDisallow: /topics/*?*&return_to=*&q=*\nDisallow: /topics/*?q=*&return_to=*\nDisallow: /topics/*?*&q=*&return_to=*\nDisallow: /topics/*?ref_loc=*&q=*\nDisallow: /topics/*?*&ref_loc=*&q=*\nDisallow: /topics/*?q=*&ref_loc=*\nDisallow: /topics/*?*&q=*&ref_loc=*\nDisallow: /topics/*?setup_organization=*&q=*\nDisallow: /topics/*?*&setup_organization=*&q=*\nDisallow: /topics/*?q=*&setup_organization=*\nDisallow: /topics/*?*&q=*&setup_organization=*\nDisallow: /topics/*?source_repo=*&q=*\nDisallow: /topics/*?*&source_repo=*&q=*\nDisallow: /topics/*?q=*&source_repo=*\nDisallow: /topics/*?*&q=*&source_repo=*\nDisallow: /topics/*?ref_page=*&q=*\nDisallow: /topics/*?*&ref_page=*&q=*\nDisallow: /topics/*?q=*&ref_page=*\nDisallow: /topics/*?*&q=*&ref_page=*\nDisallow: /topics/*?referrer=*&q=*\nDisallow: /topics/*?*&referrer=*&q=*\nDisallow: /topics/*?q=*&referrer=*\nDisallow: /topics/*?*&q=*&referrer=*\nDisallow: /topics/*?report=*&q=*\nDisallow: /topics/*?*&report=*&q=*\nDisallow: /topics/*?q=*&report=*\nDisallow: /topics/*?*&q=*&report=*\nDisallow: /topics/*?author=*&q=*\nDisallow: /topics/*?*&author=*&q=*\nDisallow: /topics/*?q=*&author=*\nDisallow: /topics/*?*&q=*&author=*\nDisallow: /topics/*?since=*&q=*\nDisallow: /topics/*?*&since=*&q=*\nDisallow: /topics/*?q=*&since=*\nDisallow: /topics/*?*&q=*&since=*\nDisallow: /topics/*?until=*&q=*\nDisallow: /topics/*?*&until=*&q=*\nDisallow: /topics/*?q=*&until=*\nDisallow: /topics/*?*&q=*&until=*\nDisallow: /topics/*?tab=*&q=*\nDisallow: /topics/*?*&tab=*&q=*\nDisallow: /topics/*?q=*&tab=*\nDisallow: /topics/*?*&q=*&tab=*\nDisallow: /orgs/*/repositories?source=*&q=*\nDisallow: /orgs/*/repositories?*&source=*&q=*\nDisallow: /orgs/*/repositories?q=*&source=*\nDisallow: /orgs/*/repositories?*&q=*&source=*\nDisallow: /orgs/*/repositories?ref_cta=*&q=*\nDisallow: /orgs/*/repositories?*&ref_cta=*&q=*\nDisallow: /orgs/*/repositories?q=*&ref_cta=*\nDisallow: /orgs/*/repositories?*&q=*&ref_cta=*\nDisallow: /orgs/*/repositories?plan=*&q=*\nDisallow: /orgs/*/repositories?*&plan=*&q=*\nDisallow: /orgs/*/repositories?q=*&plan=*\nDisallow: /orgs/*/repositories?*&q=*&plan=*\nDisallow: /orgs/*/repositories?return_to=*&q=*\nDisallow: /orgs/*/repositories?*&return_to=*&q=*\nDisallow: /orgs/*/repositories?q=*&return_to=*\nDisallow: /orgs/*/repositories?*&q=*&return_to=*\nDisallow: /orgs/*/repositories?ref_loc=*&q=*\nDisallow: /orgs/*/repositories?*&ref_loc=*&q=*\nDisallow: /orgs/*/repositories?q=*&ref_loc=*\nDisallow: /orgs/*/repositories?*&q=*&ref_loc=*\nDisallow: /orgs/*/repositories?setup_organization=*&q=*\nDisallow: /orgs/*/repositories?*&setup_organization=*&q=*\nDisallow: /orgs/*/repositories?q=*&setup_organization=*\nDisallow: /orgs/*/repositories?*&q=*&setup_organization=*\nDisallow: /orgs/*/repositories?source_repo=*&q=*\nDisallow: /orgs/*/repositories?*&source_repo=*&q=*\nDisallow: /orgs/*/repositories?q=*&source_repo=*\nDisallow: /orgs/*/repositories?*&q=*&source_repo=*\nDisallow: /orgs/*/repositories?ref_page=*&q=*\nDisallow: /orgs/*/repositories?*&ref_page=*&q=*\nDisallow: /orgs/*/repositories?q=*&ref_page=*\nDisallow: /orgs/*/repositories?*&q=*&ref_page=*\nDisallow: /orgs/*/repositories?referrer=*&q=*\nDisallow: /orgs/*/repositories?*&referrer=*&q=*\nDisallow: /orgs/*/repositories?q=*&referrer=*\nDisallow: /orgs/*/repositories?*&q=*&referrer=*\nDisallow: /orgs/*/repositories?report=*&q=*\nDisallow: /orgs/*/repositories?*&report=*&q=*\nDisallow: /orgs/*/repositories?q=*&report=*\nDisallow: /orgs/*/repositories?*&q=*&report=*\nDisallow: /orgs/*/repositories?author=*&q=*\nDisallow: /orgs/*/repositories?*&author=*&q=*\nDisallow: /orgs/*/repositories?q=*&author=*\nDisallow: /orgs/*/repositories?*&q=*&author=*\nDisallow: /orgs/*/repositories?since=*&q=*\nDisallow: /orgs/*/repositories?*&since=*&q=*\nDisallow: /orgs/*/repositories?q=*&since=*\nDisallow: /orgs/*/repositories?*&q=*&since=*\nDisallow: /orgs/*/repositories?until=*&q=*\nDisallow: /orgs/*/repositories?*&until=*&q=*\nDisallow: /orgs/*/repositories?q=*&until=*\nDisallow: /orgs/*/repositories?*&q=*&until=*\nDisallow: /orgs/*/repositories?tab=*&q=*\nDisallow: /orgs/*/repositories?*&tab=*&q=*\nDisallow: /orgs/*/repositories?q=*&tab=*\nDisallow: /orgs/*/repositories?*&q=*&tab=*\nDisallow: /biztools$\nDisallow: /biztools?*\nDisallow: /biztools/*\nDisallow: /businesses$\nDisallow: /businesses?*\nDisallow: /businesses/*\nDisallow: /enterprises$\nDisallow: /enterprises?*\nDisallow: /enterprises/*\nDisallow: /organizations$\nDisallow: /organizations?*\nDisallow: /organizations/*\nDisallow: /settings$\nDisallow: /settings?*\nDisallow: /settings/*\nDisallow: /stafftools$\nDisallow: /stafftools?*\nDisallow: /stafftools/*\nDisallow: /users/*/billing$\nDisallow: /users/*/billing?*\nDisallow: /users/*/billing/*\nDisallow: /writing$\nDisallow: /writing?*\nDisallow: /writing/*\nDisallow: /*/*/branches$\nDisallow: /*/*/branches?*\nDisallow: /*/*/branches/*\nDisallow: /*/*/comments$\nDisallow: /*/*/comments?*\nDisallow: /*/*/comments/*\nDisallow: /*/*/compare$\nDisallow: /*/*/compare?*\nDisallow: /*/*/compare/*\nDisallow: /*/*/contributors$\nDisallow: /*/*/contributors?*\nDisallow: /*/*/contributors/*\nDisallow: /*/*/contributors_list$\nDisallow: /*/*/contributors_list?*\nDisallow: /*/*/contributors_list/*\nDisallow: /*/*/contributors-list$\nDisallow: /*/*/contributors-list?*\nDisallow: /*/*/contributors-list/*\nDisallow: /*/*/download$\nDisallow: /*/*/download?*\nDisallow: /*/*/download/*\nDisallow: /*/*/forks$\nDisallow: /*/*/forks?*\nDisallow: /*/*/forks/*\nDisallow: /*/*/graphs$\nDisallow: /*/*/graphs?*\nDisallow: /*/*/graphs/*\nDisallow: /*/*/issues/new$\nDisallow: /*/*/issues/new?*\nDisallow: /*/*/issues/new/*\nDisallow: /*/*/issues/search$\nDisallow: /*/*/issues/search?*\nDisallow: /*/*/issues/search/*\nDisallow: /*/*/milestones/new$\nDisallow: /*/*/milestones/new?*\nDisallow: /*/*/milestones/new/*\nDisallow: /*/*/network$\nDisallow: /*/*/network?*\nDisallow: /*/*/network/*\nDisallow: /*/*/projects$\nDisallow: /*/*/projects?*\nDisallow: /*/*/projects/*\nDisallow: /*/*/pulse$\nDisallow: /*/*/pulse?*\nDisallow: /*/*/pulse/*\nDisallow: /*/*/pulse-new$\nDisallow: /*/*/pulse-new?*\nDisallow: /*/*/pulse-new/*\nDisallow: /*/*/pulse_committer_data$\nDisallow: /*/*/pulse_committer_data?*\nDisallow: /*/*/pulse_committer_data/*\nDisallow: /*/*/pulse_diffstat_summary$\nDisallow: /*/*/pulse_diffstat_summary?*\nDisallow: /*/*/pulse_diffstat_summary/*\nDisallow: /*/*/revisions$\nDisallow: /*/*/revisions?*\nDisallow: /*/*/revisions/*\nDisallow: /*/*/stargazers$\nDisallow: /*/*/stargazers?*\nDisallow: /*/*/stargazers/*\nDisallow: /*/*/tags$\nDisallow: /*/*/tags?*\nDisallow: /*/*/tags/*\nDisallow: /*/*/watchers$\nDisallow: /*/*/watchers?*\nDisallow: /*/*/watchers/*\nDisallow: /*?source=*\nDisallow: /*&source=*\nDisallow: /*?ref_cta=*\nDisallow: /*&ref_cta=*\nDisallow: /*?plan=*\nDisallow: /*&plan=*\nDisallow: /*?return_to=*\nDisallow: /*&return_to=*\nDisallow: /*?ref_loc=*\nDisallow: /*&ref_loc=*\nDisallow: /*?setup_organization=*\nDisallow: /*&setup_organization=*\nDisallow: /*?source_repo=*\nDisallow: /*&source_repo=*\nDisallow: /*?ref_page=*\nDisallow: /*&ref_page=*\nDisallow: /*?referrer=*\nDisallow: /*&referrer=*\nDisallow: /*?report=*\nDisallow: /*&report=*\nDisallow: /*?author=*\nDisallow: /*&author=*\nDisallow: /*?since=*\nDisallow: /*&since=*\nDisallow: /*?until=*\nDisallow: /*&until=*\nDisallow: /*?tab=*\nDisallow: /*&tab=*\nDisallow: /*?q=*\nDisallow: /*&q=*\n\nUser-agent: Bytespider\nDisallow: /\n\nUser-agent: AdsBot-Google\nUser-agent: AdsBot-Google-Mobile\nDisallow: /account-login\nDisallow: /copilot/\nDisallow: /copilot/c/\nDisallow: /ekansa/Open-Context-Data\nDisallow: /ekansa/opencontext-*\nDisallow: /Explodingstuff/\nDisallow: /gist/\nDisallow: /search/advanced\nDisallow: /search$\nDisallow: /*/*/pulse_committer_data.json$\nDisallow: /*/*/pulse_committer_data.json?*\nDisallow: /*/archive/\nDisallow: /*/blame/\nDisallow: /*/cache/\nDisallow: /*/raw/\nDisallow: /*/tree/\nDisallow: */tarball/\nDisallow: */zipball/\nDisallow: /.git/\nDisallow: */.git/\nDisallow: /*.git$\nDisallow: /*.atom$\nDisallow: /*report-abuse?report=*\nDisallow: /*/*/commits/\nDisallow: /marketplace/*/*?tab=readme$\nDisallow: /marketplace/*/*?tab=transparency$\nDisallow: /marketplace/*?*?tab=readme$\nDisallow: /marketplace/*?*?tab=transparency$\nDisallow: /marketplace/actions?tab=readme$\nDisallow: /marketplace/actions?tab=transparency$\nDisallow: /marketplace/free-trials?tab=readme$\nDisallow: /marketplace/free-trials?tab=transparency$\nDisallow: /marketplace/manage?tab=readme$\nDisallow: /marketplace/manage?tab=transparency$\nDisallow: /marketplace/models?tab=readme$\nDisallow: /marketplace/models?tab=transparency$\nDisallow: /marketplace/new?tab=readme$\nDisallow: /marketplace/new?tab=transparency$\nDisallow: /marketplace/publishers?tab=readme$\nDisallow: /marketplace/publishers?tab=transparency$\nDisallow: /marketplace/recently-added?tab=readme$\nDisallow: /marketplace/recently-added?tab=transparency$\nDisallow: /topics/*/*?q=*\nDisallow: /topics/*/*?*&q=*\nDisallow: /topics/*?*?q=*\nDisallow: /topics/*?*?*&q=*\nDisallow: /orgs/*/*/repositories?q=*\nDisallow: /orgs/*/*/repositories?*&q=*\nDisallow: /orgs/*?*/repositories?q=*\nDisallow: /orgs/*?*/repositories?*&q=*\nDisallow: /topics/*?source=*&q=*\nDisallow: /topics/*?*&source=*&q=*\nDisallow: /topics/*?q=*&source=*\nDisallow: /topics/*?*&q=*&source=*\nDisallow: /topics/*?ref_cta=*&q=*\nDisallow: /topics/*?*&ref_cta=*&q=*\nDisallow: /topics/*?q=*&ref_cta=*\nDisallow: /topics/*?*&q=*&ref_cta=*\nDisallow: /topics/*?plan=*&q=*\nDisallow: /topics/*?*&plan=*&q=*\nDisallow: /topics/*?q=*&plan=*\nDisallow: /topics/*?*&q=*&plan=*\nDisallow: /topics/*?return_to=*&q=*\nDisallow: /topics/*?*&return_to=*&q=*\nDisallow: /topics/*?q=*&return_to=*\nDisallow: /topics/*?*&q=*&return_to=*\nDisallow: /topics/*?ref_loc=*&q=*\nDisallow: /topics/*?*&ref_loc=*&q=*\nDisallow: /topics/*?q=*&ref_loc=*\nDisallow: /topics/*?*&q=*&ref_loc=*\nDisallow: /topics/*?setup_organization=*&q=*\nDisallow: /topics/*?*&setup_organization=*&q=*\nDisallow: /topics/*?q=*&setup_organization=*\nDisallow: /topics/*?*&q=*&setup_organization=*\nDisallow: /topics/*?source_repo=*&q=*\nDisallow: /topics/*?*&source_repo=*&q=*\nDisallow: /topics/*?q=*&source_repo=*\nDisallow: /topics/*?*&q=*&source_repo=*\nDisallow: /topics/*?ref_page=*&q=*\nDisallow: /topics/*?*&ref_page=*&q=*\nDisallow: /topics/*?q=*&ref_page=*\nDisallow: /topics/*?*&q=*&ref_page=*\nDisallow: /topics/*?referrer=*&q=*\nDisallow: /topics/*?*&referrer=*&q=*\nDisallow: /topics/*?q=*&referrer=*\nDisallow: /topics/*?*&q=*&referrer=*\nDisallow: /topics/*?report=*&q=*\nDisallow: /topics/*?*&report=*&q=*\nDisallow: /topics/*?q=*&report=*\nDisallow: /topics/*?*&q=*&report=*\nDisallow: /topics/*?author=*&q=*\nDisallow: /topics/*?*&author=*&q=*\nDisallow: /topics/*?q=*&author=*\nDisallow: /topics/*?*&q=*&author=*\nDisallow: /topics/*?since=*&q=*\nDisallow: /topics/*?*&since=*&q=*\nDisallow: /topics/*?q=*&since=*\nDisallow: /topics/*?*&q=*&since=*\nDisallow: /topics/*?until=*&q=*\nDisallow: /topics/*?*&until=*&q=*\nDisallow: /topics/*?q=*&until=*\nDisallow: /topics/*?*&q=*&until=*\nDisallow: /topics/*?tab=*&q=*\nDisallow: /topics/*?*&tab=*&q=*\nDisallow: /topics/*?q=*&tab=*\nDisallow: /topics/*?*&q=*&tab=*\nDisallow: /orgs/*/repositories?source=*&q=*\nDisallow: /orgs/*/repositories?*&source=*&q=*\nDisallow: /orgs/*/repositories?q=*&source=*\nDisallow: /orgs/*/repositories?*&q=*&source=*\nDisallow: /orgs/*/repositories?ref_cta=*&q=*\nDisallow: /orgs/*/repositories?*&ref_cta=*&q=*\nDisallow: /orgs/*/repositories?q=*&ref_cta=*\nDisallow: /orgs/*/repositories?*&q=*&ref_cta=*\nDisallow: /orgs/*/repositories?plan=*&q=*\nDisallow: /orgs/*/repositories?*&plan=*&q=*\nDisallow: /orgs/*/repositories?q=*&plan=*\nDisallow: /orgs/*/repositories?*&q=*&plan=*\nDisallow: /orgs/*/repositories?return_to=*&q=*\nDisallow: /orgs/*/repositories?*&return_to=*&q=*\nDisallow: /orgs/*/repositories?q=*&return_to=*\nDisallow: /orgs/*/repositories?*&q=*&return_to=*\nDisallow: /orgs/*/repositories?ref_loc=*&q=*\nDisallow: /orgs/*/repositories?*&ref_loc=*&q=*\nDisallow: /orgs/*/repositories?q=*&ref_loc=*\nDisallow: /orgs/*/repositories?*&q=*&ref_loc=*\nDisallow: /orgs/*/repositories?setup_organization=*&q=*\nDisallow: /orgs/*/repositories?*&setup_organization=*&q=*\nDisallow: /orgs/*/repositories?q=*&setup_organization=*\nDisallow: /orgs/*/repositories?*&q=*&setup_organization=*\nDisallow: /orgs/*/repositories?source_repo=*&q=*\nDisallow: /orgs/*/repositories?*&source_repo=*&q=*\nDisallow: /orgs/*/repositories?q=*&source_repo=*\nDisallow: /orgs/*/repositories?*&q=*&source_repo=*\nDisallow: /orgs/*/repositories?ref_page=*&q=*\nDisallow: /orgs/*/repositories?*&ref_page=*&q=*\nDisallow: /orgs/*/repositories?q=*&ref_page=*\nDisallow: /orgs/*/repositories?*&q=*&ref_page=*\nDisallow: /orgs/*/repositories?referrer=*&q=*\nDisallow: /orgs/*/repositories?*&referrer=*&q=*\nDisallow: /orgs/*/repositories?q=*&referrer=*\nDisallow: /orgs/*/repositories?*&q=*&referrer=*\nDisallow: /orgs/*/repositories?report=*&q=*\nDisallow: /orgs/*/repositories?*&report=*&q=*\nDisallow: /orgs/*/repositories?q=*&report=*\nDisallow: /orgs/*/repositories?*&q=*&report=*\nDisallow: /orgs/*/repositories?author=*&q=*\nDisallow: /orgs/*/repositories?*&author=*&q=*\nDisallow: /orgs/*/repositories?q=*&author=*\nDisallow: /orgs/*/repositories?*&q=*&author=*\nDisallow: /orgs/*/repositories?since=*&q=*\nDisallow: /orgs/*/repositories?*&since=*&q=*\nDisallow: /orgs/*/repositories?q=*&since=*\nDisallow: /orgs/*/repositories?*&q=*&since=*\nDisallow: /orgs/*/repositories?until=*&q=*\nDisallow: /orgs/*/repositories?*&until=*&q=*\nDisallow: /orgs/*/repositories?q=*&until=*\nDisallow: /orgs/*/repositories?*&q=*&until=*\nDisallow: /orgs/*/repositories?tab=*&q=*\nDisallow: /orgs/*/repositories?*&tab=*&q=*\nDisallow: /orgs/*/repositories?q=*&tab=*\nDisallow: /orgs/*/repositories?*&q=*&tab=*\nDisallow: /biztools$\nDisallow: /biztools?*\nDisallow: /biztools/*\nDisallow: /businesses$\nDisallow: /businesses?*\nDisallow: /businesses/*\nDisallow: /enterprises$\nDisallow: /enterprises?*\nDisallow: /enterprises/*\nDisallow: /organizations$\nDisallow: /organizations?*\nDisallow: /organizations/*\nDisallow: /settings$\nDisallow: /settings?*\nDisallow: /settings/*\nDisallow: /stafftools$\nDisallow: /stafftools?*\nDisallow: /stafftools/*\nDisallow: /users/*/billing$\nDisallow: /users/*/billing?*\nDisallow: /users/*/billing/*\nDisallow: /writing$\nDisallow: /writing?*\nDisallow: /writing/*\nDisallow: /*/*/branches$\nDisallow: /*/*/branches?*\nDisallow: /*/*/branches/*\nDisallow: /*/*/comments$\nDisallow: /*/*/comments?*\nDisallow: /*/*/comments/*\nDisallow: /*/*/compare$\nDisallow: /*/*/compare?*\nDisallow: /*/*/compare/*\nDisallow: /*/*/contributors$\nDisallow: /*/*/contributors?*\nDisallow: /*/*/contributors/*\nDisallow: /*/*/contributors_list$\nDisallow: /*/*/contributors_list?*\nDisallow: /*/*/contributors_list/*\nDisallow: /*/*/contributors-list$\nDisallow: /*/*/contributors-list?*\nDisallow: /*/*/contributors-list/*\nDisallow: /*/*/download$\nDisallow: /*/*/download?*\nDisallow: /*/*/download/*\nDisallow: /*/*/forks$\nDisallow: /*/*/forks?*\nDisallow: /*/*/forks/*\nDisallow: /*/*/graphs$\nDisallow: /*/*/graphs?*\nDisallow: /*/*/graphs/*\nDisallow: /*/*/issues/new$\nDisallow: /*/*/issues/new?*\nDisallow: /*/*/issues/new/*\nDisallow: /*/*/issues/search$\nDisallow: /*/*/issues/search?*\nDisallow: /*/*/issues/search/*\nDisallow: /*/*/milestones/new$\nDisallow: /*/*/milestones/new?*\nDisallow: /*/*/milestones/new/*\nDisallow: /*/*/network$\nDisallow: /*/*/network?*\nDisallow: /*/*/network/*\nDisallow: /*/*/projects$\nDisallow: /*/*/projects?*\nDisallow: /*/*/projects/*\nDisallow: /*/*/pulse$\nDisallow: /*/*/pulse?*\nDisallow: /*/*/pulse/*\nDisallow: /*/*/pulse-new$\nDisallow: /*/*/pulse-new?*\nDisallow: /*/*/pulse-new/*\nDisallow: /*/*/pulse_committer_data$\nDisallow: /*/*/pulse_committer_data?*\nDisallow: /*/*/pulse_committer_data/*\nDisallow: /*/*/pulse_diffstat_summary$\nDisallow: /*/*/pulse_diffstat_summary?*\nDisallow: /*/*/pulse_diffstat_summary/*\nDisallow: /*/*/revisions$\nDisallow: /*/*/revisions?*\nDisallow: /*/*/revisions/*\nDisallow: /*/*/stargazers$\nDisallow: /*/*/stargazers?*\nDisallow: /*/*/stargazers/*\nDisallow: /*/*/tags$\nDisallow: /*/*/tags?*\nDisallow: /*/*/tags/*\nDisallow: /*/*/watchers$\nDisallow: /*/*/watchers?*\nDisallow: /*/*/watchers/*\nDisallow: /*?source=*\nDisallow: /*&source=*\nDisallow: /*?ref_cta=*\nDisallow: /*&ref_cta=*\nDisallow: /*?plan=*\nDisallow: /*&plan=*\nDisallow: /*?return_to=*\nDisallow: /*&return_to=*\nDisallow: /*?ref_loc=*\nDisallow: /*&ref_loc=*\nDisallow: /*?setup_organization=*\nDisallow: /*&setup_organization=*\nDisallow: /*?source_repo=*\nDisallow: /*&source_repo=*\nDisallow: /*?ref_page=*\nDisallow: /*&ref_page=*\nDisallow: /*?referrer=*\nDisallow: /*&referrer=*\nDisallow: /*?report=*\nDisallow: /*&report=*\nDisallow: /*?author=*\nDisallow: /*&author=*\nDisallow: /*?since=*\nDisallow: /*&since=*\nDisallow: /*?until=*\nDisallow: /*&until=*\nDisallow: /*?tab=*\nDisallow: /*&tab=*\nDisallow: /*?q=*\nDisallow: /*&q=*\n\nUser-agent: bingbot\nUser-agent: adidxbot\nUser-agent: BingPreview\nDisallow: /ekansa/Open-Context-Data\nDisallow: /ekansa/opencontext-*\nDisallow: /account-login\nDisallow: */tarball/\nDisallow: */zipball/\nDisallow: /Explodingstuff/\nDisallow: /copilot/\nDisallow: /copilot/c/\n\nUser-agent: baidu\ncrawl-delay: 1\n\nUser-agent: *\nAllow: /*?tab=achievements&achievement=*\nAllow: /marketplace/*?tab=readme$\nAllow: /marketplace/*?tab=transparency$\nAllow: /enterprise/contact?*\nAllow: /enterprise/contact/data-residency?*\nAllow: /features/codespaces?*\nAllow: /features/copilot/plans?*\nAllow: /security/advanced-security/demo?*\nAllow: /security/contact-sales?*\nAllow: /security/plans?*\nAllow: /topics/*?q=*\nAllow: /topics/*?*&q=*\nAllow: /orgs/*/repositories?q=*\nAllow: /orgs/*/repositories?*&q=*\nDisallow: /account-login\nDisallow: /copilot/\nDisallow: /copilot/c/\nDisallow: /ekansa/Open-Context-Data\nDisallow: /ekansa/opencontext-*\nDisallow: /Explodingstuff/\nDisallow: /gist/\nDisallow: /search/advanced\nDisallow: /search$\nDisallow: /*/*/pulse_committer_data.json$\nDisallow: /*/*/pulse_committer_data.json?*\nDisallow: /*/archive/\nDisallow: /*/blame/\nDisallow: /*/cache/\nDisallow: /*/raw/\nDisallow: /*/tree/\nDisallow: */tarball/\nDisallow: */zipball/\nDisallow: /.git/\nDisallow: */.git/\nDisallow: /*.git$\nDisallow: /*.atom$\nDisallow: /*report-abuse?report=*\nDisallow: /*/*/commits/\nDisallow: /marketplace/*/*?tab=readme$\nDisallow: /marketplace/*/*?tab=transparency$\nDisallow: /marketplace/*?*?tab=readme$\nDisallow: /marketplace/*?*?tab=transparency$\nDisallow: /marketplace/actions?tab=readme$\nDisallow: /marketplace/actions?tab=transparency$\nDisallow: /marketplace/free-trials?tab=readme$\nDisallow: /marketplace/free-trials?tab=transparency$\nDisallow: /marketplace/manage?tab=readme$\nDisallow: /marketplace/manage?tab=transparency$\nDisallow: /marketplace/models?tab=readme$\nDisallow: /marketplace/models?tab=transparency$\nDisallow: /marketplace/new?tab=readme$\nDisallow: /marketplace/new?tab=transparency$\nDisallow: /marketplace/publishers?tab=readme$\nDisallow: /marketplace/publishers?tab=transparency$\nDisallow: /marketplace/recently-added?tab=readme$\nDisallow: /marketplace/recently-added?tab=transparency$\nDisallow: /topics/*/*?q=*\nDisallow: /topics/*/*?*&q=*\nDisallow: /topics/*?*?q=*\nDisallow: /topics/*?*?*&q=*\nDisallow: /orgs/*/*/repositories?q=*\nDisallow: /orgs/*/*/repositories?*&q=*\nDisallow: /orgs/*?*/repositories?q=*\nDisallow: /orgs/*?*/repositories?*&q=*\nDisallow: /topics/*?source=*&q=*\nDisallow: /topics/*?*&source=*&q=*\nDisallow: /topics/*?q=*&source=*\nDisallow: /topics/*?*&q=*&source=*\nDisallow: /topics/*?ref_cta=*&q=*\nDisallow: /topics/*?*&ref_cta=*&q=*\nDisallow: /topics/*?q=*&ref_cta=*\nDisallow: /topics/*?*&q=*&ref_cta=*\nDisallow: /topics/*?plan=*&q=*\nDisallow: /topics/*?*&plan=*&q=*\nDisallow: /topics/*?q=*&plan=*\nDisallow: /topics/*?*&q=*&plan=*\nDisallow: /topics/*?return_to=*&q=*\nDisallow: /topics/*?*&return_to=*&q=*\nDisallow: /topics/*?q=*&return_to=*\nDisallow: /topics/*?*&q=*&return_to=*\nDisallow: /topics/*?ref_loc=*&q=*\nDisallow: /topics/*?*&ref_loc=*&q=*\nDisallow: /topics/*?q=*&ref_loc=*\nDisallow: /topics/*?*&q=*&ref_loc=*\nDisallow: /topics/*?setup_organization=*&q=*\nDisallow: /topics/*?*&setup_organization=*&q=*\nDisallow: /topics/*?q=*&setup_organization=*\nDisallow: /topics/*?*&q=*&setup_organization=*\nDisallow: /topics/*?source_repo=*&q=*\nDisallow: /topics/*?*&source_repo=*&q=*\nDisallow: /topics/*?q=*&source_repo=*\nDisallow: /topics/*?*&q=*&source_repo=*\nDisallow: /topics/*?ref_page=*&q=*\nDisallow: /topics/*?*&ref_page=*&q=*\nDisallow: /topics/*?q=*&ref_page=*\nDisallow: /topics/*?*&q=*&ref_page=*\nDisallow: /topics/*?referrer=*&q=*\nDisallow: /topics/*?*&referrer=*&q=*\nDisallow: /topics/*?q=*&referrer=*\nDisallow: /topics/*?*&q=*&referrer=*\nDisallow: /topics/*?report=*&q=*\nDisallow: /topics/*?*&report=*&q=*\nDisallow: /topics/*?q=*&report=*\nDisallow: /topics/*?*&q=*&report=*\nDisallow: /topics/*?author=*&q=*\nDisallow: /topics/*?*&author=*&q=*\nDisallow: /topics/*?q=*&author=*\nDisallow: /topics/*?*&q=*&author=*\nDisallow: /topics/*?since=*&q=*\nDisallow: /topics/*?*&since=*&q=*\nDisallow: /topics/*?q=*&since=*\nDisallow: /topics/*?*&q=*&since=*\nDisallow: /topics/*?until=*&q=*\nDisallow: /topics/*?*&until=*&q=*\nDisallow: /topics/*?q=*&until=*\nDisallow: /topics/*?*&q=*&until=*\nDisallow: /topics/*?tab=*&q=*\nDisallow: /topics/*?*&tab=*&q=*\nDisallow: /topics/*?q=*&tab=*\nDisallow: /topics/*?*&q=*&tab=*\nDisallow: /orgs/*/repositories?source=*&q=*\nDisallow: /orgs/*/repositories?*&source=*&q=*\nDisallow: /orgs/*/repositories?q=*&source=*\nDisallow: /orgs/*/repositories?*&q=*&source=*\nDisallow: /orgs/*/repositories?ref_cta=*&q=*\nDisallow: /orgs/*/repositories?*&ref_cta=*&q=*\nDisallow: /orgs/*/repositories?q=*&ref_cta=*\nDisallow: /orgs/*/repositories?*&q=*&ref_cta=*\nDisallow: /orgs/*/repositories?plan=*&q=*\nDisallow: /orgs/*/repositories?*&plan=*&q=*\nDisallow: /orgs/*/repositories?q=*&plan=*\nDisallow: /orgs/*/repositories?*&q=*&plan=*\nDisallow: /orgs/*/repositories?return_to=*&q=*\nDisallow: /orgs/*/repositories?*&return_to=*&q=*\nDisallow: /orgs/*/repositories?q=*&return_to=*\nDisallow: /orgs/*/repositories?*&q=*&return_to=*\nDisallow: /orgs/*/repositories?ref_loc=*&q=*\nDisallow: /orgs/*/repositories?*&ref_loc=*&q=*\nDisallow: /orgs/*/repositories?q=*&ref_loc=*\nDisallow: /orgs/*/repositories?*&q=*&ref_loc=*\nDisallow: /orgs/*/repositories?setup_organization=*&q=*\nDisallow: /orgs/*/repositories?*&setup_organization=*&q=*\nDisallow: /orgs/*/repositories?q=*&setup_organization=*\nDisallow: /orgs/*/repositories?*&q=*&setup_organization=*\nDisallow: /orgs/*/repositories?source_repo=*&q=*\nDisallow: /orgs/*/repositories?*&source_repo=*&q=*\nDisallow: /orgs/*/repositories?q=*&source_repo=*\nDisallow: /orgs/*/repositories?*&q=*&source_repo=*\nDisallow: /orgs/*/repositories?ref_page=*&q=*\nDisallow: /orgs/*/repositories?*&ref_page=*&q=*\nDisallow: /orgs/*/repositories?q=*&ref_page=*\nDisallow: /orgs/*/repositories?*&q=*&ref_page=*\nDisallow: /orgs/*/repositories?referrer=*&q=*\nDisallow: /orgs/*/repositories?*&referrer=*&q=*\nDisallow: /orgs/*/repositories?q=*&referrer=*\nDisallow: /orgs/*/repositories?*&q=*&referrer=*\nDisallow: /orgs/*/repositories?report=*&q=*\nDisallow: /orgs/*/repositories?*&report=*&q=*\nDisallow: /orgs/*/repositories?q=*&report=*\nDisallow: /orgs/*/repositories?*&q=*&report=*\nDisallow: /orgs/*/repositories?author=*&q=*\nDisallow: /orgs/*/repositories?*&author=*&q=*\nDisallow: /orgs/*/repositories?q=*&author=*\nDisallow: /orgs/*/repositories?*&q=*&author=*\nDisallow: /orgs/*/repositories?since=*&q=*\nDisallow: /orgs/*/repositories?*&since=*&q=*\nDisallow: /orgs/*/repositories?q=*&since=*\nDisallow: /orgs/*/repositories?*&q=*&since=*\nDisallow: /orgs/*/repositories?until=*&q=*\nDisallow: /orgs/*/repositories?*&until=*&q=*\nDisallow: /orgs/*/repositories?q=*&until=*\nDisallow: /orgs/*/repositories?*&q=*&until=*\nDisallow: /orgs/*/repositories?tab=*&q=*\nDisallow: /orgs/*/repositories?*&tab=*&q=*\nDisallow: /orgs/*/repositories?q=*&tab=*\nDisallow: /orgs/*/repositories?*&q=*&tab=*\nDisallow: /biztools$\nDisallow: /biztools?*\nDisallow: /biztools/*\nDisallow: /businesses$\nDisallow: /businesses?*\nDisallow: /businesses/*\nDisallow: /enterprises$\nDisallow: /enterprises?*\nDisallow: /enterprises/*\nDisallow: /organizations$\nDisallow: /organizations?*\nDisallow: /organizations/*\nDisallow: /settings$\nDisallow: /settings?*\nDisallow: /settings/*\nDisallow: /stafftools$\nDisallow: /stafftools?*\nDisallow: /stafftools/*\nDisallow: /users/*/billing$\nDisallow: /users/*/billing?*\nDisallow: /users/*/billing/*\nDisallow: /writing$\nDisallow: /writing?*\nDisallow: /writing/*\nDisallow: /*/*/branches$\nDisallow: /*/*/branches?*\nDisallow: /*/*/branches/*\nDisallow: /*/*/comments$\nDisallow: /*/*/comments?*\nDisallow: /*/*/comments/*\nDisallow: /*/*/compare$\nDisallow: /*/*/compare?*\nDisallow: /*/*/compare/*\nDisallow: /*/*/contributors$\nDisallow: /*/*/contributors?*\nDisallow: /*/*/contributors/*\nDisallow: /*/*/contributors_list$\nDisallow: /*/*/contributors_list?*\nDisallow: /*/*/contributors_list/*\nDisallow: /*/*/contributors-list$\nDisallow: /*/*/contributors-list?*\nDisallow: /*/*/contributors-list/*\nDisallow: /*/*/download$\nDisallow: /*/*/download?*\nDisallow: /*/*/download/*\nDisallow: /*/*/forks$\nDisallow: /*/*/forks?*\nDisallow: /*/*/forks/*\nDisallow: /*/*/graphs$\nDisallow: /*/*/graphs?*\nDisallow: /*/*/graphs/*\nDisallow: /*/*/issues/new$\nDisallow: /*/*/issues/new?*\nDisallow: /*/*/issues/new/*\nDisallow: /*/*/issues/search$\nDisallow: /*/*/issues/search?*\nDisallow: /*/*/issues/search/*\nDisallow: /*/*/milestones/new$\nDisallow: /*/*/milestones/new?*\nDisallow: /*/*/milestones/new/*\nDisallow: /*/*/network$\nDisallow: /*/*/network?*\nDisallow: /*/*/network/*\nDisallow: /*/*/projects$\nDisallow: /*/*/projects?*\nDisallow: /*/*/projects/*\nDisallow: /*/*/pulse$\nDisallow: /*/*/pulse?*\nDisallow: /*/*/pulse/*\nDisallow: /*/*/pulse-new$\nDisallow: /*/*/pulse-new?*\nDisallow: /*/*/pulse-new/*\nDisallow: /*/*/pulse_committer_data$\nDisallow: /*/*/pulse_committer_data?*\nDisallow: /*/*/pulse_committer_data/*\nDisallow: /*/*/pulse_diffstat_summary$\nDisallow: /*/*/pulse_diffstat_summary?*\nDisallow: /*/*/pulse_diffstat_summary/*\nDisallow: /*/*/revisions$\nDisallow: /*/*/revisions?*\nDisallow: /*/*/revisions/*\nDisallow: /*/*/stargazers$\nDisallow: /*/*/stargazers?*\nDisallow: /*/*/stargazers/*\nDisallow: /*/*/tags$\nDisallow: /*/*/tags?*\nDisallow: /*/*/tags/*\nDisallow: /*/*/watchers$\nDisallow: /*/*/watchers?*\nDisallow: /*/*/watchers/*\nDisallow: /*?source=*\nDisallow: /*&source=*\nDisallow: /*?ref_cta=*\nDisallow: /*&ref_cta=*\nDisallow: /*?plan=*\nDisallow: /*&plan=*\nDisallow: /*?return_to=*\nDisallow: /*&return_to=*\nDisallow: /*?ref_loc=*\nDisallow: /*&ref_loc=*\nDisallow: /*?setup_organization=*\nDisallow: /*&setup_organization=*\nDisallow: /*?source_repo=*\nDisallow: /*&source_repo=*\nDisallow: /*?ref_page=*\nDisallow: /*&ref_page=*\nDisallow: /*?referrer=*\nDisallow: /*&referrer=*\nDisallow: /*?report=*\nDisallow: /*&report=*\nDisallow: /*?author=*\nDisallow: /*&author=*\nDisallow: /*?since=*\nDisallow: /*&since=*\nDisallow: /*?until=*\nDisallow: /*&until=*\nDisallow: /*?tab=*\nDisallow: /*&tab=*\nDisallow: /*?q=*\nDisallow: /*&q=*\n","agents":{"GPTBot":true,"OAI-SearchBot":true,"ChatGPT-User":true,"ClaudeBot":true,"Claude-SearchBot":true,"PerplexityBot":true,"Google-Extended":true,"Applebot-Extended":true,"CCBot":true,"Bytespider":false,"Meta-ExternalAgent":true,"Amazonbot":true}},"sitemap":{"status":200,"content_type":"application/xml","bytes":32446,"sha256":"ebdf1e30684d4ee14dc69515bef0381e652011e0c782f5e8caf5882306db10cd","final_host":"github.com","final_path":"/sitemap_marketing.xml","ms":322,"url":"https://github.com/sitemap_marketing.xml","declared":true,"xml":true},"files":{"llms_txt":{"status":200,"content_type":"text/plain","bytes":28658,"sha256":"71d217ec3186152052ca1050932c0e6e48e766d3bc5a1359b50aa140436d297f","final_host":"github.com","final_path":"/llms.txt","ms":87,"path":"/llms.txt","html":false,"json":null,"present":true},"agents_md":{"status":406,"content_type":"text/html","bytes":0,"sha256":"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855","final_host":"github.com","final_path":"/agents.md","ms":122,"path":"/agents.md","html":true,"json":null,"present":false},"agent_skills":{"status":404,"content_type":"application/json","bytes":21,"sha256":"e28bf7d9fcc22cdfde0c7c8f31e3648a4847e7bda9cb69f309f24257eee3dd41","final_host":"github.com","final_path":"/.well-known/agent-skills/index.json","ms":94,"path":"/.well-known/agent-skills/index.json","html":false,"json":null,"present":false},"mcp_card":{"status":404,"content_type":"application/json","bytes":21,"sha256":"e28bf7d9fcc22cdfde0c7c8f31e3648a4847e7bda9cb69f309f24257eee3dd41","final_host":"github.com","final_path":"/.well-known/mcp/server-card.json","ms":88,"path":"/.well-known/mcp/server-card.json","html":false,"json":null,"present":false},"api_catalog":{"status":404,"content_type":"text/html","bytes":224733,"sha256":"043fe355cf5194539f75b73ee6e6705cddf06bf31a4bfe73939163745875f4e4","final_host":"github.com","final_path":"/.well-known/api-catalog","ms":180,"path":"/.well-known/api-catalog","html":true,"json":null,"present":false},"a2a_card":{"status":404,"content_type":"application/json","bytes":21,"sha256":"e28bf7d9fcc22cdfde0c7c8f31e3648a4847e7bda9cb69f309f24257eee3dd41","final_host":"github.com","final_path":"/.well-known/agent-card.json","ms":80,"path":"/.well-known/agent-card.json","html":false,"json":null,"present":false}},"rings":{"v":1,"candidates":[],"mcp":[],"a2a":null,"webmcp":{"model_context":false,"bridge":false,"mcp_json":false,"link_rel":false,"any":false},"handshakes":0,"get_path":{"status":200,"content_type":"text/html","ms":316,"error":null},"get_host":{"status":530,"content_type":"text/plain","ms":10,"error":null}},"sf":[4,0,0,0,0,0,1,0,1],"derived":{"reachable":true,"home_class":"answers","robots_served":true,"robots_sitemap":true,"content_signal":false,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":["Bytespider"],"blocks_any_ai":true,"files":{"llms_txt":true,"agents_md":false,"agent_skills":false,"mcp_card":false,"api_catalog":false,"a2a_card":false},"any_capability":false,"ai_refused":[],"ai_thin":[],"aipref":0,"aipref_measured":true,"sf":[4,0,0,0,0,0,1,0,1],"rings":{"mcp_declared":false,"mcp_live":false,"mcp_auth":false,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":3,"fh":"github.com","fp_other_host":[],"fp":"99b771e96063c6b5","fp_bits":{"gptbot":10,"claudebot":12,"oaisearch":10,"perplexity":10}},"derive_v":5,"frame":"top1k","shard":4},{"v":1,"kind":"crawlcheck-census-record","run":43,"domain":"wikipedia.org","rank":30,"measured_at":"2026-11-13T08:07:00.657Z","home":{"browser":{"status":200,"content_type":"text/html","bytes":122506,"sha256":"0d267456014b06bd08d6e50a554fb32abd30407b5f56dee637c26614bac05349","final_host":"www.wikipedia.org","final_path":"/","ms":146,"identity":"Mobile browser","html":true,"fp":"1f628783c51e34fe"},"gptbot":{"status":200,"content_type":"text/html","bytes":122506,"sha256":"0d267456014b06bd08d6e50a554fb32abd30407b5f56dee637c26614bac05349","final_host":"www.wikipedia.org","final_path":"/","ms":146,"identity":"GPTBot","html":true,"fp":"1f628783c51e34fe"},"claudebot":{"status":403,"content_type":"text/plain","bytes":82,"sha256":"41609a6edee1e154f2092d6b2c64e26c5aa5a21d445ae636fe5a1f24b1ebc905","final_host":"wikipedia.org","final_path":"/","ms":108,"identity":"ClaudeBot","html":false},"oaisearch":{"status":200,"content_type":"text/html","bytes":122506,"sha256":"0d267456014b06bd08d6e50a554fb32abd30407b5f56dee637c26614bac05349","final_host":"www.wikipedia.org","final_path":"/","ms":145,"identity":"OAI-SearchBot","html":true,"fp":"1f628783c51e34fe"},"perplexity":{"status":200,"content_type":"text/html","bytes":122506,"sha256":"0d267456014b06bd08d6e50a554fb32abd30407b5f56dee637c26614bac05349","final_host":"www.wikipedia.org","final_path":"/","ms":117,"identity":"PerplexityBot","html":true,"fp":"1f628783c51e34fe"}},"robots":{"status":200,"content_type":"text/plain","bytes":28275,"sha256":"48b98d80c8444d37cdf50cc9a4d2257b66039477e581066d2356062a4cbd678f","final_host":"en.wikipedia.org","final_path":"/robots.txt","ms":371,"html":false,"text":"# robots.txt for http://www.wikipedia.org/ and friends\n#\n# Please note: There are a lot of pages on this site, and there are\n# some misbehaved spiders out there that go _way_ too fast. If you're\n# irresponsible, your access to the site may be blocked.\n#\n\n# Observed spamming large amounts of https://en.wikipedia.org/?curid=NNNNNN\n# and ignoring 429 ratelimit responses, claims to respect robots:\n# http://mj12bot.com/\nUser-agent: MJ12bot\nDisallow: /\n\n# advertising-related bots:\nUser-agent: Mediapartners-Google*\nDisallow: /\n\n# Wikipedia work bots:\nUser-agent: IsraBot\nDisallow:\n\nUser-agent: Orthogaffe\nDisallow:\n\n# Crawlers that are kind enough to obey, but which we'd rather not have\n# unless they're feeding search engines.\nUser-agent: UbiCrawler\nDisallow: /\n\nUser-agent: DOC\nDisallow: /\n\nUser-agent: Zao\nDisallow: /\n\n# Some bots are known to be trouble, particularly those designed to copy\n# entire sites. Please obey robots.txt.\nUser-agent: sitecheck.internetseer.com\nDisallow: /\n\nUser-agent: Zealbot\nDisallow: /\n\nUser-agent: MSIECrawler\nDisallow: /\n\nUser-agent: SiteSnagger\nDisallow: /\n\nUser-agent: WebStripper\nDisallow: /\n\nUser-agent: WebCopier\nDisallow: /\n\nUser-agent: Fetch\nDisallow: /\n\nUser-agent: Offline Explorer\nDisallow: /\n\nUser-agent: Teleport\nDisallow: /\n\nUser-agent: TeleportPro\nDisallow: /\n\nUser-agent: WebZIP\nDisallow: /\n\nUser-agent: linko\nDisallow: /\n\nUser-agent: HTTrack\nDisallow: /\n\nUser-agent: Microsoft.URL.Control\nDisallow: /\n\nUser-agent: Xenu\nDisallow: /\n\nUser-agent: larbin\nDisallow: /\n\nUser-agent: libwww\nDisallow: /\n\nUser-agent: ZyBORG\nDisallow: /\n\nUser-agent: Download Ninja\nDisallow: /\n\n# Misbehaving, requests much too fast\nUser-agent: fast\nDisallow: /\n\n# Sorry, wget in its recursive mode is a frequent problem.\n# Please read the man page and use it properly; there is a\n# --wait option you can use to set the delay between hits,\n# for instance.\nUser-agent: wget\nDisallow: /\n\n# The 'grub' distributed client has been *very* poorly behaved.\nUser-agent: grub-client\nDisallow: /\n\n# Doesn't follow robots.txt anyway, but...\nUser-agent: k2spider\nDisallow: /\n\n# Hits many times per second, not acceptable\n# http://www.nameprotect.com/botinfo.html\nUser-agent: NPBot\nDisallow: /\n\n# A capture bot, downloads gazillions of pages with no public benefit\n# http://www.webreaper.net/\nUser-agent: WebReaper\nDisallow: /\n\n# Per their statement, semrushbot respects crawl-delay directives\n# We want them to overall stay within reasonable request rates to\n# the backend (20 rps); keeping in mind that the crawl-delay will\n# be applied by site and not globally by the bot, 5 seconds seem\n# like a reasonable approximation\nUser-agent: SemrushBot\nCrawl-delay: 5\n\n#\n# Friendly, low-speed bots are welcome viewing article pages, but not\n# dynamically-generated pages please.\n#\n# Inktomi's \"Slurp\" can read a minimum delay between hits; if your\n# bot supports such a thing using the 'Crawl-delay' or another\n# instruction, please let us know.\n#\n# There is a special exception for API mobileview to allow dynamic\n# mobile web & app views to load section content.\n# These views aren't HTTP-cached but use parser cache aggressively\n# and don't expose Special-pages and other uncached content.\n#\n# Another exception is for REST API documentation, located at\n# /api/rest_v1/?doc.\n#\nUser-agent: *\nAllow: /w/api.php?action=mobileview&\nAllow: /w/load.php?\nAllow: /api/rest_v1/?doc\nAllow: /w/rest.php/site/v1/sitemap\nDisallow: /w/\nDisallow: /api/\nDisallow: /trap/\nDisallow: /wiki/Special:\nDisallow: /wiki/Spezial:\nDisallow: /wiki/Spesial:\nDisallow: /wiki/Special%3A\nDisallow: /wiki/Spezial%3A\nDisallow: /wiki/Spesial%3A\n#\n# ar:\nDisallow: /wiki/%D8%AE%D8%A7%D8%B5:Search\nDisallow: /wiki/%D8%AE%D8%A7%D8%B5%3ASearch\n#\n# dewiki:\n# T6937\n# sensible deletion and meta user discussion pages:\nDisallow: /wiki/Wikipedia:L%C3%B6schkandidaten/\nDisallow: /wiki/Wikipedia:Löschkandidaten/\nDisallow: /wiki/Wikipedia:Vandalensperrung/\nDisallow: /wiki/Wikipedia:Benutzersperrung/\nDisallow: /wiki/Wikipedia:Vermittlungsausschuss/\nDisallow: /wiki/Wikipedia:Administratoren/Probleme/\nDisallow: /wiki/Wikipedia:Adminkandidaturen/\nDisallow: /wiki/Wikipedia:Qualitätssicherung/\nDisallow: /wiki/Wikipedia:Qualit%C3%A4tssicherung/\n# 4937#5\nDisallow: /wiki/Wikipedia:Vandalismusmeldung/\nDisallow: /wiki/Wikipedia:Gesperrte_Lemmata/\nDisallow: /wiki/Wikipedia:Löschprüfung/\nDisallow: /wiki/Wikipedia:L%C3%B6schprüfung/\nDisallow: /wiki/Wikipedia:Administratoren/Notizen/\nDisallow: /wiki/Wikipedia:Schiedsgericht/Anfragen/\nDisallow: /wiki/Wikipedia:L%C3%B6schpr%C3%BCfung/\n# T14111\nDisallow: /wiki/Wikipedia:Checkuser/\nDisallow: /wiki/Wikipedia_Diskussion:Checkuser/\nDisallow: /wiki/Wikipedia_Diskussion:Adminkandidaturen/\n# T15961\nDisallow: /wiki/Wikipedia:Spam-Blacklist-Log\nDisallow: /wiki/Wikipedia%3ASpam-Blacklist-Log\nDisallow: /wiki/Wikipedia_Diskussion:Spam-Blacklist-Log\nDisallow: /wiki/Wikipedia_Diskussion%3ASpam-Blacklist-Log\n#\n# enwiki:\n# Folks get annoyed when VfD discussions end up the number 1 google hit for\n# their name. See T6776\nDisallow: /wiki/Wikipedia:Articles_for_deletion/\nDisallow: /wiki/Wikipedia%3AArticles_for_deletion/\nDisallow: /wiki/Wikipedia:Votes_for_deletion/\nDisallow: /wiki/Wikipedia%3AVotes_for_deletion/\nDisallow: /wiki/Wikipedia:Pages_for_deletion/\nDisallow: /wiki/Wikipedia%3APages_for_deletion/\nDisallow: /wiki/Wikipedia:Miscellany_for_deletion/\nDisallow: /wiki/Wikipedia%3AMiscellany_for_deletion/\nDisallow: /wiki/Wikipedia:Miscellaneous_deletion/\nDisallow: /wiki/Wikipedia%3AMiscellaneous_deletion/\nDisallow: /wiki/Wikipedia:Copyright_problems\nDisallow: /wiki/Wikipedia%3ACopyright_problems\nDisallow: /wiki/Wikipedia:Protected_titles/\nDisallow: /wiki/Wikipedia%3AProtected_titles/\n# T15398\nDisallow: /wiki/Wikipedia:WikiProject_Spam/\nDisallow: /wiki/Wikipedia%3AWikiProject_Spam/\n# T16075\nDisallow: /wiki/MediaWiki:Spam-blacklist\nDisallow: /wiki/MediaWiki%3ASpam-blacklist\nDisallow: /wiki/MediaWiki_talk:Spam-blacklist\nDisallow: /wiki/MediaWiki_talk%3ASpam-blacklist\n# T13261\nDisallow: /wiki/Wikipedia:Requests_for_arbitration/\nDisallow: /wiki/Wikipedia%3ARequests_for_arbitration/\nDisallow: /wiki/Wikipedia:Requests_for_comment/\nDisallow: /wiki/Wikipedia%3ARequests_for_comment/\nDisallow: /wiki/Wikipedia:Requests_for_adminship/\nDisallow: /wiki/Wikipedia%3ARequests_for_adminship/\n# T12288\nDisallow: /wiki/Wikipedia_talk:Articles_for_deletion/\nDisallow: /wiki/Wikipedia_talk%3AArticles_for_deletion/\nDisallow: /wiki/Wikipedia_talk:Votes_for_deletion/\nDisallow: /wiki/Wikipedia_talk%3AVotes_for_deletion/\nDisallow: /wiki/Wikipedia_talk:Pages_for_deletion/\nDisallow: /wiki/Wikipedia_talk%3APages_for_deletion/\nDisallow: /wiki/Wikipedia_talk:Miscellany_for_deletion/\nDisallow: /wiki/Wikipedia_talk%3AMiscellany_for_deletion/\nDisallow: /wiki/Wikipedia_talk:Miscellaneous_deletion/\nDisallow: /wiki/Wikipedia_talk%3AMiscellaneous_deletion/\n# T16793\nDisallow: /wiki/Wikipedia:Changing_username\nDisallow: /wiki/Wikipedia%3AChanging_username\nDisallow: /wiki/Wikipedia:Changing_username/\nDisallow: /wiki/Wikipedia%3AChanging_username/\nDisallow: /wiki/Wikipedia_talk:Changing_username\nDisallow: /wiki/Wikipedia_talk%3AChanging_username\nDisallow: /wiki/Wikipedia_talk:Changing_username/\nDisallow: /wiki/Wikipedia_talk%3AChanging_username/\n#\n# eswiki:\n# T8746\nDisallow: /wiki/Wikipedia:Consultas_de_borrado/\nDisallow: /wiki/Wikipedia%3AConsultas_de_borrado/\n#\n# fiwiki:\n# T10695\nDisallow: /wiki/Wikipedia:Poistettavat_sivut\nDisallow: /wiki/K%C3%A4ytt%C3%A4j%C3%A4:\nDisallow: /wiki/Käyttäjä:\nDisallow: /wiki/Keskustelu_k%C3%A4ytt%C3%A4j%C3%A4st%C3%A4:\nDisallow: /wiki/Keskustelu_käyttäjästä:\nDisallow: /wiki/Wikipedia:Yll%C3%A4pit%C3%A4j%C3%A4t/\nDisallow: /wiki/Wikipedia:Ylläpitäjät/\n#\n# hewiki:\nDisallow: /wiki/%D7%9E%D7%99%D7%95%D7%97%D7%93:Search\nDisallow: /wiki/%D7%9E%D7%99%D7%95%D7%97%D7%93%3ASearch\n#T11517\nDisallow: /wiki/ויקיפדיה:רשימת_מועמדים_למחיקה/\nDisallow: /wiki/ויקיפדיה%3Aרשימת_מועמדים_למחיקה/\nDisallow: /wiki/%D7%95%D7%99%D7%A7%D7%99%D7%A4%D7%93%D7%99%D7%94:%D7%A8%D7%A9%D7%99%D7%9E%D7%AA_%D7%9E%D7%95%D7%A2%D7%9E%D7%93%D7%99%D7%9D_%D7%9C%D7%9E%D7%97%D7%99%D7%A7%D7%94/\nDisallow: /wiki/%D7%95%D7%99%D7%A7%D7%99%D7%A4%D7%93%D7%99%D7%94%3A%D7%A8%D7%A9%D7%99%D7%9E%D7%AA_%D7%9E%D7%95%D7%A2%D7%9E%D7%93%D7%99%D7%9D_%D7%9C%D7%9E%D7%97%D7%99%D7%A7%D7%94/\nDisallow: /wiki/ויקיפדיה:ערכים_לא_קיימים_ומוגנים\nDisallow: /wiki/ויקיפדיה%3Aערכים_לא_קיימים_ומוגנים\nDisallow: /wiki/%D7%95%D7%99%D7%A7%D7%99%D7%A4%D7%93%D7%99%D7%94:%D7%A2%D7%A8%D7%9B%D7%99%D7%9D_%D7%9C%D7%90_%D7%A7%D7%99%D7%99%D7%9E%D7%99%D7%9D_%D7%95%D7%9E%D7%95%D7%92%D7%A0%D7%99%D7%9D\nDisallow: /wiki/%D7%95%D7%99%D7%A7%D7%99%D7%A4%D7%93%D7%99%D7%94%3A%D7%A2%D7%A8%D7%9B%D7%99%D7%9D_%D7%9C%D7%90_%D7%A7%D7%99%D7%99%D7%9E%D7%99%D7%9D_%D7%95%D7%9E%D7%95%D7%92%D7%A0%D7%99%D7%9D\nDisallow: /wiki/ויקיפדיה:דפים_לא_קיימים_ומוגנים\nDisallow: /wiki/ויקיפדיה%3Aדפים_לא_קיימים_ומוגנים\nDisallow: /wiki/%D7%95%D7%99%D7%A7%D7%99%D7%A4%D7%93%D7%99%D7%94:%D7%93%D7%A4%D7%99%D7%9D_%D7%9C%D7%90_%D7%A7%D7%99%D7%99%D7%9E%D7%99%D7%9D_%D7%95%D7%9E%D7%95%D7%92%D7%A0%D7%99%D7%9D\nDisallow: /wiki/%D7%95%D7%99%D7%A7%D7%99%D7%A4%D7%93%D7%99%D7%94%3A%D7%93%D7%A4%D7%99%D7%9D_%D7%9C%D7%90_%D7%A7%D7%99%D7%99%D7%9E%D7%99%D7%9D_%D7%95%D7%9E%D7%95%D7%92%D7%A0%D7%99%D7%9D\n#\n# huwiki:\nDisallow: /wiki/Speci%C3%A1lis:Search\nDisallow: /wiki/Speci%C3%A1lis%3ASearch\n#\n# itwiki:\n# T7545\nDisallow: /wiki/Wikipedia:Pagine_da_cancellare\nDisallow: /wiki/Wikipedia%3APagine_da_cancellare\nDisallow: /wiki/Wikipedia:Utenti_problematici\nDisallow: /wiki/Wikipedia%3AUtenti_problematici\nDisallow: /wiki/Wikipedia:Vandalismi_in_corso\nDisallow: /wiki/Wikipedia%3AVandalismi_in_corso\nDisallow: /wiki/Wikipedia:Amministratori\nDisallow: /wiki/Wikipedia%3AAmministratori\nDisallow: /wiki/Wikipedia:Proposte_di_cancellazione_semplificata\nDisallow: /wiki/Wikipedia%3AProposte_di_cancellazione_semplificata\nDisallow: /wiki/Categoria:Da_cancellare_subito\nDisallow: /wiki/Categoria%3ADa_cancellare_subito\nDisallow: /wiki/Wikipedia:Sospette_violazioni_di_copyright\nDisallow: /wiki/Wikipedia%3ASospette_violazioni_di_copyright\nDisallow: /wiki/Categoria:Da_controllare_per_copyright\nDisallow: /wiki/Categoria%3ADa_controllare_per_copyright\nDisallow: /wiki/Progetto:Rimozione_contributi_sospetti\nDisallow: /wiki/Progetto%3ARimozione_contributi_sospetti\nDisallow: /wiki/Categoria:Da_cancellare_subito_per_violazione_integrale_copyright\nDisallow: /wiki/Categoria%3ADa_cancellare_subito_per_violazione_integrale_copyright\nDisallow: /wiki/Progetto:Cococo\nDisallow: /wiki/Progetto%3ACococo\nDisallow: /wiki/Discussioni_progetto:Cococo\nDisallow: /wiki/Discussioni_progetto%3ACococo\n#\n# jawiki\nDisallow: /wiki/%E7%89%B9%E5%88%A5:Search\nDisallow: /wiki/%E7%89%B9%E5%88%A5%3ASearch\n# T7239\nDisallow: /wiki/Wikipedia:%E5%89%8A%E9%99%A4%E4%BE%9D%E9%A0%BC/\nDisallow: /wiki/Wikipedia%3A%E5%89%8A%E9%99%A4%E4%BE%9D%E9%A0%BC/\nDisallow: /wiki/Wikipedia:%E5%88%A9%E7%94%A8%E8%80%85%E3%83%9A%E3%83%BC%E3%82%B8%E3%81%AE%E5%89%8A%E9%99%A4%E4%BE%9D%E9%A0%BC\nDisallow: /wiki/Wikipedia%3A%E5%88%A9%E7%94%A8%E8%80%85%E3%83%9A%E3%83%BC%E3%82%B8%E3%81%AE%E5%89%8A%E9%99%A4%E4%BE%9D%E9%A0%BC\n# nowiki\n# T13432\nDisallow: /wiki/Bruker:\nDisallow: /wiki/Bruker%3A\nDisallow: /wiki/Brukerdiskusjon\nDisallow: /wiki/Wikipedia:Administratorer\nDisallow: /wiki/Wikipedia%3AAdministratorer\nDisallow: /wiki/Wikipedia-diskusjon:Administratorer\nDisallow: /wiki/Wikipedia-diskusjon%3AAdministratorer\nDisallow: /wiki/Wikipedia:Sletting\nDisallow: /wiki/Wikipedia%3ASletting\nDisallow: /wiki/Wikipedia-diskusjon:Sletting\nDisallow: /wiki/Wikipedia-diskusjon%3ASletting\n#\n# plwiki\n# T10067\nDisallow: /wiki/Wikipedia:Strony_do_usuni%C4%99cia\nDisallow: /wiki/Wikipedia%3AStrony_do_usuni%C4%99cia\nDisallow: /wiki/Wikipedia:Do_usuni%C4%99cia\nDisallow: /wiki/Wikipedia%3ADo_usuni%C4%99cia\nDisallow: /wiki/Wikipedia:SDU/\nDisallow: /wiki/Wikipedia%3ASDU/\nDisallow: /wiki/Wikipedia:Strony_podejrzane_o_naruszenie_praw_autorskich\nDisallow: /wiki/Wikipedia%3AStrony_podejrzane_o_naruszenie_praw_autorskich\n#\n# ptwiki:\n# T7394\nDisallow: /wiki/Wikipedia:Páginas_para_eliminar/\nDisallow: /wiki/Wikipedia:P%C3%A1ginas_para_eliminar/\nDisallow: /wiki/Wikipedia%3AP%C3%A1ginas_para_eliminar/\nDisallow: /wiki/Wikipedia_Discussão:Páginas_para_eliminar/\nDisallow: /wiki/Wikipedia_Discuss%C3%A3o:P%C3%A1ginas_para_eliminar/\nDisallow: /wiki/Wikipedia_Discuss%C3%A3o%3AP%C3%A1ginas_para_eliminar/\n#\n# rowiki:\n# T14546\nDisallow: /wiki/Wikipedia:Pagini_de_%C5%9Fters\nDisallow: /wiki/Wikipedia%3APagini_de_%C5%9Fters\nDisallow: /wiki/Discu%C5%A3ie_Wikipedia:Pagini_de_%C5%9Fters\nDisallow: /wiki/Discu%C5%A3ie_Wikipedia%3APagini_de_%C5%9Fters\n#\n# ruwiki:\nDisallow: /wiki/%D0%A1%D0%BF%D0%B5%D1%86%D0%B8%D0%B0%D0%BB%D1%8C%D0%BD%D1%8B%D0%B5:Search\nDisallow: /wiki/%D0%A1%D0%BF%D0%B5%D1%86%D0%B8%D0%B0%D0%BB%D1%8C%D0%BD%D1%8B%D0%B5%3ASearch\n#\n# svwiki:\n# T12229\nDisallow: /wiki/Wikipedia%3ASidor_f%C3%B6reslagna_f%C3%B6r_radering\nDisallow: /wiki/Wikipedia:Sidor_f%C3%B6reslagna_f%C3%B6r_radering\nDisallow: /wiki/Wikipedia:Sidor_föreslagna_för_radering\nDisallow: /wiki/Användare\nDisallow: /wiki/Anv%C3%A4ndare\nDisallow: /wiki/Användardiskussion\nDisallow: /wiki/Anv%C3%A4ndardiskussion\nDisallow: /wiki/Wikipedia:Skyddade_sidnamn\nDisallow: /wiki/Wikipedia%3ASkyddade_sidnamn\n# T13291\nDisallow: /wiki/Wikipedia:Sidor_som_bör_raderas\nDisallow: /wiki/Wikipedia:Sidor_som_b%C3%B6r_raderas\nDisallow: /wiki/Wikipedia%3ASidor_som_b%C3%B6r_raderas\n#\n# zhwiki:\n# T7104\nDisallow: /wiki/Wikipedia:删除投票/侵权\nDisallow: /wiki/Wikipedia:%E5%88%A0%E9%99%A4%E6%8A%95%E7%A5%A8/%E4%BE%B5%E6%9D%83\nDisallow: /wiki/Wikipedia:删除投票和请求\nDisallow: /wiki/Wikipedia:%E5%88%A0%E9%99%A4%E6%8A%95%E7%A5%A8%E5%92%8C%E8%AF%B7%E6%B1%82\nDisallow: /wiki/Category:快速删除候选\nDisallow: /wiki/Category:%E5%BF%AB%E9%80%9F%E5%88%A0%E9%99%A4%E5%80%99%E9%80%89\nDisallow: /wiki/Category:维基百科需要翻译的文章\nDisallow: /wiki/Category:%E7%BB%B4%E5%9F%BA%E7%99%BE%E7%A7%91%E9%9C%80%E8%A6%81%E7%BF%BB%E8%AF%91%E7%9A%84%E6%96%87%E7%AB%A0\n#\n# sister projects\n#\n# enwikinews:\n# T7340\nDisallow: /wiki/Portal:Prepared_stories/\nDisallow: /wiki/Portal%3APrepared_stories/\n#\n# itwikinews\n# T11138\nDisallow: /wiki/Wikinotizie:Richieste_di_cancellazione\nDisallow: /wiki/Wikinotizie:Sospette_violazioni_di_copyright\nDisallow: /wiki/Categoria:Da_cancellare_subito\nDisallow: /wiki/Categoria:Da_cancellare_subito_per_violazione_integrale_copyright\nDisallow: /wiki/Wikinotizie:Storie_in_preparazione\n#\n# enwikiquote:\n# T17095\nDisallow: /wiki/Wikiquote:Votes_for_deletion/\nDisallow: /wiki/Wikiquote%3AVotes_for_deletion/\nDisallow: /wiki/Wikiquote_talk:Votes_for_deletion/\nDisallow: /wiki/Wikiquote_talk%3AVotes_for_deletion/\nDisallow: /wiki/Wikiquote:Votes_for_deletion_archive/\nDisallow: /wiki/Wikiquote%3AVotes_for_deletion_archive/\nDisallow: /wiki/Wikiquote_talk:Votes_for_deletion_archive/\nDisallow: /wiki/Wikiquote_talk%3AVotes_for_deletion_archive/\n#\n# enwikibooks\nDisallow: /wiki/Wikibooks:Votes_for_deletion\n#\n\n\nSitemap: https://en.wikipedia.org/w/rest.php/site/v1/sitemap/0\n\n#\n#\n#----------------------------------------------------------#\n#\n#\n#\n#\n# Localisable part of robots.txt for en.wikipedia.org\n#\n# Edit at https://en.wikipedia.org/w/index.php?title=MediaWiki:Robots.txt&action=edit\n# Don't add newlines here. All rules set here are active for every user-agent.\n#\n# Please check any changes using a syntax validator\n# Enter https://en.wikipedia.org/robots.txt as the URL to check.\n#\n# https://phabricator.wikimedia.org/T16075\nDisallow: /wiki/MediaWiki:Spam-blacklist\nDisallow: /wiki/MediaWiki%3ASpam-blacklist\nDisallow: /wiki/MediaWiki_talk:Spam-blacklist\nDisallow: /wiki/MediaWiki_talk%3ASpam-blacklist\nDisallow: /wiki/Wikipedia:WikiProject_Spam\nDisallow: /wiki/Wikipedia_talk:WikiProject_Spam\n#\n# Folks get annoyed when XfD discussions end up the number 1 google hit for\n# their name. \n# https://phabricator.wikimedia.org/T16075\nDisallow: /wiki/Wikipedia:Articles_for_deletion\nDisallow: /wiki/Wikipedia%3AArticles_for_deletion\nDisallow: /wiki/Wikipedia:Votes_for_deletion\nDisallow: /wiki/Wikipedia%3AVotes_for_deletion\nDisallow: /wiki/Wikipedia:Pages_for_deletion\nDisallow: /wiki/Wikipedia%3APages_for_deletion\nDisallow: /wiki/Wikipedia:Miscellany_for_deletion\nDisallow: /wiki/Wikipedia%3AMiscellany_for_deletion\nDisallow: /wiki/Wikipedia:Miscellaneous_deletion\nDisallow: /wiki/Wikipedia%3AMiscellaneous_deletion\nDisallow: /wiki/Wikipedia:Categories_for_discussion\nDisallow: /wiki/Wikipedia%3ACategories_for_discussion\nDisallow: /wiki/Wikipedia:Templates_for_deletion\nDisallow: /wiki/Wikipedia%3ATemplates_for_deletion\nDisallow: /wiki/Wikipedia:Redirects_for_discussion\nDisallow: /wiki/Wikipedia%3ARedirects_for_discussion\nDisallow: /wiki/Wikipedia:Deletion_review\nDisallow: /wiki/Wikipedia%3ADeletion_review\nDisallow: /wiki/Wikipedia:WikiProject_Deletion_sorting\nDisallow: /wiki/Wikipedia%3AWikiProject_Deletion_sorting\nDisallow: /wiki/Wikipedia:Files_for_deletion\nDisallow: /wiki/Wikipedia%3AFiles_for_deletion\nDisallow: /wiki/Wikipedia:Files_for_discussion\nDisallow: /wiki/Wikipedia%3AFiles_for_discussion\nDisallow: /wiki/Wikipedia:Possibly_unfree_files\nDisallow: /wiki/Wikipedia%3APossibly_unfree_files\n#\n# https://phabricator.wikimedia.org/T12288\nDisallow: /wiki/Wikipedia_talk:Articles_for_deletion\nDisallow: /wiki/Wikipedia_talk%3AArticles_for_deletion\nDisallow: /wiki/Wikipedia_talk:Votes_for_deletion\nDisallow: /wiki/Wikipedia_talk%3AVotes_for_deletion\nDisallow: /wiki/Wikipedia_talk:Pages_for_deletion\nDisallow: /wiki/Wikipedia_talk%3APages_for_deletion\nDisallow: /wiki/Wikipedia_talk:Miscellany_for_deletion\nDisallow: /wiki/Wikipedia_talk%3AMiscellany_for_deletion\nDisallow: /wiki/Wikipedia_talk:Miscellaneous_deletion\nDisallow: /wiki/Wikipedia_talk%3AMiscellaneous_deletion\nDisallow: /wiki/Wikipedia_talk:Templates_for_deletion\nDisallow: /wiki/Wikipedia_talk%3ATemplates_for_deletion\nDisallow: /wiki/Wikipedia_talk:Categories_for_discussion\nDisallow: /wiki/Wikipedia_talk%3ACategories_for_discussion\nDisallow: /wiki/Wikipedia_talk:Deletion_review\nDisallow: /wiki/Wikipedia_talk%3ADeletion_review\nDisallow: /wiki/Wikipedia_talk:WikiProject_Deletion_sorting\nDisallow: /wiki/Wikipedia_talk%3AWikiProject_Deletion_sorting\nDisallow: /wiki/Wikipedia_talk:Files_for_deletion\nDisallow: /wiki/Wikipedia_talk%3AFiles_for_deletion\nDisallow: /wiki/Wikipedia_talk:Files_for_discussion\nDisallow: /wiki/Wikipedia_talk%3AFiles_for_discussion\nDisallow: /wiki/Wikipedia_talk:Possibly_unfree_files\nDisallow: /wiki/Wikipedia_talk%3APossibly_unfree_files\n#\nDisallow: /wiki/Wikipedia:Copyright_problems\nDisallow: /wiki/Wikipedia%3ACopyright_problems\nDisallow: /wiki/Wikipedia_talk:Copyright_problems\nDisallow: /wiki/Wikipedia_talk%3ACopyright_problems\nDisallow: /wiki/Wikipedia:Suspected_copyright_violations\nDisallow: /wiki/Wikipedia%3ASuspected_copyright_violations\nDisallow: /wiki/Wikipedia_talk:Suspected_copyright_violations\nDisallow: /wiki/Wikipedia_talk%3ASuspected_copyright_violations\nDisallow: /wiki/Wikipedia:Contributor_copyright_investigations\nDisallow: /wiki/Wikipedia%3AContributor_copyright_investigations\nDisallow: /wiki/Wikipedia:Contributor_copyright_investigations\nDisallow: /wiki/Wikipedia%3AContributor_copyright_investigations\nDisallow: /wiki/Wikipedia_talk:Contributor_copyright_investigations\nDisallow: /wiki/Wikipedia_talk%3AContributor_copyright_investigations\nDisallow: /wiki/Wikipedia_talk:Contributor_copyright_investigations\nDisallow: /wiki/Wikipedia_talk%3AContributor_copyright_investigations\nDisallow: /wiki/Wikipedia:Protected_titles\nDisallow: /wiki/Wikipedia%3AProtected_titles\nDisallow: /wiki/Wikipedia_talk:Protected_titles\nDisallow: /wiki/Wikipedia_talk%3AProtected_titles\nDisallow: /wiki/Wikipedia:Articles_for_creation\nDisallow: /wiki/Wikipedia%3AArticles_for_creation\nDisallow: /wiki/Wikipedia_talk:Articles_for_creation\nDisallow: /wiki/Wikipedia_talk%3AArticles_for_creation\nDisallow: /wiki/Wikipedia_talk:Article_wizard\nDisallow: /wiki/Wikipedia_talk%3AArticle_wizard\n#\n# https://phabricator.wikimedia.org/T13261\nDisallow: /wiki/Wikipedia:Requests_for_arbitration\nDisallow: /wiki/Wikipedia%3ARequests_for_arbitration\nDisallow: /wiki/Wikipedia_talk:Requests_for_arbitration\nDisallow: /wiki/Wikipedia_talk%3ARequests_for_arbitration\nDisallow: /wiki/Wikipedia:Requests_for_comment\nDisallow: /wiki/Wikipedia%3ARequests_for_comment\nDisallow: /wiki/Wikipedia_talk:Requests_for_comment\nDisallow: /wiki/Wikipedia_talk%3ARequests_for_comment\nDisallow: /wiki/Wikipedia:Requests_for_adminship\nDisallow: /wiki/Wikipedia%3ARequests_for_adminship\nDisallow: /wiki/Wikipedia_talk:Requests_for_adminship\nDisallow: /wiki/Wikipedia_talk%3ARequests_for_adminship\n#\n# https://phabricator.wikimedia.org/T14111\nDisallow: /wiki/Wikipedia:Requests_for_checkuser\nDisallow: /wiki/Wikipedia%3ARequests_for_checkuser\nDisallow: /wiki/Wikipedia_talk:Requests_for_checkuser\nDisallow: /wiki/Wikipedia_talk%3ARequests_for_checkuser\n#\n# https://phabricator.wikimedia.org/T15398\nDisallow: /wiki/Wikipedia:WikiProject_Spam\nDisallow: /wiki/Wikipedia%3AWikiProject_Spam\n#\n# https://phabricator.wikimedia.org/T16793\nDisallow: /wiki/Wikipedia:Changing_username\nDisallow: /wiki/Wikipedia%3AChanging_username\nDisallow: /wiki/Wikipedia:Changing_username\nDisallow: /wiki/Wikipedia%3AChanging_username\nDisallow: /wiki/Wikipedia_talk:Changing_username\nDisallow: /wiki/Wikipedia_talk%3AChanging_username\nDisallow: /wiki/Wikipedia_talk:Changing_username\nDisallow: /wiki/Wikipedia_talk%3AChanging_username\n#\nDisallow: /wiki/Wikipedia:Administrators%27_noticeboard\nDisallow: /wiki/Wikipedia%3AAdministrators%27_noticeboard\nDisallow: /wiki/Wikipedia_talk:Administrators%27_noticeboard\nDisallow: /wiki/Wikipedia_talk%3AAdministrators%27_noticeboard\nDisallow: /wiki/Wikipedia:Community_sanction_noticeboard\nDisallow: /wiki/Wikipedia%3ACommunity_sanction_noticeboard\nDisallow: /wiki/Wikipedia_talk:Community_sanction_noticeboard\nDisallow: /wiki/Wikipedia_talk%3ACommunity_sanction_noticeboard\nDisallow: /wiki/Wikipedia:Bureaucrats%27_noticeboard\nDisallow: /wiki/Wikipedia%3ABureaucrats%27_noticeboard\nDisallow: /wiki/Wikipedia_talk:Bureaucrats%27_noticeboard\nDisallow: /wiki/Wikipedia_talk%3ABureaucrats%27_noticeboard\n#\nDisallow: /wiki/Wikipedia:Sockpuppet_investigations\nDisallow: /wiki/Wikipedia%3ASockpuppet_investigations\nDisallow: /wiki/Wikipedia_talk:Sockpuppet_investigations\nDisallow: /wiki/Wikipedia_talk%3ASockpuppet_investigations\n#\nDisallow: /wiki/Wikipedia:Neutral_point_of_view/Noticeboard\nDisallow: /wiki/Wikipedia%3ANeutral_point_of_view/Noticeboard\nDisallow: /wiki/Wikipedia_talk:Neutral_point_of_view/Noticeboard\nDisallow: /wiki/Wikipedia_talk%3ANeutral_point_of_view/Noticeboard\n#\nDisallow: /wiki/Wikipedia:No_original_research/noticeboard\nDisallow: /wiki/Wikipedia%3ANo_original_research/noticeboard\nDisallow: /wiki/Wikipedia_talk:No_original_research/noticeboard\nDisallow: /wiki/Wikipedia_talk%3ANo_original_research/noticeboard\n#\nDisallow: /wiki/Wikipedia:Fringe_theories/Noticeboard\nDisallow: /wiki/Wikipedia%3AFringe_theories/Noticeboard\nDisallow: /wiki/Wikipedia_talk:Fringe_theories/Noticeboard\nDisallow: /wiki/Wikipedia_talk%3AFringe_theories/Noticeboard\n#\nDisallow: /wiki/Wikipedia:Conflict_of_interest/Noticeboard\nDisallow: /wiki/Wikipedia%3AConflict_of_interest/Noticeboard\nDisallow: /wiki/Wikipedia_talk:Conflict_of_interest/Noticeboard\nDisallow: /wiki/Wikipedia_talk%3AConflict_of_interest/Noticeboard\n#\nDisallow: /wiki/Wikipedia:Long-term_abuse\nDisallow: /wiki/Wikipedia%3ALong-term_abuse\nDisallow: /wiki/Wikipedia_talk:Long-term_abuse\nDisallow: /wiki/Wikipedia_talk%3ALong-term_abuse\nDisallow: /wiki/Wikipedia:Long_term_abuse\nDisallow: /wiki/Wikipedia%3ALong_term_abuse\nDisallow: /wiki/Wikipedia_talk:Long_term_abuse\nDisallow: /wiki/Wikipedia_talk%3ALong_term_abuse\n#\nDisallow: /wiki/Wikipedia:Wikiquette_assistance\nDisallow: /wiki/Wikipedia%3AWikiquette_assistance\n#\nDisallow: /wiki/Wikipedia:Abuse_reports\nDisallow: /wiki/Wikipedia%3AAbuse_reports\nDisallow: /wiki/Wikipedia_talk:Abuse_reports\nDisallow: /wiki/Wikipedia_talk%3AAbuse_reports\nDisallow: /wiki/Wikipedia:Abuse_response\nDisallow: /wiki/Wikipedia%3AAbuse_response\nDisallow: /wiki/Wikipedia_talk:Abuse_response\nDisallow: /wiki/Wikipedia_talk%3AAbuse_response\n#\nDisallow: /wiki/Wikipedia:Reliable_sources/Noticeboard\nDisallow: /wiki/Wikipedia%3AReliable_sources/Noticeboard\nDisallow: /wiki/Wikipedia_talk:Reliable_sources/Noticeboard\nDisallow: /wiki/Wikipedia_talk%3AReliable_sources/Noticeboard\n#\nDisallow: /wiki/Wikipedia:Suspected_sock_puppets\nDisallow: /wiki/Wikipedia%3ASuspected_sock_puppets\nDisallow: /wiki/Wikipedia_talk:Suspected_sock_puppets\nDisallow: /wiki/Wikipedia_talk%3ASuspected_sock_puppets\n#\nDisallow: /wiki/Wikipedia:Biographies_of_living_persons/Noticeboard\nDisallow: /wiki/Wikipedia%3ABiographies_of_living_persons/Noticeboard\nDisallow: /wiki/Wikipedia_talk:Biographies_of_living_persons/Noticeboard\nDisallow: /wiki/Wikipedia_talk%3ABiographies_of_living_persons/Noticeboard\nDisallow: /wiki/Wikipedia:Biographies_of_living_persons%2FNoticeboard\nDisallow: /wiki/Wikipedia%3ABiographies_of_living_persons%2FNoticeboard\nDisallow: /wiki/Wikipedia_talk:Biographies_of_living_persons%2FNoticeboard\nDisallow: /wiki/Wikipedia_talk%3ABiographies_of_living_persons%2FNoticeboard\n#\nDisallow: /wiki/Wikipedia:Content_noticeboard\nDisallow: /wiki/Wikipedia%3AContent_noticeboard\nDisallow: /wiki/Wikipedia_talk:Content_noticeboard\nDisallow: /wiki/Wikipedia_talk%3AContent_noticeboard\n#\nDisallow: /wiki/Template:Editnotices\nDisallow: /wiki/Template%3AEditnotices\n#\nDisallow: /wiki/Wikipedia:Arbitration\nDisallow: /wiki/Wikipedia%3AArbitration\nDisallow: /wiki/Wikipedia_talk:Arbitration\nDisallow: /wiki/Wikipedia_talk%3AArbitration\n#\nDisallow: /wiki/Wikipedia:Arbitration_Committee\nDisallow: /wiki/Wikipedia%3AArbitration_Committee\nDisallow: /wiki/Wikipedia_talk:Arbitration_Committee\nDisallow: /wiki/Wikipedia_talk%3AArbitration_Committee\n#\nDisallow: /wiki/Wikipedia:Arbitration_Committee_Elections\nDisallow: /wiki/Wikipedia%3AArbitration_Committee_Elections\nDisallow: /wiki/Wikipedia_talk:Arbitration_Committee_Elections\nDisallow: /wiki/Wikipedia_talk%3AArbitration_Committee_Elections\n#\nDisallow: /wiki/Wikipedia:Mediation_Committee\nDisallow: /wiki/Wikipedia%3AMediation_Committee\nDisallow: /wiki/Wikipedia_talk:Mediation_Committee\nDisallow: /wiki/Wikipedia_talk%3AMediation_Committee\n#\nDisallow: /wiki/Wikipedia:Mediation_Cabal/Cases\nDisallow: /wiki/Wikipedia%3AMediation_Cabal/Cases\n#\nDisallow: /wiki/Wikipedia:Requests_for_bureaucratship\nDisallow: /wiki/Wikipedia%3ARequests_for_bureaucratship\nDisallow: /wiki/Wikipedia_talk:Requests_for_bureaucratship\nDisallow: /wiki/Wikipedia_talk%3ARequests_for_bureaucratship\n#\nDisallow: /wiki/Wikipedia:Administrator_review\nDisallow: /wiki/Wikipedia%3AAdministrator_review\nDisallow: /wiki/Wikipedia_talk:Administrator_review\nDisallow: /wiki/Wikipedia_talk%3AAdministrator_review\n#\nDisallow: /wiki/Wikipedia:Editor_review\nDisallow: /wiki/Wikipedia%3AEditor_review\nDisallow: /wiki/Wikipedia_talk:Editor_review\nDisallow: /wiki/Wikipedia_talk%3AEditor_review\n#\nDisallow: /wiki/Wikipedia:Article_Incubator\nDisallow: /wiki/Wikipedia%3AArticle_Incubator\nDisallow: /wiki/Wikipedia_talk:Article_Incubator\nDisallow: /wiki/Wikipedia_talk%3AArticle_Incubator\n#\nDisallow: /wiki/Category:Noindexed_pages\nDisallow: /wiki/Category%3ANoindexed_pages\n#\n# User sandboxes for modules and Template Styles are placed in these subpages for testing\n#\nDisallow: /wiki/Module:Sandbox\nDisallow: /wiki/Module%3ASandbox\nDisallow: /wiki/Template:TemplateStyles_sandbox\nDisallow: /wiki/Template%3ATemplateStyles_sandbox\n# Adminship and arbitration discussions are blocked from search engines, so blocking newer adminship processes (as of 2024) as well\nDisallow: /wiki/Wikipedia:Administrator_recall\nDisallow: /wiki/Wikipedia%3AAdministrator_recall\nDisallow: /wiki/Wikipedia_talk:Administrator_recall\nDisallow: /wiki/Wikipedia_talk%3AAdministrator_recall\nDisallow: /wiki/Wikipedia:Administrator_elections\nDisallow: /wiki/Wikipedia%3AAdministrator_elections\nDisallow: /wiki/Wikipedia_talk:Administrator_elections\nDisallow: /wiki/Wikipedia_talk%3AAdministrator_elections","agents":{"GPTBot":true,"OAI-SearchBot":true,"ChatGPT-User":true,"ClaudeBot":true,"Claude-SearchBot":true,"PerplexityBot":true,"Google-Extended":true,"Applebot-Extended":true,"CCBot":true,"Bytespider":true,"Meta-ExternalAgent":true,"Amazonbot":true}},"sitemap":{"status":403,"content_type":"text/html","bytes":3360,"sha256":"7fd8089abd10fabff071caac5aaeaabc9bc1fa00498227ef6aa022a0f8e50a5b","final_host":"en.wikipedia.org","final_path":"/w/rest.php/site/v1/sitemap/0","ms":11,"url":"https://en.wikipedia.org/w/rest.php/site/v1/sitemap/0","declared":true,"xml":false},"files":{"llms_txt":{"status":404,"content_type":"text/html","bytes":2692,"sha256":"ee1efb6d6f284ee741b66f61d5baac16f8034d67feef6c51681982ab60a06a9f","final_host":"en.wikipedia.org","final_path":"/llms.txt","ms":47,"path":"/llms.txt","html":true,"json":null,"present":false},"agents_md":{"status":404,"content_type":"text/html","bytes":2695,"sha256":"2e8e528146a7f212165514f7c0b51994a33c8bc22cd69a2e129679b193bc83aa","final_host":"en.wikipedia.org","final_path":"/agents.md","ms":134,"path":"/agents.md","html":true,"json":null,"present":false},"agent_skills":{"status":404,"content_type":"text/html","bytes":2773,"sha256":"c7b071b55ca051a0fde81d87b070a8b1b66fa71f36f63fd136614db7db68d17b","final_host":"en.wikipedia.org","final_path":"/.well-known/agent-skills/index.json","ms":52,"path":"/.well-known/agent-skills/index.json","html":true,"json":null,"present":false},"mcp_card":{"status":404,"content_type":"text/html","bytes":2764,"sha256":"65a730402eeab88bdb2e21843c2c9f2a59fbe90d9dbdbedb7bdb56ec0a20a62c","final_host":"en.wikipedia.org","final_path":"/.well-known/mcp/server-card.json","ms":37,"path":"/.well-known/mcp/server-card.json","html":true,"json":null,"present":false},"api_catalog":{"status":404,"content_type":"text/html","bytes":2737,"sha256":"9321a2060e71ab7f661cea22389c9453d798be8fdddd854ad3fb10b1ccc2aff0","final_host":"en.wikipedia.org","final_path":"/.well-known/api-catalog","ms":42,"path":"/.well-known/api-catalog","html":true,"json":null,"present":false},"a2a_card":{"status":404,"content_type":"text/html","bytes":2749,"sha256":"de9924b979bbdf4135e5a534f74db7b6c3b93ee915160ec2e1571c0b3e9dac14","final_host":"en.wikipedia.org","final_path":"/.well-known/agent-card.json","ms":36,"path":"/.well-known/agent-card.json","html":true,"json":null,"present":false}},"rings":{"v":1,"candidates":[],"mcp":[],"a2a":null,"webmcp":{"model_context":false,"bridge":false,"mcp_json":false,"link_rel":false,"any":false},"handshakes":0,"get_path":{"status":404,"content_type":"text/html","ms":35,"error":null},"get_host":{"status":530,"content_type":"text/plain","ms":18,"error":null}},"sf":[4,0,0,0,0,0,0,0,1],"derived":{"reachable":true,"home_class":"answers","robots_served":true,"robots_sitemap":true,"content_signal":false,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":[],"blocks_any_ai":false,"files":{"llms_txt":false,"agents_md":false,"agent_skills":false,"mcp_card":false,"api_catalog":false,"a2a_card":false},"any_capability":false,"ai_refused":["claudebot"],"ai_thin":[],"aipref":0,"aipref_measured":true,"sf":[4,0,0,0,0,0,0,0,1],"rings":{"mcp_declared":false,"mcp_live":false,"mcp_auth":false,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":3,"fh":"www.wikipedia.org","fp_other_host":[],"fp":"1f628783c51e34fe","fp_bits":{"gptbot":0,"oaisearch":0,"perplexity":0}},"derive_v":5,"frame":"top1k","shard":5},{"v":1,"kind":"crawlcheck-census-record","run":43,"domain":"appsflyersdk.com","rank":31,"measured_at":"2026-11-13T08:07:15.177Z","home":{"browser":{"status":403,"content_type":"text/xml","bytes":146,"sha256":"95769eb326e23b8cd1b7a9b82d884a713a9724f49b2794ac5b967d8cd706de83","final_host":"appsflyersdk.com","final_path":"/","ms":39,"identity":"Mobile browser","html":false},"gptbot":{"status":403,"content_type":"text/xml","bytes":146,"sha256":"95769eb326e23b8cd1b7a9b82d884a713a9724f49b2794ac5b967d8cd706de83","final_host":"appsflyersdk.com","final_path":"/","ms":19,"identity":"GPTBot","html":false},"claudebot":{"status":403,"content_type":"text/xml","bytes":146,"sha256":"95769eb326e23b8cd1b7a9b82d884a713a9724f49b2794ac5b967d8cd706de83","final_host":"appsflyersdk.com","final_path":"/","ms":6,"identity":"ClaudeBot","html":false},"oaisearch":{"status":403,"content_type":"text/xml","bytes":146,"sha256":"95769eb326e23b8cd1b7a9b82d884a713a9724f49b2794ac5b967d8cd706de83","final_host":"appsflyersdk.com","final_path":"/","ms":18,"identity":"OAI-SearchBot","html":false},"perplexity":{"status":403,"content_type":"text/xml","bytes":146,"sha256":"95769eb326e23b8cd1b7a9b82d884a713a9724f49b2794ac5b967d8cd706de83","final_host":"appsflyersdk.com","final_path":"/","ms":6,"identity":"PerplexityBot","html":false}},"robots":{"status":403,"content_type":"text/xml","bytes":146,"sha256":"95769eb326e23b8cd1b7a9b82d884a713a9724f49b2794ac5b967d8cd706de83","final_host":"appsflyersdk.com","final_path":"/robots.txt","ms":88,"html":false,"text":null,"agents":null},"sitemap":null,"files":null,"derived":{"reachable":false,"home_class":"refused_us","robots_served":false,"robots_sitemap":false,"content_signal":false,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":[],"blocks_any_ai":false,"files":{"llms_txt":false,"agents_md":false,"agent_skills":false,"mcp_card":false,"api_catalog":false,"a2a_card":false},"any_capability":false,"ai_refused":[],"ai_thin":[],"rings":{"mcp_declared":false,"mcp_live":false,"mcp_auth":false,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":3,"fh":"appsflyersdk.com","fp_other_host":[]},"derive_v":5,"frame":"top1k","shard":6},{"v":1,"kind":"crawlcheck-census-record","run":43,"domain":"bing.com","rank":32,"measured_at":"2026-11-13T08:08:20.375Z","home":{"browser":{"status":200,"content_type":"text/html","bytes":53187,"sha256":"7da086485bf5d7fba2db5bb6508eb68b9ecad5c35b9c7f908702eb0c9918f2c7","final_host":"www.bing.com","final_path":"/","ms":255,"identity":"Mobile browser","html":true},"gptbot":{"status":200,"content_type":"text/html","bytes":79018,"sha256":"2199f5e803f5f610989bbd4d72b8ae6fb207de3a8f68c775cbea4bde00583c05","final_host":"www.bing.com","final_path":"/","ms":195,"identity":"GPTBot","html":true},"claudebot":{"status":200,"content_type":"text/html","bytes":78894,"sha256":"01e3342296e3fb927e9fe07d0ea71eaedb37f8c4b4c52f9312295c77f396897f","final_host":"www.bing.com","final_path":"/","ms":204,"identity":"ClaudeBot","html":true},"oaisearch":{"status":200,"content_type":"text/html","bytes":78930,"sha256":"ec9cb8cb3f2d621f81717346f88b0f461e89075cfe5938b792b4b6df077abc9d","final_host":"www.bing.com","final_path":"/","ms":216,"identity":"OAI-SearchBot","html":true},"perplexity":{"status":200,"content_type":"text/html","bytes":78942,"sha256":"eeaa8962396ef8894a298dcea65d6a91fc4600ec5c501b23ec5c19a42627181a","final_host":"www.bing.com","final_path":"/","ms":198,"identity":"PerplexityBot","html":true}},"robots":{"status":200,"content_type":"text/plain","bytes":2676,"sha256":"40d2da70c3b443225d5b01cc7b05117944980347637cc95ab285c6e4f5548235","final_host":"www.bing.com","final_path":"/robots.txt","ms":247,"html":false,"text":"User-agent: msnbot-media\r\nDisallow: /\r\nAllow: /th?\r\n\r\nUser-agent: Twitterbot\r\nDisallow:\r\n\r\nUser-agent: *\r\nDisallow: /academic/profile\r\nDisallow: /academic/search\r\nDisallow: /account/\r\nDisallow: /aclick\r\nDisallow: /alink\r\nDisallow: /amp/\r\nAllow: /api/maps/\r\nDisallow: /api/\r\nDisallow: /bfp/search\r\nDisallow: /bing-site-safety\r\nDisallow: /blogs/search/\r\nDisallow: /ck/\r\nDisallow: /copilotsearch?q=\r\nDisallow: /cr$\r\nDisallow: /cr?\r\nDisallow: /entities/search\r\nDisallow: /entityexplore$\r\nDisallow: /entityexplore?\r\nDisallow: /fd/\r\nDisallow: /fun/api/\r\nDisallow: /fun/g/\r\nDisallow: /history\r\nDisallow: /hotel/reviews?\r\nDisallow: /hotels/search\r\nDisallow: /HpImageArchive.aspx\r\nDisallow: /hpm\r\nDisallow: /hpmob\r\nDisallow: /images/async/?\r\nDisallow: /images/async?\r\nDisallow: /images/hosted/?\r\nDisallow: /images/hosted/search?\r\nDisallow: /images/hosted?\r\nDisallow: /images/search%3F\r\nDisallow: /images/search%5C\r\nDisallow: /images/search/?\r\nDisallow: /images/search?\r\nDisallow: /images/searchbyimage\r\nDisallow: /images?\r\nDisallow: /inboxcommerce\r\nDisallow: /local\r\nDisallow: /maps/adsendpoint\r\nDisallow: /merchant/reviews?\r\nDisallow: /notifications/\r\nDisallow: /offers/proxy/dealsserver/api/log\r\nDisallow: /offers/proxy/dealsserver/buy\r\nDisallow: /ping\r\nDisallow: /product/reviews?\r\nDisallow: /profile/history?\r\nDisallow: /proFile/history?\r\nDisallow: /Proxy.ashx\r\nDisallow: /results\r\nDisallow: /rewardsapp/\r\nDisallow: /search\r\nDisallow: /Search\r\nDisallow: /settings\r\nDisallow: /shenghuo\r\nAllow: /shop$\r\nAllow: /shop/\r\nDisallow: /shop/collections\r\nDisallow: /shop/productpage\r\nDisallow: /shop/topics\r\nDisallow: /shop?*\r\nDisallow: /social/search?\r\nDisallow: /spbasic\r\nDisallow: /spresults\r\nDisallow: /static/\r\nDisallow: /th$\r\nDisallow: /th?\r\nDisallow: /translator/?ref=\r\nDisallow: /travel/api\r\nDisallow: /travel/css\r\nDisallow: /travel/flight/flightSearch\r\nDisallow: /travel/flight/flightSearchAction\r\nDisallow: /travel/flight/search/?\r\nDisallow: /travel/flight/search?\r\nDisallow: /travel/flights\r\nDisallow: /travel/flight-search\r\nDisallow: /travel/hotel/hotelMiniSearchRequest\r\nDisallow: /travel/hotel/hotelSearch\r\nDisallow: /travel/hotels/checkout\r\nDisallow: /travel/hotels/detail\r\nDisallow: /travel/hotels/search/?\r\nDisallow: /travel/hotels/search?\r\nDisallow: /travel/hotel-search\r\nDisallow: /travel/scripts\r\nDisallow: /travel/secure\r\nDisallow: /url\r\nDisallow: /videos/browsing\r\nDisallow: /videos/explore\r\nDisallow: /videos/favorites\r\nDisallow: /videos/feed\r\nDisallow: /videos/music\r\nDisallow: /videos/trending\r\nDisallow: /widget/cr\r\nDisallow: /widget/entity/search/?\r\nDisallow: /widget/render\r\nDisallow: /widget/snapshot\r\nDisallow: /work\r\n\r\nSitemap: https://www.bing.com/sitemap.xml","agents":{"GPTBot":true,"OAI-SearchBot":true,"ChatGPT-User":true,"ClaudeBot":true,"Claude-SearchBot":true,"PerplexityBot":true,"Google-Extended":true,"Applebot-Extended":true,"CCBot":true,"Bytespider":true,"Meta-ExternalAgent":true,"Amazonbot":true}},"sitemap":{"status":200,"content_type":"text/xml","bytes":1187,"sha256":"b01f48436b21eb3993fa19ff9244e963061c9be6b2fa159d18de0d8995334c13","final_host":"www.bing.com","final_path":"/sitemap.xml","ms":2,"url":"https://www.bing.com/sitemap.xml","declared":true,"xml":true},"files":{"llms_txt":{"status":404,"content_type":"text/html","bytes":0,"sha256":"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855","final_host":"www.bing.com","final_path":"/llms.txt","ms":138,"path":"/llms.txt","html":true,"json":null,"present":false},"agents_md":{"status":404,"content_type":"text/html","bytes":76771,"sha256":"a25cda532e7b794a9a6939200b27bd43d7458abafa399c0f4ef309e860f79e7f","final_host":"www.bing.com","final_path":"/agents.md","ms":187,"path":"/agents.md","html":true,"json":null,"present":false},"agent_skills":{"status":404,"content_type":"text/html","bytes":0,"sha256":"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855","final_host":"www.bing.com","final_path":"/.well-known/agent-skills/index.json","ms":83,"path":"/.well-known/agent-skills/index.json","html":true,"json":null,"present":false},"mcp_card":{"status":404,"content_type":"text/html","bytes":0,"sha256":"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855","final_host":"www.bing.com","final_path":"/.well-known/mcp/server-card.json","ms":127,"path":"/.well-known/mcp/server-card.json","html":true,"json":null,"present":false},"api_catalog":{"status":404,"content_type":"text/html","bytes":76399,"sha256":"31e05b0fcd38cdcb122b3327b07efd651a2e6762b3d4bae0458731da1c72bc6d","final_host":"www.bing.com","final_path":"/.well-known/api-catalog","ms":118,"path":"/.well-known/api-catalog","html":true,"json":null,"present":false},"a2a_card":{"status":404,"content_type":"text/html","bytes":0,"sha256":"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855","final_host":"www.bing.com","final_path":"/.well-known/agent-card.json","ms":207,"path":"/.well-known/agent-card.json","html":true,"json":null,"present":false}},"rings":{"v":1,"candidates":[],"mcp":[],"a2a":null,"webmcp":{"model_context":false,"bridge":false,"mcp_json":false,"link_rel":false,"any":false},"handshakes":0,"get_path":{"status":404,"content_type":"text/html","ms":153,"error":null},"get_host":{"status":200,"content_type":"text/html","ms":284,"error":null}},"sf":[1,0,0,0,0,0,0,0,1],"derived":{"reachable":true,"home_class":"answers","robots_served":true,"robots_sitemap":true,"content_signal":false,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":[],"blocks_any_ai":false,"files":{"llms_txt":false,"agents_md":false,"agent_skills":false,"mcp_card":false,"api_catalog":false,"a2a_card":false},"any_capability":false,"ai_refused":[],"ai_thin":[],"aipref":0,"aipref_measured":true,"sf":[1,0,0,0,0,0,0,0,1],"rings":{"mcp_declared":false,"mcp_live":false,"mcp_auth":false,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":3,"fh":"www.bing.com","fp_other_host":[]},"derive_v":5,"frame":"top1k","shard":7},{"v":1,"kind":"crawlcheck-census-record","run":43,"domain":"whatsapp.net","rank":33,"measured_at":"2026-11-13T07:56:55.238Z","home":{"browser":{"status":400,"content_type":"text/html","bytes":6278,"sha256":"97470ef4dd8aa158b391b882585e74b0d0e96a11be77bb6b6c1a6a3fb0b82896","final_host":"www.whatsapp.com","final_path":"/","ms":376,"identity":"Mobile browser","html":true},"gptbot":{"status":200,"content_type":"text/html","bytes":266300,"sha256":"2b1ce634ccc31db688afe74aca7d2403b44353b29b4d1a0a0074ede0d888334c","final_host":"www.whatsapp.com","final_path":"/","ms":736,"identity":"GPTBot","html":true,"fp":"031ccaf7ab074b16"},"claudebot":{"status":200,"content_type":"text/html","bytes":266232,"sha256":"2a4250c8bd0b7721ffe3ed08df97ea9751b7a66be7d3a152f5eb183282514782","final_host":"www.whatsapp.com","final_path":"/","ms":538,"identity":"ClaudeBot","html":true,"fp":"031ccaf7ab074b16"},"oaisearch":{"status":200,"content_type":"text/html","bytes":266294,"sha256":"c17f70d0910d8d31d7d90af2ce4185118ae3d4d809c88ee85bbb2e8b40c3b1d7","final_host":"www.whatsapp.com","final_path":"/","ms":525,"identity":"OAI-SearchBot","html":true,"fp":"031ccaf7ab074b16"},"perplexity":{"status":200,"content_type":"text/html","bytes":282048,"sha256":"78b6488ad82ceb6cd413abf054ea79dedb4fa0155b0244ae100c706a8b6f4899","final_host":"www.whatsapp.com","final_path":"/","ms":612,"identity":"PerplexityBot","html":true,"fp":"219a4af7ab0f4b16"}},"robots":{"status":200,"content_type":"text/plain","bytes":1225,"sha256":"6d6febc5de572ecaca22e0169030eb4e4cd41ecaf12af30d64f2cdd9c000d0cb","final_host":"www.whatsapp.com","final_path":"/robots.txt","ms":345,"html":false,"text":"# Notice: Collection of data on Facebook through automated means is\n# prohibited unless you have express written permission from Facebook\n# and may only be conducted for the limited purpose contained in said\n# permission.\n# See: http://www.facebook.com/apps/site_scraping_tos_terms.php\n\nUser-agent: __usage_ai-training\nDisallow: /\n\nUser-agent: Amazonbot\nDisallow: /\n\nUser-agent: Applebot-Extended\nDisallow: /\n\nUser-agent: Brightbot\nDisallow: /\n\nUser-agent: ClaudeBot\nDisallow: /\n\nUser-agent: Google-Extended\nDisallow: /\n\nUser-agent: GPTBot\nDisallow: /\n\nUser-agent: PerplexityBot\nDisallow: /\n\nUser-agent: PetalBot\nDisallow: /\n\nUser-agent: Scrapy\nDisallow: /\n\nUser-agent: uptimerobot\nDisallow: /\n\nUser-agent: viberbot\nDisallow: /\n\nUser-agent: YaK\nDisallow: /\n\nUser-agent: Yandex\nDisallow: /\n\nUser-agent: Yeti\nDisallow: /\n\nUser-agent: facebookexternalhit\nAllow: *\n\nUser-agent: meta-externalads\nAllow: *\n\nUser-agent: *\nDisallow: /*cursor=\nDisallow: /*fb_comment_id=\nDisallow: /ajax/\nDisallow: /tealium/\nDisallow: /intern/\nDisallow: /internal/\nDisallow: /login/\nDisallow: /oidc/callback/\nDisallow: /*.php\nDisallow: /android/third_party\nDisallow: /settings\n\nSitemap: https://www.whatsapp.com/sitemap/www_whatsapp_com_sitemap.xml.gz","agents":{"GPTBot":false,"OAI-SearchBot":true,"ChatGPT-User":true,"ClaudeBot":false,"Claude-SearchBot":true,"PerplexityBot":false,"Google-Extended":false,"Applebot-Extended":false,"CCBot":true,"Bytespider":true,"Meta-ExternalAgent":true,"Amazonbot":false}},"sitemap":null,"files":null,"derived":{"reachable":false,"home_class":"refused_us","robots_served":false,"robots_sitemap":false,"content_signal":false,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":[],"blocks_any_ai":false,"files":{"llms_txt":false,"agents_md":false,"agent_skills":false,"mcp_card":false,"api_catalog":false,"a2a_card":false},"any_capability":false,"ai_refused":[],"ai_thin":[],"rings":{"mcp_declared":false,"mcp_live":false,"mcp_auth":false,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":3,"fh":"www.whatsapp.com","fp_other_host":[]},"derive_v":5,"frame":"top1k","shard":0},{"v":1,"kind":"crawlcheck-census-record","run":43,"domain":"apple-dns.net","rank":34,"measured_at":"2026-11-13T07:57:55.549Z","home":{"browser":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"apple-dns.net","final_path":"/","ms":42,"identity":"Mobile browser","html":false},"gptbot":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"apple-dns.net","final_path":"/","ms":8,"identity":"GPTBot","html":false},"claudebot":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"apple-dns.net","final_path":"/","ms":0,"identity":"ClaudeBot","html":false},"oaisearch":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"apple-dns.net","final_path":"/","ms":0,"identity":"OAI-SearchBot","html":false},"perplexity":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"apple-dns.net","final_path":"/","ms":0,"identity":"PerplexityBot","html":false}},"robots":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"apple-dns.net","final_path":"/robots.txt","ms":0,"html":false,"text":null,"agents":null},"sitemap":null,"files":null,"derived":{"reachable":false,"home_class":"no_site","robots_served":false,"robots_sitemap":false,"content_signal":false,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":[],"blocks_any_ai":false,"files":{"llms_txt":false,"agents_md":false,"agent_skills":false,"mcp_card":false,"api_catalog":false,"a2a_card":false},"any_capability":false,"ai_refused":[],"ai_thin":[],"rings":{"mcp_declared":false,"mcp_live":false,"mcp_auth":false,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":3,"fh":"apple-dns.net","fp_other_host":[]},"derive_v":5,"frame":"top1k","shard":1},{"v":1,"kind":"crawlcheck-census-record","run":43,"domain":"aaplimg.com","rank":35,"measured_at":"2026-11-13T07:58:58.649Z","home":{"browser":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"aaplimg.com","final_path":"/","ms":6,"identity":"Mobile browser","html":false},"gptbot":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"aaplimg.com","final_path":"/","ms":1,"identity":"GPTBot","html":false},"claudebot":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"aaplimg.com","final_path":"/","ms":1,"identity":"ClaudeBot","html":false},"oaisearch":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"aaplimg.com","final_path":"/","ms":1,"identity":"OAI-SearchBot","html":false},"perplexity":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"aaplimg.com","final_path":"/","ms":1,"identity":"PerplexityBot","html":false}},"robots":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"aaplimg.com","final_path":"/robots.txt","ms":0,"html":false,"text":null,"agents":null},"sitemap":null,"files":null,"derived":{"reachable":false,"home_class":"no_site","robots_served":false,"robots_sitemap":false,"content_signal":false,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":[],"blocks_any_ai":false,"files":{"llms_txt":false,"agents_md":false,"agent_skills":false,"mcp_card":false,"api_catalog":false,"a2a_card":false},"any_capability":false,"ai_refused":[],"ai_thin":[],"rings":{"mcp_declared":false,"mcp_live":false,"mcp_auth":false,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":3,"fh":"aaplimg.com","fp_other_host":[]},"derive_v":5,"frame":"top1k","shard":2},{"v":1,"kind":"crawlcheck-census-record","run":43,"domain":"fastly.net","rank":36,"measured_at":"2026-11-13T07:59:19.049Z","home":{"browser":{"status":200,"content_type":"text/html","bytes":227467,"sha256":"3ffd1dcfeb01e8ab7683e15289d03c081583714f73a3fd6adcfc65995c2a749d","final_host":"www.fastly.com","final_path":"/","ms":46,"identity":"Mobile browser","html":true,"fp":"d749449900b33e78"},"gptbot":{"status":200,"content_type":"text/html","bytes":227467,"sha256":"3ffd1dcfeb01e8ab7683e15289d03c081583714f73a3fd6adcfc65995c2a749d","final_host":"www.fastly.com","final_path":"/","ms":33,"identity":"GPTBot","html":true,"fp":"d749449900b33e78"},"claudebot":{"status":200,"content_type":"text/html","bytes":227467,"sha256":"3ffd1dcfeb01e8ab7683e15289d03c081583714f73a3fd6adcfc65995c2a749d","final_host":"www.fastly.com","final_path":"/","ms":18,"identity":"ClaudeBot","html":true,"fp":"d749449900b33e78"},"oaisearch":{"status":200,"content_type":"text/html","bytes":227467,"sha256":"3ffd1dcfeb01e8ab7683e15289d03c081583714f73a3fd6adcfc65995c2a749d","final_host":"www.fastly.com","final_path":"/","ms":31,"identity":"OAI-SearchBot","html":true,"fp":"d749449900b33e78"},"perplexity":{"status":200,"content_type":"text/html","bytes":227467,"sha256":"3ffd1dcfeb01e8ab7683e15289d03c081583714f73a3fd6adcfc65995c2a749d","final_host":"www.fastly.com","final_path":"/","ms":15,"identity":"PerplexityBot","html":true,"fp":"d749449900b33e78"}},"robots":{"status":200,"content_type":"text/plain","bytes":398,"sha256":"23afd0d2868ab27736e15dd91bfa2b061fff83da62d60d131f335d21311dc6f3","final_host":"www.fastly.com","final_path":"/robots.txt","ms":83,"html":false,"text":"User-agent: *\nDisallow: /ja/*\nDisallow: /labs/*\nDisallow: /production-2019/*\nDisallow: /cassets/*\nDisallow: /resources?*\nDisallow: /altitude/2020\nDisallow: /astro-header/*\nDisallow: /*/astro-header/*\nDisallow: /feature-o11y/*\nDisallow: /*/feature-o11y/*\nDisallow: /learning/fastly-research-methodology-glossary\nSitemap: https://www.fastly.com/sitemap/sitemap-index.xml\nHost: https://www.fastly.com\n","agents":{"GPTBot":true,"OAI-SearchBot":true,"ChatGPT-User":true,"ClaudeBot":true,"Claude-SearchBot":true,"PerplexityBot":true,"Google-Extended":true,"Applebot-Extended":true,"CCBot":true,"Bytespider":true,"Meta-ExternalAgent":true,"Amazonbot":true}},"sitemap":{"status":200,"content_type":"application/xml","bytes":380,"sha256":"d4951d59091004f4b2646cfdce10fcca2264b3a7a4c3e6737af619655d916450","final_host":"www.fastly.com","final_path":"/sitemap/sitemap-index.xml","ms":5,"url":"https://www.fastly.com/sitemap/sitemap-index.xml","declared":true,"xml":true},"files":{"llms_txt":{"status":200,"content_type":"text/plain","bytes":16618,"sha256":"7d44aa6ffeddff002cd037eae424224165bc5c5c669c58e9c2e7497d58e2e028","final_host":"www.fastly.com","final_path":"/llms.txt","ms":34,"path":"/llms.txt","html":false,"json":null,"present":true},"agents_md":{"status":404,"content_type":"text/html","bytes":108841,"sha256":"acd7c0ba3773d6c6d3d82c39273e1f589c8c0a13e5d98c02d0fb617d9a24e8c8","final_host":"www.fastly.com","final_path":"/agents.md","ms":310,"path":"/agents.md","html":true,"json":null,"present":false},"agent_skills":{"status":404,"content_type":"text/html","bytes":108841,"sha256":"acd7c0ba3773d6c6d3d82c39273e1f589c8c0a13e5d98c02d0fb617d9a24e8c8","final_host":"www.fastly.com","final_path":"/.well-known/agent-skills/index.json","ms":310,"path":"/.well-known/agent-skills/index.json","html":true,"json":null,"present":false},"mcp_card":{"status":404,"content_type":"text/html","bytes":108841,"sha256":"acd7c0ba3773d6c6d3d82c39273e1f589c8c0a13e5d98c02d0fb617d9a24e8c8","final_host":"www.fastly.com","final_path":"/.well-known/mcp/server-card.json","ms":582,"path":"/.well-known/mcp/server-card.json","html":true,"json":null,"present":false},"api_catalog":{"status":404,"content_type":"text/html","bytes":108841,"sha256":"acd7c0ba3773d6c6d3d82c39273e1f589c8c0a13e5d98c02d0fb617d9a24e8c8","final_host":"www.fastly.com","final_path":"/.well-known/api-catalog","ms":354,"path":"/.well-known/api-catalog","html":true,"json":null,"present":false},"a2a_card":{"status":404,"content_type":"text/html","bytes":108841,"sha256":"acd7c0ba3773d6c6d3d82c39273e1f589c8c0a13e5d98c02d0fb617d9a24e8c8","final_host":"www.fastly.com","final_path":"/.well-known/agent-card.json","ms":297,"path":"/.well-known/agent-card.json","html":true,"json":null,"present":false}},"rings":{"v":1,"candidates":[],"mcp":[],"a2a":null,"webmcp":{"model_context":false,"bridge":false,"mcp_json":false,"link_rel":false,"any":false},"handshakes":0,"get_path":{"status":404,"content_type":"text/html","ms":321,"error":null},"get_host":{"status":530,"content_type":"text/plain","ms":8,"error":null}},"sf":[4,0,0,0,0,0,0,0,0],"derived":{"reachable":true,"home_class":"answers","robots_served":true,"robots_sitemap":true,"content_signal":false,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":[],"blocks_any_ai":false,"files":{"llms_txt":true,"agents_md":false,"agent_skills":false,"mcp_card":false,"api_catalog":false,"a2a_card":false},"any_capability":false,"ai_refused":[],"ai_thin":[],"aipref":0,"aipref_measured":true,"sf":[4,0,0,0,0,0,0,0,0],"rings":{"mcp_declared":false,"mcp_live":false,"mcp_auth":false,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":3,"fh":"www.fastly.com","fp_other_host":[],"fp":"d749449900b33e78","fp_bits":{"gptbot":0,"claudebot":0,"oaisearch":0,"perplexity":0}},"derive_v":5,"frame":"top1k","shard":3},{"v":1,"kind":"crawlcheck-census-record","run":43,"domain":"googleusercontent.com","rank":37,"measured_at":"2026-11-13T08:05:49.397Z","home":{"browser":{"status":404,"content_type":"text/html","bytes":1561,"sha256":"23926e9185d8d43c02807a838ffb373cc1977726094a4e46807c66ada9dd7660","final_host":"googleusercontent.com","final_path":"/","ms":75,"identity":"Mobile browser","html":true},"gptbot":{"status":404,"content_type":"text/html","bytes":1561,"sha256":"23926e9185d8d43c02807a838ffb373cc1977726094a4e46807c66ada9dd7660","final_host":"googleusercontent.com","final_path":"/","ms":72,"identity":"GPTBot","html":true},"claudebot":{"status":404,"content_type":"text/html","bytes":1561,"sha256":"23926e9185d8d43c02807a838ffb373cc1977726094a4e46807c66ada9dd7660","final_host":"googleusercontent.com","final_path":"/","ms":71,"identity":"ClaudeBot","html":true},"oaisearch":{"status":404,"content_type":"text/html","bytes":1561,"sha256":"23926e9185d8d43c02807a838ffb373cc1977726094a4e46807c66ada9dd7660","final_host":"googleusercontent.com","final_path":"/","ms":65,"identity":"OAI-SearchBot","html":true},"perplexity":{"status":404,"content_type":"text/html","bytes":1561,"sha256":"23926e9185d8d43c02807a838ffb373cc1977726094a4e46807c66ada9dd7660","final_host":"googleusercontent.com","final_path":"/","ms":28,"identity":"PerplexityBot","html":true}},"robots":{"status":404,"content_type":"text/html","bytes":1571,"sha256":"6adee83a3eba87d30fdca3997d27097697f2767805f222a0f78c3f04392004d2","final_host":"googleusercontent.com","final_path":"/robots.txt","ms":81,"html":true,"text":null,"agents":null},"sitemap":null,"files":null,"derived":{"reachable":false,"home_class":"other","robots_served":false,"robots_sitemap":false,"content_signal":false,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":[],"blocks_any_ai":false,"files":{"llms_txt":false,"agents_md":false,"agent_skills":false,"mcp_card":false,"api_catalog":false,"a2a_card":false},"any_capability":false,"ai_refused":[],"ai_thin":[],"rings":{"mcp_declared":false,"mcp_live":false,"mcp_auth":false,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":3,"fh":"googleusercontent.com","fp_other_host":[]},"derive_v":5,"frame":"top1k","shard":4},{"v":1,"kind":"crawlcheck-census-record","run":43,"domain":"doubleclick.net","rank":38,"measured_at":"2026-11-13T08:07:00.657Z","home":{"browser":{"status":200,"content_type":"text/html","bytes":124906,"sha256":"06b68f56fba9eb2fdc8353aea5c9d5f4a250ed6754bcd18aae30765c7e5267a8","final_host":"marketingplatform.google.com","final_path":"/about/enterprise","ms":92,"identity":"Mobile browser","html":true,"fp":"a65a9187a0c6597f"},"gptbot":{"status":200,"content_type":"text/html","bytes":124906,"sha256":"4b4f92c618187c54fcbc2cdc69999883ea8bde643377fb880f13a55e76986d95","final_host":"marketingplatform.google.com","final_path":"/about/enterprise","ms":55,"identity":"GPTBot","html":true,"fp":"a65a9187a0c6597f"},"claudebot":{"status":200,"content_type":"text/html","bytes":124906,"sha256":"d5311bd63af5876886223152ddb4ebb82c79c615cac2e64201f2b6ecdd939fe9","final_host":"marketingplatform.google.com","final_path":"/about/enterprise","ms":55,"identity":"ClaudeBot","html":true,"fp":"a65a9187a0c6597f"},"oaisearch":{"status":200,"content_type":"text/html","bytes":124906,"sha256":"974043989d5412d97acb0944744e7617d35c5983fcf7030d848fbe3c34abb3d9","final_host":"marketingplatform.google.com","final_path":"/about/enterprise","ms":55,"identity":"OAI-SearchBot","html":true,"fp":"a65a9187a0c6597f"},"perplexity":{"status":200,"content_type":"text/html","bytes":124906,"sha256":"2404d5614ac95467f7dd569e22b0542c8419ebbb02b13ca131978234c231d5d7","final_host":"marketingplatform.google.com","final_path":"/about/enterprise","ms":55,"identity":"PerplexityBot","html":true,"fp":"a65a9187a0c6597f"}},"robots":{"status":200,"content_type":"text/plain","bytes":6681,"sha256":"e8bf1ce79810901a9fef98deac93aaa532b1be5091ccd19f944dc8d86028a5c3","final_host":"doubleclick.net","final_path":"/robots.txt","ms":40,"html":false,"text":"User-agent: *\nUser-agent: Yandex\nDisallow: /search\nAllow: /search/about\nAllow: /search/howsearchworks\nDisallow: /sdch\nDisallow: /groups\nDisallow: /index.html?\nDisallow: /?\nDisallow: /goto?\nAllow: /?hl=\nDisallow: /?hl=*&\nAllow: /?hl=*&gws_rd=ssl$\nDisallow: /?hl=*&*&gws_rd=ssl\nAllow: /?gws_rd=ssl$\nAllow: /?pt1=true$\nDisallow: /imgres\nDisallow: /u/\nDisallow: /setprefs\nDisallow: /m?\nDisallow: /m/\nAllow:    /m/finance\nDisallow: /wml?\nDisallow: /wml/?\nDisallow: /wml/search?\nDisallow: /xhtml?\nDisallow: /xhtml/?\nDisallow: /xhtml/search?\nDisallow: /xml?\nDisallow: /imode?\nDisallow: /imode/?\nDisallow: /imode/search?\nDisallow: /jsky?\nDisallow: /jsky/?\nDisallow: /jsky/search?\nDisallow: /pda?\nDisallow: /pda/?\nDisallow: /pda/search?\nDisallow: /local?\nDisallow: /local_url\nDisallow: /products?\nDisallow: /product_\nDisallow: /products_\nDisallow: /products;\nDisallow: /print\nDisallow: /books/\nDisallow: /bkshp?*dq=\nDisallow: /bkshp?*q=\nDisallow: /books?*dq=\nDisallow: /books?*q=\nDisallow: /books?*qtid=\nDisallow: /books?*output=\nDisallow: /books?*pg=\nDisallow: /books?*jtp=\nDisallow: /books?*jscmd=\nDisallow: /books?*buy=\nDisallow: /books?*zoom=\nAllow: /books/about\nAllow: /books?*zoom=1\nAllow: /books?*zoom=5\nAllow: /books/content?*zoom=1\nAllow: /books/content?*zoom=5\nDisallow: /patents?\nDisallow: /patents/download/\nDisallow: /patents/pdf/\nDisallow: /patents/related/\nDisallow: /scholar\nDisallow: /citations?\nAllow: /citations?user=\nAllow: /citations?view_op=new_profile\nAllow: /citations?view_op=top_venues\nAllow: /scholar_share\nDisallow: /s?\nDisallow: /maps?\nAllow: /maps?daddr=\nAllow: /maps?entry=wc\nAllow: /maps?f=\nAllow: /maps?hl=\nAllow: /maps?q=\nAllow: /maps?saddr=\nAllow: /maps?sid=\nAllow: /maps?*output=classic\nAllow: /maps?*file=\nDisallow: /mapslt?\nDisallow: /maphp?\nDisallow: /maps/\nAllow: /maps/$\nAllow: /maps/@\nAllow: /maps/?daddr=\nAllow: /maps/?entry=wc\nAllow: /maps/?f=\nAllow: /maps/?hl=\nAllow: /maps/?q=\nAllow: /maps/?saddr=\nAllow: /maps/?sid=\nAllow: /maps/api/staticmap?\nAllow: /maps/search/\nAllow: /maps/sitemap.xml\nAllow: /maps/sitemaps/\nAllow: /maps/dir/\nAllow: /maps/d/\nAllow: /maps/reserve\nAllow: /maps/about\nAllow: /maps/contrib/\nAllow: /maps/match\nAllow: /maps/ms?\nAllow: /maps/place/\nAllow: /maps/_/\nAllow: /search?*tbm=map\nAllow: /maps/vt?\nAllow: /maps/preview\nDisallow: /maps/api/js/\nAllow: /maps/api/js\nDisallow: /mld?\nDisallow: /staticmap?\nDisallow: /help/maps/streetview/partners/welcome/\nDisallow: /help/maps/indoormaps/partners/\nDisallow: /lochp?\nDisallow: /ie?\nDisallow: /uds/\nDisallow: /transit?\nDisallow: /trends?\nDisallow: /trends/music?\nDisallow: /trends/hottrends?\nDisallow: /trends/viz?\nDisallow: /trends/embed.js?\nDisallow: /trends/fetchComponent?\nDisallow: /trends/beta\nDisallow: /trends/topics\nDisallow: /trends/explore?\nDisallow: /trends/api\nDisallow: /musica\nDisallow: /musicl\nDisallow: /musics\nDisallow: /urchin_test/\nDisallow: /movies?\nDisallow: /wapsearch?\nDisallow: /reviews/search?\nDisallow: /cbk\nDisallow: /profiles/me\nDisallow: /s2/profiles/me\nAllow: /s2/profiles\nAllow: /s2/oz\nAllow: /s2/photos\nAllow: /s2/search/social\nAllow: /s2/static\nDisallow: /s2\nDisallow: /transconsole/portal/\nDisallow: /aclk\nDisallow: /tbproxy/\nDisallow: /support/forum/search?\nDisallow: /reviews/polls/\nDisallow: /hosted/images/\nDisallow: /accounts/ClientLogin\nDisallow: /accounts/ClientAuth\nDisallow: /accounts/o8\nAllow: /accounts/o8/id\nDisallow: /quality_form?\nDisallow: /labs/popgadget/search\nDisallow: /compressiontest/\nDisallow: /analytics/feeds/\nDisallow: /analytics/partners/comments/\nDisallow: /analytics/portal/\nDisallow: /analytics/uploads/\nAllow: /alerts/manage\nAllow: /alerts/remove\nDisallow: /alerts/\nAllow: /alerts/$\nDisallow: /phone/compare/?\nDisallow: /travel/clk\nDisallow: /travel/lodging/clk\nDisallow: /travel/lodging/imp\nDisallow: /travel/entity\nDisallow: /travel/search\nDisallow: /travel/flights/booking\nDisallow: /travel/flights/s/\nDisallow: /travel/flights/search\nDisallow: /travel/hotels/stories\nDisallow: /travel/hotels/*/stories\nDisallow: /travel/story\nDisallow: /hotelfinder/rpc\nDisallow: /hotels/rpc\nDisallow: /evaluation/\nDisallow: /forms/perks/\nDisallow: /shopping/suppliers/search\nDisallow: /edu/cs4hs/\nDisallow: /trustedstores/s/\nDisallow: /trustedstores/tm2\nDisallow: /trustedstores/verify\nDisallow: /shopping?\nDisallow: /shopping/product/\nDisallow: /shopping/seller\nDisallow: /shopping/ratings/account/metrics\nDisallow: /shopping/ratings/merchant/immersivedetails\nDisallow: /shopping/reviewer\nDisallow: /shopping/search\nDisallow: /shopping/deals\nAllow: /shopping?udm=28$\nAllow: /shopping?udm=shop$\nDisallow: /storefront\nDisallow: /storepicker\nDisallow: /about/careers/applications/candidate-prep\nDisallow: /about/careers/applications/connect-with-a-googler\nDisallow: /about/careers/applications/jobs/results?page=\nDisallow: /about/careers/applications/jobs/results/?page=\nDisallow: /about/careers/applications/jobs/results?*&page=\nDisallow: /about/careers/applications/jobs/results/?*&page=\nDisallow: /landing/signout.html\nDisallow: /gallery/\nDisallow: /landing/now/ontap/\nAllow: /maps/reserve\nAllow: /maps/reserve/partners\nDisallow: /maps/reserve/api/\nDisallow: /maps/reserve/search\nDisallow: /maps/reserve/bookings\nDisallow: /maps/reserve/settings\nDisallow: /maps/reserve/manage\nDisallow: /maps/reserve/payment\nDisallow: /maps/reserve/receipt\nDisallow: /maps/reserve/sellersignup\nDisallow: /maps/reserve/feedback\nDisallow: /maps/reserve/terms\nDisallow: /maps/reserve/m/\nDisallow: /maps/reserve/b/\nDisallow: /maps/reserve/partner-dashboard\nDisallow: /local/cars\nDisallow: /local/dealership/\nDisallow: /local/dining/\nDisallow: /local/place/products/\nDisallow: /local/place/reviews/\nDisallow: /local/place/rap/\nDisallow: /local/tab/\nDisallow: /localservices/\nAllow: /localservices/coupon/\nDisallow: /nonprofits/account/\nDisallow: /uviewer\nDisallow: /landing/cmsnext-root/\n\n# AdsBot\nUser-agent: AdsBot-Google\nDisallow: /maps/api/js/\nAllow: /maps/api/js\nDisallow: /maps/api/place/js/\nDisallow: /maps/api/staticmap\nDisallow: /maps/api/streetview\n\n# New user agent groups must also have a user agent reference in the global (*)\n# group. See \"Order of precedence\" section in\n# https://goo.gle/rep#order-of-precedence-for-user-agents\nUser-agent: Yandex\nDisallow: /about/careers/applications/jobs/results\nDisallow: /about/careers/applications-a/jobs/results\n\n# Crawlers of certain social media sites are allowed to access page markup when\n# google.com/imgres* links are shared. To learn more, please contact\n# images-robots-allowlist@google.com.\nUser-agent: facebookexternalhit\nUser-agent: Twitterbot\nAllow: /imgres\nAllow: /search\nDisallow: /groups\nDisallow: /hosted/images/\nDisallow: /m/\n\nSitemap: https://www.google.com/sitemap.xml\n","agents":{"GPTBot":true,"OAI-SearchBot":true,"ChatGPT-User":true,"ClaudeBot":true,"Claude-SearchBot":true,"PerplexityBot":true,"Google-Extended":true,"Applebot-Extended":true,"CCBot":true,"Bytespider":true,"Meta-ExternalAgent":true,"Amazonbot":true}},"sitemap":{"status":200,"content_type":"text/xml","bytes":2047,"sha256":"0bc80feb7f7900da84ecf0cafdc78fb4dc3a9550413b40722eb73c6c02d0f49c","final_host":"www.google.com","final_path":"/sitemap.xml","ms":25,"url":"https://www.google.com/sitemap.xml","declared":true,"xml":true},"files":{"llms_txt":{"status":404,"content_type":"text/html","bytes":1569,"sha256":"47d71ed470dfe1ca13eb2af8fa019b95871a1bd1381a518ed71359e69b9c3cc7","final_host":"doubleclick.net","final_path":"/llms.txt","ms":19,"path":"/llms.txt","html":true,"json":null,"present":false},"agents_md":{"status":404,"content_type":"text/html","bytes":1570,"sha256":"f08e316602488d86581ce85b3076a13c82fc3f97cdac3deba23801fb66ca908d","final_host":"doubleclick.net","final_path":"/agents.md","ms":19,"path":"/agents.md","html":true,"json":null,"present":false},"agent_skills":{"status":404,"content_type":"text/html","bytes":1596,"sha256":"eee9f4f975e761645c6c385ec645439385500d0062bff7170ed09a8835d6f92b","final_host":"doubleclick.net","final_path":"/.well-known/agent-skills/index.json","ms":19,"path":"/.well-known/agent-skills/index.json","html":true,"json":null,"present":false},"mcp_card":{"status":404,"content_type":"text/html","bytes":1593,"sha256":"e5db4a1d5f7596a7d62e54e192ab48bbf195116fa33a2c7e160fdaf6131945a8","final_host":"doubleclick.net","final_path":"/.well-known/mcp/server-card.json","ms":18,"path":"/.well-known/mcp/server-card.json","html":true,"json":null,"present":false},"api_catalog":{"status":404,"content_type":"text/html","bytes":1584,"sha256":"7de25ab2e3ce288cf29d538f80ed48bf0d5759f87b613e29ed345db4675698aa","final_host":"doubleclick.net","final_path":"/.well-known/api-catalog","ms":17,"path":"/.well-known/api-catalog","html":true,"json":null,"present":false},"a2a_card":{"status":404,"content_type":"text/html","bytes":1588,"sha256":"64d2189fbe1531e7fb6ab2e6040a932d87ae1feb811d4781df85ea7ef6b4d651","final_host":"doubleclick.net","final_path":"/.well-known/agent-card.json","ms":18,"path":"/.well-known/agent-card.json","html":true,"json":null,"present":false}},"rings":{"v":1,"candidates":[],"mcp":[],"a2a":null,"webmcp":{"model_context":false,"bridge":false,"mcp_json":false,"link_rel":false,"any":false},"handshakes":0,"get_path":{"status":404,"content_type":"text/html","ms":21,"error":null},"get_host":{"status":530,"content_type":"text/plain","ms":28,"error":null}},"sf":[-1,0,0,1,0,0,3,0,0],"derived":{"reachable":true,"home_class":"answers","robots_served":true,"robots_sitemap":true,"content_signal":false,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":[],"blocks_any_ai":false,"files":{"llms_txt":false,"agents_md":false,"agent_skills":false,"mcp_card":false,"api_catalog":false,"a2a_card":false},"any_capability":false,"ai_refused":[],"ai_thin":[],"aipref":0,"aipref_measured":true,"sf":[-1,0,0,1,0,0,3,0,0],"rings":{"mcp_declared":false,"mcp_live":false,"mcp_auth":false,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":3,"fh":"marketingplatform.google.com","fp_other_host":[],"fp":"a65a9187a0c6597f","fp_bits":{"gptbot":0,"claudebot":0,"oaisearch":0,"perplexity":0}},"derive_v":5,"frame":"top1k","shard":5},{"v":1,"kind":"crawlcheck-census-record","run":43,"domain":"cloud.microsoft","rank":39,"measured_at":"2026-11-13T08:07:15.177Z","home":{"browser":{"status":200,"content_type":"text/html","bytes":304345,"sha256":"d08f0a87ef7d097f08dc29cee97694c987bd912e178bf5627c12171ee1330740","final_host":"m365.cloud.microsoft","final_path":"/","ms":642,"identity":"Mobile browser","html":true,"fp":"ca62646b23ab42a1"},"gptbot":{"status":200,"content_type":"text/html","bytes":331705,"sha256":"7ededbf5cb1313d3212cef199fe597b4a861d7c2c8d3238c4c85cd685199a7f5","final_host":"m365.cloud.microsoft","final_path":"/","ms":197,"identity":"GPTBot","html":true,"fp":"c943656b22fb42a1"},"claudebot":{"status":200,"content_type":"text/html","bytes":331705,"sha256":"7ededbf5cb1313d3212cef199fe597b4a861d7c2c8d3238c4c85cd685199a7f5","final_host":"m365.cloud.microsoft","final_path":"/","ms":474,"identity":"ClaudeBot","html":true,"fp":"c943656b22fb42a1"},"oaisearch":{"status":200,"content_type":"text/html","bytes":331705,"sha256":"7ededbf5cb1313d3212cef199fe597b4a861d7c2c8d3238c4c85cd685199a7f5","final_host":"m365.cloud.microsoft","final_path":"/","ms":191,"identity":"OAI-SearchBot","html":true,"fp":"c943656b22fb42a1"},"perplexity":{"status":200,"content_type":"text/html","bytes":331705,"sha256":"7ededbf5cb1313d3212cef199fe597b4a861d7c2c8d3238c4c85cd685199a7f5","final_host":"m365.cloud.microsoft","final_path":"/","ms":37,"identity":"PerplexityBot","html":true,"fp":"c943656b22fb42a1"}},"robots":{"status":200,"content_type":"text/html","bytes":331705,"sha256":"7ededbf5cb1313d3212cef199fe597b4a861d7c2c8d3238c4c85cd685199a7f5","final_host":"m365.cloud.microsoft","final_path":"/","ms":477,"html":true,"text":null,"agents":null},"sitemap":{"status":200,"content_type":"text/html","bytes":331705,"sha256":"7ededbf5cb1313d3212cef199fe597b4a861d7c2c8d3238c4c85cd685199a7f5","final_host":"m365.cloud.microsoft","final_path":"/","ms":213,"url":"https://cloud.microsoft/sitemap.xml","declared":false,"xml":false},"files":{"llms_txt":{"status":200,"content_type":"text/html","bytes":331705,"sha256":"7ededbf5cb1313d3212cef199fe597b4a861d7c2c8d3238c4c85cd685199a7f5","final_host":"m365.cloud.microsoft","final_path":"/","ms":149,"path":"/llms.txt","html":true,"json":null,"present":false},"agents_md":{"status":200,"content_type":"text/html","bytes":331705,"sha256":"7ededbf5cb1313d3212cef199fe597b4a861d7c2c8d3238c4c85cd685199a7f5","final_host":"m365.cloud.microsoft","final_path":"/","ms":211,"path":"/agents.md","html":true,"json":null,"present":false},"agent_skills":{"status":200,"content_type":"text/html","bytes":331705,"sha256":"7ededbf5cb1313d3212cef199fe597b4a861d7c2c8d3238c4c85cd685199a7f5","final_host":"m365.cloud.microsoft","final_path":"/","ms":202,"path":"/.well-known/agent-skills/index.json","html":true,"json":null,"present":false},"mcp_card":{"status":200,"content_type":"text/html","bytes":331705,"sha256":"7ededbf5cb1313d3212cef199fe597b4a861d7c2c8d3238c4c85cd685199a7f5","final_host":"m365.cloud.microsoft","final_path":"/","ms":192,"path":"/.well-known/mcp/server-card.json","html":true,"json":null,"present":false},"api_catalog":{"status":200,"content_type":"text/html","bytes":331705,"sha256":"7ededbf5cb1313d3212cef199fe597b4a861d7c2c8d3238c4c85cd685199a7f5","final_host":"m365.cloud.microsoft","final_path":"/","ms":668,"path":"/.well-known/api-catalog","html":true,"json":null,"present":false},"a2a_card":{"status":200,"content_type":"text/html","bytes":331705,"sha256":"7ededbf5cb1313d3212cef199fe597b4a861d7c2c8d3238c4c85cd685199a7f5","final_host":"m365.cloud.microsoft","final_path":"/","ms":216,"path":"/.well-known/agent-card.json","html":true,"json":null,"present":false}},"rings":{"v":1,"candidates":[],"mcp":[],"a2a":null,"webmcp":{"model_context":false,"bridge":false,"mcp_json":false,"link_rel":false,"any":false},"handshakes":0,"get_path":{"status":200,"content_type":"text/html","ms":179,"error":null},"get_host":{"status":530,"content_type":"text/plain","ms":9,"error":null}},"sf":[1,0,0,0,0,0,2,16,1],"derived":{"reachable":true,"home_class":"answers","robots_served":false,"robots_sitemap":false,"content_signal":false,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":[],"blocks_any_ai":false,"files":{"llms_txt":false,"agents_md":false,"agent_skills":false,"mcp_card":false,"api_catalog":false,"a2a_card":false},"any_capability":false,"ai_refused":[],"ai_thin":[],"sf":[1,0,0,0,0,0,2,16,1],"rings":{"mcp_declared":false,"mcp_live":false,"mcp_auth":false,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":3,"fh":"m365.cloud.microsoft","fp_other_host":[],"fp":"ca62646b23ab42a1","fp_bits":{"gptbot":8,"claudebot":8,"oaisearch":8,"perplexity":8}},"derive_v":5,"frame":"top1k","shard":6},{"v":1,"kind":"crawlcheck-census-record","run":43,"domain":"office.net","rank":40,"measured_at":"2026-11-13T08:08:20.375Z","home":{"browser":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"office.net","final_path":"/","ms":18,"identity":"Mobile browser","html":false},"gptbot":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"office.net","final_path":"/","ms":9,"identity":"GPTBot","html":false},"claudebot":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"office.net","final_path":"/","ms":9,"identity":"ClaudeBot","html":false},"oaisearch":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"office.net","final_path":"/","ms":1,"identity":"OAI-SearchBot","html":false},"perplexity":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"office.net","final_path":"/","ms":0,"identity":"PerplexityBot","html":false}},"robots":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"office.net","final_path":"/robots.txt","ms":1,"html":false,"text":null,"agents":null},"sitemap":null,"files":null,"derived":{"reachable":false,"home_class":"no_site","robots_served":false,"robots_sitemap":false,"content_signal":false,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":[],"blocks_any_ai":false,"files":{"llms_txt":false,"agents_md":false,"agent_skills":false,"mcp_card":false,"api_catalog":false,"a2a_card":false},"any_capability":false,"ai_refused":[],"ai_thin":[],"rings":{"mcp_declared":false,"mcp_live":false,"mcp_auth":false,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":3,"fh":"office.net","fp_other_host":[]},"derive_v":5,"frame":"top1k","shard":7},{"v":1,"kind":"crawlcheck-census-record","run":43,"domain":"microsoftonline.com","rank":41,"measured_at":"2026-11-13T07:56:55.238Z","home":{"browser":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"microsoftonline.com","final_path":"/","ms":93,"identity":"Mobile browser","html":false},"gptbot":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"microsoftonline.com","final_path":"/","ms":62,"identity":"GPTBot","html":false},"claudebot":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"microsoftonline.com","final_path":"/","ms":1,"identity":"ClaudeBot","html":false},"oaisearch":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"microsoftonline.com","final_path":"/","ms":2,"identity":"OAI-SearchBot","html":false},"perplexity":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"microsoftonline.com","final_path":"/","ms":1,"identity":"PerplexityBot","html":false}},"robots":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"microsoftonline.com","final_path":"/robots.txt","ms":0,"html":false,"text":null,"agents":null},"sitemap":null,"files":null,"derived":{"reachable":false,"home_class":"no_site","robots_served":false,"robots_sitemap":false,"content_signal":false,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":[],"blocks_any_ai":false,"files":{"llms_txt":false,"agents_md":false,"agent_skills":false,"mcp_card":false,"api_catalog":false,"a2a_card":false},"any_capability":false,"ai_refused":[],"ai_thin":[],"rings":{"mcp_declared":false,"mcp_live":false,"mcp_auth":false,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":3,"fh":"microsoftonline.com","fp_other_host":[]},"derive_v":5,"frame":"top1k","shard":0},{"v":1,"kind":"crawlcheck-census-record","run":43,"domain":"trafficmanager.net","rank":42,"measured_at":"2026-11-13T07:57:55.549Z","home":{"browser":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"trafficmanager.net","final_path":"/","ms":4,"identity":"Mobile browser","html":false},"gptbot":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"trafficmanager.net","final_path":"/","ms":5,"identity":"GPTBot","html":false},"claudebot":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"trafficmanager.net","final_path":"/","ms":1,"identity":"ClaudeBot","html":false},"oaisearch":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"trafficmanager.net","final_path":"/","ms":0,"identity":"OAI-SearchBot","html":false},"perplexity":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"trafficmanager.net","final_path":"/","ms":0,"identity":"PerplexityBot","html":false}},"robots":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"trafficmanager.net","final_path":"/robots.txt","ms":1,"html":false,"text":null,"agents":null},"sitemap":null,"files":null,"derived":{"reachable":false,"home_class":"no_site","robots_served":false,"robots_sitemap":false,"content_signal":false,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":[],"blocks_any_ai":false,"files":{"llms_txt":false,"agents_md":false,"agent_skills":false,"mcp_card":false,"api_catalog":false,"a2a_card":false},"any_capability":false,"ai_refused":[],"ai_thin":[],"rings":{"mcp_declared":false,"mcp_live":false,"mcp_auth":false,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":3,"fh":"trafficmanager.net","fp_other_host":[]},"derive_v":5,"frame":"top1k","shard":1},{"v":1,"kind":"crawlcheck-census-record","run":43,"domain":"sharepoint.com","rank":43,"measured_at":"2026-11-13T07:58:58.649Z","home":{"browser":{"status":200,"content_type":"text/html","bytes":485730,"sha256":"7355aed49e269648692af30f1473d921f968d70f25164a4b48f29d8ca5444f2f","final_host":"www.microsoft.com","final_path":"/en-us/microsoft-365/sharepoint/collaboration","ms":608,"identity":"Mobile browser","html":true,"fp":"8a3f00c9abd92b5b"},"gptbot":{"status":200,"content_type":"text/html","bytes":485730,"sha256":"b708b7e7426534ab592262be01dfcca7ca2e44fd14d0bb94f32a071ead599668","final_host":"www.microsoft.com","final_path":"/en-us/microsoft-365/sharepoint/collaboration","ms":215,"identity":"GPTBot","html":true,"fp":"8a3f00c98bd92b5b"},"claudebot":{"status":200,"content_type":"text/html","bytes":485730,"sha256":"cf02e9796b6b3fa6f601233934c39758b3716aa9d1fc7f51b5b86b04f545dd6d","final_host":"www.microsoft.com","final_path":"/en-us/microsoft-365/sharepoint/collaboration","ms":218,"identity":"ClaudeBot","html":true,"fp":"8a3f00c98bd92b5b"},"oaisearch":{"status":200,"content_type":"text/html","bytes":485730,"sha256":"4cab73167789ff8c977e8de151aa1b8831e25c231ba6039a1174358731660fb4","final_host":"www.microsoft.com","final_path":"/en-us/microsoft-365/sharepoint/collaboration","ms":271,"identity":"OAI-SearchBot","html":true,"fp":"8a3f00c98bd92b5b"},"perplexity":{"status":200,"content_type":"text/html","bytes":485730,"sha256":"5f2e912bb01bcce8fba0ca2a04b129b8305ca4a774b94711c800d5dc564c0187","final_host":"www.microsoft.com","final_path":"/en-us/microsoft-365/sharepoint/collaboration","ms":224,"identity":"PerplexityBot","html":true,"fp":"8a3f00c9abd92b5b"}},"robots":{"status":520,"content_type":"text/plain","bytes":16,"sha256":"b59e493586b4118c563aaa751e04a7f7bc1c9741d241e0c1d3ba9405c323c570","final_host":"www.microsoft.com","final_path":"/en-us/microsoft-365/sharepoint/collaboration","ms":187,"html":false,"text":null,"agents":null},"sitemap":{"status":520,"content_type":"text/plain","bytes":16,"sha256":"b59e493586b4118c563aaa751e04a7f7bc1c9741d241e0c1d3ba9405c323c570","final_host":"www.microsoft.com","final_path":"/en-us/microsoft-365/sharepoint/collaboration","ms":187,"url":"https://sharepoint.com/sitemap.xml","declared":false,"xml":false},"files":{"llms_txt":{"status":520,"content_type":"text/plain","bytes":16,"sha256":"b59e493586b4118c563aaa751e04a7f7bc1c9741d241e0c1d3ba9405c323c570","final_host":"www.microsoft.com","final_path":"/en-us/microsoft-365/sharepoint/collaboration","ms":175,"path":"/llms.txt","html":false,"json":null,"present":false},"agents_md":{"status":520,"content_type":"text/plain","bytes":16,"sha256":"b59e493586b4118c563aaa751e04a7f7bc1c9741d241e0c1d3ba9405c323c570","final_host":"www.microsoft.com","final_path":"/en-us/microsoft-365/sharepoint/collaboration","ms":125,"path":"/agents.md","html":false,"json":null,"present":false},"agent_skills":{"status":400,"content_type":null,"bytes":0,"sha256":"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855","final_host":"sharepoint.com","final_path":"/.well-known/agent-skills/index.json","ms":230,"path":"/.well-known/agent-skills/index.json","html":false,"json":null,"present":false},"mcp_card":{"status":null,"error":"timeout","ms":8003,"path":"/.well-known/mcp/server-card.json","html":null,"json":null,"present":false},"api_catalog":{"status":400,"content_type":null,"bytes":0,"sha256":"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855","final_host":"sharepoint.com","final_path":"/.well-known/api-catalog","ms":3923,"path":"/.well-known/api-catalog","html":false,"json":null,"present":false},"a2a_card":{"status":503,"content_type":"text/html","bytes":983,"sha256":"ae77c99ab3d56f167f501bce349f107d28ecf23f7e0f6d5b9b4a9f5413451fc1","final_host":"sharepoint.com","final_path":"/.well-known/agent-card.json","ms":89,"path":"/.well-known/agent-card.json","html":true,"json":null,"present":false}},"rings":{"v":1,"candidates":[],"mcp":[],"a2a":null,"webmcp":{"model_context":false,"bridge":false,"mcp_json":false,"link_rel":false,"any":false},"handshakes":0,"get_path":{"status":520,"content_type":"text/plain","ms":137,"error":null},"get_host":{"status":200,"content_type":"text/html","ms":2133,"error":null}},"sf":[4,0,0,0,0,0,4,0,0],"derived":{"reachable":true,"home_class":"answers","robots_served":false,"robots_sitemap":false,"content_signal":false,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":[],"blocks_any_ai":false,"files":{"llms_txt":false,"agents_md":false,"agent_skills":false,"mcp_card":false,"api_catalog":false,"a2a_card":false},"any_capability":false,"ai_refused":[],"ai_thin":[],"sf":[4,0,0,0,0,0,4,0,0],"rings":{"mcp_declared":false,"mcp_live":false,"mcp_auth":false,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":3,"fh":"www.microsoft.com","fp_other_host":[],"fp":"8a3f00c9abd92b5b","fp_bits":{"gptbot":1,"claudebot":1,"oaisearch":1,"perplexity":0}},"derive_v":5,"frame":"top1k","shard":2},{"v":1,"kind":"crawlcheck-census-record","run":43,"domain":"netflix.com","rank":44,"measured_at":"2026-11-13T07:59:19.049Z","home":{"browser":{"status":200,"content_type":"text/html","bytes":747405,"sha256":"8c9d266c3f56876a0ec960f036bcc1ab47b5cd68fefdc740e577595b1f5b55d5","final_host":"www.netflix.com","final_path":"/","ms":637,"identity":"Mobile browser","html":true,"fp":"c492080d141e988f"},"gptbot":{"status":200,"content_type":"text/html","bytes":733813,"sha256":"3023a244252cc93e79cfd091e5647fdfb33f28eafd8ca34cb3756044c013e188","final_host":"www.netflix.com","final_path":"/","ms":556,"identity":"GPTBot","html":true,"fp":"449a0805105e988f"},"claudebot":{"status":200,"content_type":"text/html","bytes":734021,"sha256":"c92a72501b606d38d25d652089d14db6357e613d0a2a78d7282289c83c0e303d","final_host":"www.netflix.com","final_path":"/","ms":452,"identity":"ClaudeBot","html":true,"fp":"c492080d141e988f"},"oaisearch":{"status":200,"content_type":"text/html","bytes":733836,"sha256":"a742979e8dee5bb738aeb1eac8748b7fa02b320ad0a62014e69ac640df716323","final_host":"www.netflix.com","final_path":"/","ms":441,"identity":"OAI-SearchBot","html":true,"fp":"449a0805105e988f"},"perplexity":{"status":200,"content_type":"text/html","bytes":733936,"sha256":"d9fc6bd71c07bb6ad07e951f446f5bbf42286221342f1fd706191e7b2d51f6ab","final_host":"www.netflix.com","final_path":"/","ms":518,"identity":"PerplexityBot","html":true,"fp":"449a0805105e988f"}},"robots":{"status":200,"content_type":"text/plain","bytes":3790,"sha256":"1e14fa0284e9757b52b48dde38b390effae68afdd72447088e72f18f49b1aa09","final_host":"www.netflix.com","final_path":"/robots.txt","ms":247,"html":false,"text":"User-agent: *\nDisallow: /\n\nUser-agent: googlebot\nUser-agent: Googlebot-Video\nUser-agent: Applebot\nUser-agent: bingbot\nUser-agent: Baiduspider\nUser-agent: Baiduspider-mobile\nUser-agent: Baiduspider-video\nUser-agent: Baiduspider-image\nUser-agent: NaverBot\nUser-agent: Y!J-WSC\nUser-agent: YJ-WSC\nUser-agent: Yeti\nUser-agent: Yandex\nUser-agent: YandexBot\nUser-agent: YandexMobileBot\nUser-agent: YandexVideo\nUser-agent: YandexWebmaster\nUser-agent: YandexSitelinks\nUser-agent: SeznamBot\nUser-agent: facebookexternalhit\nUser-agent: GPTBot\nUser-agent: ChatGPT-User\nUser-agent: OAI-SearchBot\nUser-agent: Google-Extended\nUser-agent: ClaudeBot\nUser-agent: Claude-User\nUser-agent: Claude-SearchBot\nUser-agent: PerplexityBot\nAllow: /\n\nDisallow: /accountstatus\nDisallow: /AccountStatus\nDisallow: /aui/inbound\nDisallow: /authenticate\nDisallow: /autologin\nDisallow: /clearcookies\nDisallow: /companies\nDisallow: /editpayment\nDisallow: /emailunsubscribe\nDisallow: /error\nDisallow: /eula\nDisallow: /geooverride\nDisallow: /help\nDisallow: /imagelibrary\nDisallow: /learnmorelayer\nDisallow: /learnmorelayertv\nDisallow: /login\nDisallow: /loginhelp\nDisallow: /loginhelp/lookup\nDisallow: /loginhelpsucess\nDisallow: /LoginHelp\nDisallow: /password\nDisallow: /logout\nDisallow: /Logout\nDisallow: /mcd\nDisallow: /modernizr\nDisallow: /n/\nDisallow: /notamember\nDisallow: /notfound\nDisallow: /notices\nDisallow: /nrdapp\nDisallow: /optout\nDisallow: /overviewblockseeother\nDisallow: /popup/codewhatisthis\nDisallow: /popupdetails\nDisallow: /PopupDetails\nDisallow: /popupprivacypolicy\nDisallow: /privacypolicychanges\nDisallow: /registration\nDisallow: /rememberme\nDisallow: /signout\nDisallow: /signurl\nDisallow: /subscriptioncancel\nDisallow: /tastesurvey\nDisallow: /termsofusechanges\nDisallow: /tvsignup\nDisallow: /upcomingevents\nDisallow: /verifyidentity\nDisallow: /whysecure\n\nDisallow: /arabic\nDisallow: /Arabic\nDisallow: /chinese\nDisallow: /Chinese\nDisallow: /korean\nDisallow: /Korean\n\nDisallow: /airtel\nDisallow: /anan\nDisallow: /bouyguestelecom\nDisallow: /britishairways\nDisallow: /brutus\nDisallow: /comhem\nDisallow: /courts\nDisallow: /csl\nDisallow: /elisa\nDisallow: /entertain\nDisallow: /FireTV\nDisallow: /firetv\nDisallow: /freemonth\nDisallow: /kpn\nDisallow: /lg\nDisallow: /maxis\nDisallow: /Maxis\nDisallow: /meo\nDisallow: /Meo\nDisallow: /orangefrance\nDisallow: /Panasonic\nDisallow: /panasonic\nDisallow: /playstation\nDisallow: /proximus\nDisallow: /qantas\nDisallow: /samsung\nDisallow: /Sony\nDisallow: /sony\nDisallow: /talktalk\nDisallow: /tdc\nDisallow: /telenor\nDisallow: /telfort\nDisallow: /tim\nDisallow: /virginaustralia\nDisallow: /vodafone\nDisallow: /vodafonedemobilelaunch\nDisallow: /xboxone\nDisallow: /xfinity\nDisallow: /xs4all\nDisallow: /ziggo\n\nDisallow: /accountaccess\nDisallow: /AccountAccess\nDisallow: /BillingActivity\nDisallow: /browse\nDisallow: /browse/*\nAllow: /browse/genre/*\nDisallow: /CancelPlan\nDisallow: /ChangePlan\nDisallow: /changeplan\nDisallow: /deviceManagement\nDisallow: /DoNotTest\nDisallow: /EditProfiles\nDisallow: /email\nDisallow: /EmailPreferences\nDisallow: /entrytrap\nDisallow: /HdToggle\nDisallow: /LanguagePreferences\nDisallow: /ManageDevices\nDisallow: /ManageProfiles\nDisallow: /MoviesYouveSeen\nDisallow: /NewWatchInstantlyRSS\nDisallow: /NewWatchInstantlyRSS/*\nDisallow: /payment\nDisallow: /Payment\nDisallow: /phonenumber\nDisallow: /pin\nDisallow: /profiles\nDisallow: /profiles/*\nDisallow: /ProfilesGate\nDisallow: /search\nDisallow: /search/*\nDisallow: /viewingactivity\nDisallow: /WiViewingActivity\nDisallow: /yourAccount\nDisallow: /youraccount\nDisallow: /YourAccount\nDisallow: /YourAccountPayment\n\nUser-agent: AdsBot-Google\nUser-agent: Twitterbot\nUser-agent: Adidxbot\nAllow: /\n\nUser-agent: Yahoo Pipes 1.0\nUser-agent: Facebot\nDisallow: /\nAllow: /tudum\n\nSitemap: https://www.netflix.com/sitemap/index\n","agents":{"GPTBot":true,"OAI-SearchBot":true,"ChatGPT-User":true,"ClaudeBot":true,"Claude-SearchBot":true,"PerplexityBot":true,"Google-Extended":true,"Applebot-Extended":true,"CCBot":false,"Bytespider":false,"Meta-ExternalAgent":false,"Amazonbot":false}},"sitemap":{"status":200,"content_type":"text/xml","bytes":2567109,"sha256":"cf3e80e950eb151a67925ffa31dc74901e9b93a2617e0925226a960785d48264","final_host":"www.netflix.com","final_path":"/sitemap/index","ms":136,"url":"https://www.netflix.com/sitemap/index","declared":true,"xml":true},"files":{"llms_txt":{"status":200,"content_type":"text/html","bytes":23392,"sha256":"dc4a555a2e9b16cf849503b2ace16e8f1a7197caf3c1c89d21004552a87f5f20","final_host":"www.netflix.com","final_path":"/notfound","ms":171,"path":"/llms.txt","html":true,"json":null,"present":false},"agents_md":{"status":200,"content_type":"text/html","bytes":23393,"sha256":"f5ff295b65132e50729a5152500f55e6c0ab16e6ac62d14be4705e599f767b34","final_host":"www.netflix.com","final_path":"/notfound","ms":171,"path":"/agents.md","html":true,"json":null,"present":false},"agent_skills":{"status":200,"content_type":"text/html","bytes":23419,"sha256":"94b37232dfd0be7ba97cdc3f40d72e0bc7b1140c6091262b83bd5ed5d6519b86","final_host":"www.netflix.com","final_path":"/notfound","ms":165,"path":"/.well-known/agent-skills/index.json","html":true,"json":null,"present":false},"mcp_card":{"status":200,"content_type":"text/html","bytes":23415,"sha256":"36959e8e6ead26b61611a8df09444c0c1f62d9eb3c2faa96019b4fafefa36442","final_host":"www.netflix.com","final_path":"/notfound","ms":157,"path":"/.well-known/mcp/server-card.json","html":true,"json":null,"present":false},"api_catalog":{"status":200,"content_type":"text/html","bytes":23402,"sha256":"402c8ad2ddfe98e43d44894c3a49dad4dae6d54c906c42c6a9e70139482a91e7","final_host":"www.netflix.com","final_path":"/notfound","ms":141,"path":"/.well-known/api-catalog","html":true,"json":null,"present":false},"a2a_card":{"status":200,"content_type":"text/html","bytes":23406,"sha256":"25f973a98b386457b0dad5b7b3863dd961569e981818b99fbe384425f5dd1f11","final_host":"www.netflix.com","final_path":"/notfound","ms":139,"path":"/.well-known/agent-card.json","html":true,"json":null,"present":false}},"rings":{"v":1,"candidates":[],"mcp":[],"a2a":null,"webmcp":{"model_context":false,"bridge":false,"mcp_json":false,"link_rel":false,"any":false},"handshakes":0,"get_path":{"status":200,"content_type":"text/html","ms":184,"error":null},"get_host":{"status":530,"content_type":"text/plain","ms":42,"error":null}},"sf":[1,0,0,0,0,0,2,0,0],"derived":{"reachable":true,"home_class":"answers","robots_served":true,"robots_sitemap":true,"content_signal":false,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":["CCBot","Bytespider","Meta-ExternalAgent","Amazonbot"],"blocks_any_ai":true,"files":{"llms_txt":false,"agents_md":false,"agent_skills":false,"mcp_card":false,"api_catalog":false,"a2a_card":false},"any_capability":false,"ai_refused":[],"ai_thin":[],"aipref":0,"aipref_measured":true,"sf":[1,0,0,0,0,0,2,0,0],"rings":{"mcp_declared":false,"mcp_live":false,"mcp_auth":false,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":3,"fh":"www.netflix.com","fp_other_host":[],"fp":"c492080d141e988f","fp_bits":{"gptbot":5,"claudebot":0,"oaisearch":5,"perplexity":5}},"derive_v":5,"frame":"top1k","shard":3},{"v":1,"kind":"crawlcheck-census-record","run":43,"domain":"gandi.net","rank":45,"measured_at":"2026-11-13T08:05:49.397Z","home":{"browser":{"status":200,"content_type":"text/html","bytes":189318,"sha256":"4173125bd794cbc777d9c7012f703355908edd0e9aef37e0b41b1eca5318fbfc","final_host":"www.gandi.net","final_path":"/en","ms":1187,"identity":"Mobile browser","html":true,"fp":"323a26a3def3f8d3"},"gptbot":{"status":200,"content_type":"text/html","bytes":189318,"sha256":"4173125bd794cbc777d9c7012f703355908edd0e9aef37e0b41b1eca5318fbfc","final_host":"www.gandi.net","final_path":"/en","ms":1185,"identity":"GPTBot","html":true,"fp":"323a26a3def3f8d3"},"claudebot":{"status":200,"content_type":"text/html","bytes":189318,"sha256":"4173125bd794cbc777d9c7012f703355908edd0e9aef37e0b41b1eca5318fbfc","final_host":"www.gandi.net","final_path":"/en","ms":1173,"identity":"ClaudeBot","html":true,"fp":"323a26a3def3f8d3"},"oaisearch":{"status":200,"content_type":"text/html","bytes":189318,"sha256":"4173125bd794cbc777d9c7012f703355908edd0e9aef37e0b41b1eca5318fbfc","final_host":"www.gandi.net","final_path":"/en","ms":969,"identity":"OAI-SearchBot","html":true,"fp":"323a26a3def3f8d3"},"perplexity":{"status":200,"content_type":"text/html","bytes":189318,"sha256":"4173125bd794cbc777d9c7012f703355908edd0e9aef37e0b41b1eca5318fbfc","final_host":"www.gandi.net","final_path":"/en","ms":1106,"identity":"PerplexityBot","html":true,"fp":"323a26a3def3f8d3"}},"robots":{"status":200,"content_type":"text/plain","bytes":9023,"sha256":"3fd451a6143f6a84977d65af0ae1e26a153466423e8d7e08e672f3d1466e1cdc","final_host":"www.gandi.net","final_path":"/robots.txt","ms":1027,"html":false,"text":"User-agent: *\r\n\r\n\r\n###### Sitemaps ######\r\nSitemap: https://www.gandi.net/sitemap.xml\r\nSitemap: https://news.gandi.net/sitemaps.xml\r\n\r\n###### Disalow ######\r\nDisallow: /*redirect=*\r\nDisallow: /*navigation=*\r\nDisallow: /*client_id=*\r\nDisallow: /*locale=*\r\nDisallow: /*tld=*\r\nDisallow: /*bulk=*\r\nDisallow: /*bu_id=*\r\nDisallow: /*opened=*\r\nDisallow: /*type=*\r\nDisallow: /*template=*\r\nDisallow: /*size=*\r\nDisallow: /*options=*\r\nDisallow: /*error=*\r\nDisallow: /*grid=*\r\nDisallow: /*currency=*\r\nDisallow: /*?c=*\r\nDisallow: /*&c=*\r\nDisallow: /*?sw=*\r\nDisallow: /*&sw=*\r\nDisallow: /*taxes=*\r\nDisallow: /pdf/*\r\nDisallow: /*/sample/*\r\nDisallow: */prices$\r\nDisallow: */rules$\r\nDisallow: */dates$\r\nDisallow: */news$\r\nAllow: */tld/news$\r\nDisallow: *?country=*\r\nDisallow: *?l=*\r\nDisallow: *?lang=*\r\nDisallow: /gcart/*\r\n\r\n\r\nUser-Agent: facebookexternalhit/1.1\r\nDisallow:\r\n\r\nUser-Agent: Facebot\r\nDisallow:\r\n\r\n###### Suppression robots ######\r\nUser-agent: AhrefsBot\r\nDisallow: /\r\nUser-agent: Alexibot\r\nDisallow: /\r\nUser-agent: Aqua_Products\r\nDisallow: /\r\nUser-agent: asterias\r\nDisallow: /\r\nUser-agent: b2w/0.1\r\nDisallow: /\r\nUser-agent: BackDoorBot/1.0\r\nDisallow: /\r\nUser-agent: BacklinkCrawler\r\nDisallow: /\r\nUser-agent: bdbrandprotect\r\nDisallow: /\r\nUser-agent: Black Hole\r\nDisallow: /\r\nUser-agent: BlowFish/1.0\r\nDisallow: /\r\nUser-agent: boitho.com-dc\r\nDisallow: /\r\nUser-agent: Bookmark search tool\r\nDisallow: /\r\nUser-agent: BotALot\r\nDisallow: /\r\nUser-agent: BotRightHere\r\nDisallow: /\r\nUser-agent: BPImageWalker\r\nDisallow: /\r\nUser-agent: BuiltBotTough\r\nDisallow: /\r\nUser-agent: Bullseye/1.0\r\nDisallow: /\r\nUser-agent: BunnySlippers\r\nDisallow: /\r\nUser-agent: CazoodleBot\r\nDisallow: /\r\nUser-agent: Cegbfeieh\r\nDisallow: /\r\nUser-agent: CheeseBot\r\nDisallow: /\r\nUser-agent: CherryPicker\r\nDisallow: /\r\nUser-agent: CherryPickerElite/1.0\r\nDisallow: /\r\nUser-agent: CherryPickerSE/1.0\r\nDisallow: /\r\nUser-agent: ConveraCrawler\r\nDisallow: /\r\nUser-agent: Copernic\r\nDisallow: /\r\nUser-agent: CopyRightCheck\r\nDisallow: /\r\nUser-agent: cosmos\r\nDisallow: /\r\nUser-agent: Crescent\r\nDisallow: /\r\nUser-agent: Crescent Internet ToolPak HTTP OLE Control v.1.0\r\nDisallow: /\r\nUser-agent: discobot\r\nDisallow: /\r\nUser-agent: DittoSpyder\r\nDisallow: /\r\nUser-agent: dotbot\r\nDisallow: /\r\nUser-agent: Download Ninja\r\nDisallow: /\r\nUser-agent: EmailCollector\r\nDisallow: /\r\nUser-agent: EmailSiphon\r\nDisallow: /\r\nUser-agent: EmailWolf\r\nDisallow: /\r\nUser-agent: EroCrawler\r\nDisallow: /\r\nUser-agent: ExtractorPro\r\nDisallow: /\r\nUser-agent: Ezooms\r\nDisallow: /\r\nUser-agent: FairAd Client\r\nDisallow: /\r\nUser-agent: Fasterfox\r\nDisallow: /\r\nUser-agent: Fetch\r\nDisallow: /\r\nUser-agent: findlinks\r\nDisallow: /\r\nUser-agent: Flaming AttackBot\r\nDisallow: /\r\nUser-agent: Foobot\r\nDisallow: /\r\nUser-agent: Gaisbot\r\nDisallow: /\r\nUser-agent: GetRight/4.2\r\nDisallow: /\r\nUser-agent: Gigabot\r\nDisallow: /\r\nUser-agent: grub\r\nDisallow: /\r\nUser-agent: grub-client\r\nDisallow: /\r\nUser-agent: Harvest/1.5\r\nDisallow: /\r\nUser-agent: heritrix\r\nDisallow: /\r\nUser-agent: hloader\r\nDisallow: /\r\nUser-agent: httplib\r\nDisallow: /\r\nUser-agent: HTTrack\r\nDisallow: / \r\nUser-agent: HTTrack 3.0\r\nDisallow: /\r\nUser-agent: HuaweiSymantecSpider\r\nDisallow: /\r\nUser-agent: humanlinks\r\nDisallow: /\r\nUser-agent: ichiro\r\nDisallow: /\r\nUser-agent: InfoNaviRobot\r\nDisallow: /\r\nUser-agent: IRLbot\r\nDisallow: /\r\nUser-agent: Iron33/1.0.2\r\nDisallow: /\r\nUser-agent: JennyBot\r\nDisallow: /\r\nUser-agent: kalooga\r\nDisallow: /\r\nUser-agent: Kenjin Spider\r\nDisallow: /\r\nUser-agent: Keyword Density/0.9\r\nDisallow: /\r\nUser-agent: larbin\r\nDisallow: /\r\nUser-agent: LexiBot\r\nDisallow: /\r\nUser-agent: LexxeBot\r\nDisallow: /\r\nUser-agent: libWeb/clsHTTP\r\nDisallow: /\r\nUser-agent: libwww\r\nDisallow: /\r\nUser-agent: LinkextractorPro\r\nDisallow: /\r\nUser-agent: linko\r\nDisallow: /\r\nUser-agent: LinkScan/8.1a Unix\r\nDisallow: /\r\nUser-agent: LinkWalker\r\nDisallow: /\r\nUser-agent: LNSpiderguy\r\nDisallow: /\r\nUser-agent: lwp-trivial\r\nDisallow: /\r\nUser-agent: lwp-trivial/1.34\r\nDisallow: /\r\nUser-agent: Mata Hari\r\nDisallow: /\r\nUser-agent: Microsoft URL Control\r\nDisallow: /\r\nUser-agent: Microsoft URL Control - 5.01.4511\r\nDisallow: /\r\nUser-agent: Microsoft URL Control - 6.00.8169\r\nDisallow: /\r\nUser-agent: Microsoft.URL.Control\r\nDisallow: /\r\nUser-agent: MIIxpc\r\nDisallow: /\r\nUser-agent: MIIxpc/4.2\r\nDisallow: /\r\nUser-agent: Mister PiX\r\nDisallow: /\r\nUser-agent: MJ12bot\r\nDisallow: /\r\nUser-agent: MLBot\r\nDisallow: /\r\nUser-agent: moget\r\nDisallow: /\r\nUser-agent: moget/2.1\r\nDisallow: /\r\nUser-agent: Mozilla/4.0 (compatible; BullsEye; Windows 95)\r\nDisallow: /\r\nUser-agent: MSIECrawler\r\nDisallow: /\r\nUser-agent: NaverBot\r\nDisallow: /\r\nUser-agent: NerdyBot\r\nDisallow: /\r\nUser-agent: Net\\ Vampire\r\nDisallow: /\r\nUser-agent: NetAnts\r\nDisallow: /\r\nUser-agent: NetMechanic\r\nDisallow: /\r\nUser-agent: NICErsPRO\r\nDisallow: /\r\nUser-agent: Nutch\r\nDisallow: /\r\nUser-agent: Ocelli\r\nDisallow: /\r\nUser-agent: Offline Explorer\r\nDisallow: /\r\nUser-agent: Openbot\r\nDisallow: /\r\nUser-agent: Openfind\r\nDisallow: /\r\nUser-agent: Openfind data gatherer\r\nDisallow: /\r\nUser-agent: Oracle Ultra Search\r\nDisallow: /\r\nUser-agent: penthesilea\r\nDisallow: /\r\nUser-agent: PerMan\r\nDisallow: /\r\nUser-agent: PhpDig\r\nDisallow: /\r\nUser-agent: ProPowerBot/2.14\r\nDisallow: /\r\nUser-agent: ProWebWalker\r\nDisallow: /\r\nUser-agent: psbot\r\nDisallow: /\r\nUser-agent: Python-urllib\r\nDisallow: /\r\nUser-agent: QueryN Metasearch\r\nDisallow: /\r\nUser-agent: Radiation Retriever 1.1\r\nDisallow: /\r\nUser-agent: RepoMonkey\r\nDisallow: /\r\nUser-agent: RepoMonkey Bait & Tackle/v1.01\r\nDisallow: /\r\nUser-agent: RMA\r\nDisallow: /\r\nUser-agent: rogerbot\r\nDisallow: /\r\nUser-agent: SCOUTjET\r\nDisallow: /\r\nUser-agent: SearchDaimon.com-dc\r\nDisallow: /\r\nUser-agent: searchpreview\r\nDisallow: /\r\nUser-agent: SEOENGBot\r\nDisallow: /\r\nUser-agent: ShopWiki\r\nDisallow: /\r\nUser-agent: sitecheck.internetseer.com\r\nDisallow: /\r\nUser-agent: SiteSnagger\r\nDisallow: /\r\nUser-agent: Snapbot\r\nDisallow: /\r\nUser-agent: Sogou\\ web\\ spider\r\nDisallow: /\r\nUser-agent: Sosospider\r\nDisallow: /\r\nUser-agent: SpankBot\r\nDisallow: /\r\nUser-agent: spanner\r\nDisallow: /\r\nUser-agent: spbot\r\nDisallow: /\r\nUser-agent: SurveyBot\r\nDisallow: /\r\nUser-agent: suzuran\r\nDisallow: /\r\nUser-agent: Szukacz/1.4\r\nDisallow: /\r\nUser-agent: Teleport\r\nDisallow: /\r\nUser-agent: TeleportPro\r\nDisallow: /\r\nUser-agent: Telesoft\r\nDisallow: /\r\nUser-agent: Templeton\r\nDisallow: /\r\nUser-agent: The Intraformant\r\nDisallow: /\r\nUser-agent: TheNomad\r\nDisallow: /\r\nUser-agent: TightTwatBot\r\nDisallow: /\r\nUser-agent: Titan\r\nDisallow: /\r\nUser-agent: toCrawl/UrlDispatcher\r\nDisallow: /\r\nUser-agent: True_Robot\r\nDisallow: /\r\nUser-agent: True_Robot/1.0\r\nDisallow: /\r\nUser-agent: turingos\r\nDisallow: /\r\nUser-agent: TurnitinBot\r\nDisallow: /\r\nUser-agent: TurnitinBot/1.5\r\nDisallow: /\r\nUser-agent: TwengaBot\r\nDisallow: /\r\nUser-agent: Twiceler\r\nDisallow: /\r\nUser-agent: Updownerbot\r\nDisallow: /\r\nUser-agent: URL Control\r\nDisallow: /\r\nUser-agent: URL_Spider_Pro\r\nDisallow: /\r\nUser-agent: URLy Warning\r\nDisallow: /\r\nUser-agent: VCI\r\nDisallow: /\r\nUser-agent: VCI WebViewer VCI WebViewer Win32\r\nDisallow: /\r\nUser-agent: Web Image Collector\r\nDisallow: /\r\nUser-agent: WebAuto\r\nDisallow: /\r\nUser-agent: WebBandit\r\nDisallow: /\r\nUser-agent: WebBandit/3.50\r\nDisallow: /\r\nUser-agent: WebCapture 2.0\r\nDisallow: /\r\nUser-agent: WebCopier\r\nDisallow: /\r\nUser-agent: WebCopier v.2.2\r\nDisallow: /\r\nUser-agent: WebCopier v3.2a\r\nDisallow: /\r\nUser-agent: WebEnhancer\r\nDisallow: /\r\nUser-agent: WebmasterWorldForumBot\r\nDisallow: /\r\nUser-agent: WebSauger\r\nDisallow: /\r\nUser-agent: Website Quester\r\nDisallow: /\r\nUser-agent: Webster Pro\r\nDisallow: /\r\nUser-agent: WebStripper\r\nDisallow: /\r\nUser-agent: WebZIP\r\nDisallow: /\r\nUser-agent: WebZip/4.0\r\nDisallow: /\r\nUser-agent: WebZIP/4.21\r\nDisallow: /\r\nUser-agent: WebZIP/5.0\r\nDisallow: /\r\nUser-agent: wget\r\nDisallow: /\r\nUser-agent: Wget/1.5.3\r\nDisallow: /\r\nUser-agent: Wget/1.6\r\nDisallow: /\r\nUser-agent: woozweb-monitoring\r\nDisallow: /\r\nUser-agent: woriobot\r\nDisallow: /\r\nUser-agent: wprocketbot\r\nAllow: /\r\nUser-agent: WWW-Collector-E\r\nDisallow: /\r\nUser-agent: Xenu\r\nDisallow: /\r\nUser-agent: Xenu's\r\nDisallow: /\r\nUser-agent: Xenu's Link Sleuth 1.1c\r\nDisallow: /\r\nUser-agent: YACYBIT\r\nDisallow: /\r\nUser-agent: yacybot\r\nDisallow: /\r\nUser-agent: Yandex\r\nDisallow: /\r\nUser-agent: YandexBot\r\nDisallow: /\r\nUser-agent: Yeti\r\nDisallow: /\r\nUser-agent: YodaoBot\r\nDisallow: /\r\nUser-agent: Zealbot\r\nDisallow: /\r\nUser-agent: Zeus\r\nDisallow: /\r\nUser-agent: Zeus 32297 Webster Pro V2.9 Win32\r\nDisallow: /\r\nUser-agent: Zeus Link Scout\r\nDisallow: /\r\nUser-agent: ZyBorg\r\nDisallow: /\r\nUser-agent: Mozilla/4.0 (compatible; BullsEye; Windows 95)\r\nDisallow: /\r\nUser-agent: NerdyBot\r\nDisallow: /\r\nUser-agent: WebAuto\r\nDisallow: /\r\nUser-agent: wprocketbot\r\nDisallow: /\r\nUser-agent: DIIbot\r\nDisallow: /\r\nUser-agent: eCatch\r\nDisallow: /\r\nUser-agent: Jetbot\r\nDisallow: /\r\nUser-agent: Openfind data gathere\r\nDisallow: /\r\nUser-agent: QuepasaCreep\r\nDisallow: /\r\nUser-agent: Roverbot\r\nDisallow: /\r\nUser-agent: Foobot\r\nDisallow: /\r\nUser-agent: RepoMonkey Bait & Tackle/v1.01\r\nDisallow: /\r\nUser-agent: ExtractorPro\r\nDisallow: /","agents":{"GPTBot":true,"OAI-SearchBot":true,"ChatGPT-User":true,"ClaudeBot":true,"Claude-SearchBot":true,"PerplexityBot":true,"Google-Extended":true,"Applebot-Extended":true,"CCBot":true,"Bytespider":true,"Meta-ExternalAgent":true,"Amazonbot":true}},"sitemap":{"status":200,"content_type":"text/xml","bytes":529,"sha256":"20e45cfdab4c73574ebe3829d032be6b389f600aac558cddcc91ecbe5f9681f9","final_host":"www.gandi.net","final_path":"/sitemap.xml","ms":11,"url":"https://www.gandi.net/sitemap.xml","declared":true,"xml":true},"files":{"llms_txt":{"status":404,"content_type":"text/html","bytes":118145,"sha256":"faa3c94bfe16278cb8f380b5f0884f67ce86ac93c2cca2d440addef229e2f37c","final_host":"www.gandi.net","final_path":"/llms.txt","ms":914,"path":"/llms.txt","html":true,"json":null,"present":false},"agents_md":{"status":404,"content_type":"text/html","bytes":118146,"sha256":"9ada5eac863d5aaa6c66916336c794713e0a3254699baa145ed2a48c6b6eb1c8","final_host":"www.gandi.net","final_path":"/agents.md","ms":616,"path":"/agents.md","html":true,"json":null,"present":false},"agent_skills":{"status":404,"content_type":"text/html","bytes":118172,"sha256":"fd8227d5b383a8c4415529a00828e83fdcc6a4117a8b8ca42c02f81c2cd66b69","final_host":"www.gandi.net","final_path":"/.well-known/agent-skills/index.json","ms":1011,"path":"/.well-known/agent-skills/index.json","html":true,"json":null,"present":false},"mcp_card":{"status":404,"content_type":"text/html","bytes":118169,"sha256":"ec4a1c17b758ede2cb13e28d389066a66be29a8500576be27f118a1d3a0c000b","final_host":"www.gandi.net","final_path":"/.well-known/mcp/server-card.json","ms":635,"path":"/.well-known/mcp/server-card.json","html":true,"json":null,"present":false},"api_catalog":{"status":404,"content_type":"text/html","bytes":118160,"sha256":"4546f178591bd0bd621661ccdede7d4ef470df6ebd97d4378313a07ff956fef7","final_host":"www.gandi.net","final_path":"/.well-known/api-catalog","ms":984,"path":"/.well-known/api-catalog","html":true,"json":null,"present":false},"a2a_card":{"status":404,"content_type":"text/html","bytes":118164,"sha256":"49d2380866df75e240748aff8d14675dc85693ebb8c93bd2bd42f5b6076cd883","final_host":"www.gandi.net","final_path":"/.well-known/agent-card.json","ms":908,"path":"/.well-known/agent-card.json","html":true,"json":null,"present":false}},"rings":{"v":1,"candidates":[],"mcp":[],"a2a":null,"webmcp":{"model_context":false,"bridge":false,"mcp_json":false,"link_rel":false,"any":false},"handshakes":0,"get_path":{"status":200,"content_type":"text/html","ms":325,"error":null},"get_host":{"status":530,"content_type":"text/plain","ms":369,"error":null}},"sf":[-1,0,0,0,0,0,0,0,0],"derived":{"reachable":true,"home_class":"answers","robots_served":true,"robots_sitemap":true,"content_signal":false,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":[],"blocks_any_ai":false,"files":{"llms_txt":false,"agents_md":false,"agent_skills":false,"mcp_card":false,"api_catalog":false,"a2a_card":false},"any_capability":false,"ai_refused":[],"ai_thin":[],"aipref":0,"aipref_measured":true,"sf":[-1,0,0,0,0,0,0,0,0],"rings":{"mcp_declared":false,"mcp_live":false,"mcp_auth":false,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":3,"fh":"www.gandi.net","fp_other_host":[],"fp":"323a26a3def3f8d3","fp_bits":{"gptbot":0,"claudebot":0,"oaisearch":0,"perplexity":0}},"derive_v":5,"frame":"top1k","shard":4},{"v":1,"kind":"crawlcheck-census-record","run":43,"domain":"cloudfront.net","rank":46,"measured_at":"2026-11-13T08:07:00.657Z","home":{"browser":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"cloudfront.net","final_path":"/","ms":33,"identity":"Mobile browser","html":false},"gptbot":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"cloudfront.net","final_path":"/","ms":32,"identity":"GPTBot","html":false},"claudebot":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"cloudfront.net","final_path":"/","ms":32,"identity":"ClaudeBot","html":false},"oaisearch":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"cloudfront.net","final_path":"/","ms":32,"identity":"OAI-SearchBot","html":false},"perplexity":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"cloudfront.net","final_path":"/","ms":1,"identity":"PerplexityBot","html":false}},"robots":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"cloudfront.net","final_path":"/robots.txt","ms":1,"html":false,"text":null,"agents":null},"sitemap":null,"files":null,"derived":{"reachable":false,"home_class":"no_site","robots_served":false,"robots_sitemap":false,"content_signal":false,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":[],"blocks_any_ai":false,"files":{"llms_txt":false,"agents_md":false,"agent_skills":false,"mcp_card":false,"api_catalog":false,"a2a_card":false},"any_capability":false,"ai_refused":[],"ai_thin":[],"rings":{"mcp_declared":false,"mcp_live":false,"mcp_auth":false,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":3,"fh":"cloudfront.net","fp_other_host":[]},"derive_v":5,"frame":"top1k","shard":5},{"v":1,"kind":"crawlcheck-census-record","run":43,"domain":"youtu.be","rank":47,"measured_at":"2026-11-13T08:07:15.177Z","home":{"browser":{"status":200,"content_type":"text/html","bytes":354114,"sha256":"2310ff091cc0f61a22a6343082851cca41726909d46fa99af44db8ae4586fb03","final_host":"m.youtube.com","final_path":"/","ms":515,"identity":"Mobile browser","html":true},"gptbot":{"status":200,"content_type":"text/html","bytes":970659,"sha256":"e6c879e6f996aa2e994904b6601756afaae740ac09ce2a3b37c9ab21fcb2af76","final_host":"www.youtube.com","final_path":"/","ms":391,"identity":"GPTBot","html":true},"claudebot":{"status":200,"content_type":"text/html","bytes":871258,"sha256":"ce4762c47becb81448378ecd4b96d408b9541006c79e9bb83afd41d986d8d85d","final_host":"www.youtube.com","final_path":"/","ms":371,"identity":"ClaudeBot","html":true},"oaisearch":{"status":200,"content_type":"text/html","bytes":870898,"sha256":"32490bae05fdc0b54f1879eaf40da083869b8dd12074ef4fc505df8a43abdde3","final_host":"www.youtube.com","final_path":"/","ms":376,"identity":"OAI-SearchBot","html":true},"perplexity":{"status":200,"content_type":"text/html","bytes":974096,"sha256":"d62cc650d34d8716dcc10e45f95f40c2eae9aac3cdec1b3ca11f91fd044a39e5","final_host":"www.youtube.com","final_path":"/","ms":375,"identity":"PerplexityBot","html":true}},"robots":{"status":200,"content_type":"text/plain","bytes":511,"sha256":"c290485c8002c51c24c95765948dbcec9ec88c1f181638d245418a36194fd1a0","final_host":"youtu.be","final_path":"/robots.txt","ms":38,"html":false,"text":"# robots.txt file for youtu.be\n\nUser-agent: *\nDisallow: /api/\nDisallow: /comment\nDisallow: /feeds/videos.xml\nDisallow: /get_video\nDisallow: /get_video_info\nDisallow: /get_midroll_info\nDisallow: /live_chat\nDisallow: /login\nDisallow: /results\nDisallow: /signup\nDisallow: /t/terms\nDisallow: /timedtext_video\nDisallow: /verify_age\nDisallow: /watch_ajax\nDisallow: /watch_fragments_ajax\nDisallow: /watch_popup\nDisallow: /watch_queue_ajax\nDisallow: /youtubei/\n\nAllow: /apple-app-site-association\nAllow: /.well-known/*\n","agents":{"GPTBot":true,"OAI-SearchBot":true,"ChatGPT-User":true,"ClaudeBot":true,"Claude-SearchBot":true,"PerplexityBot":true,"Google-Extended":true,"Applebot-Extended":true,"CCBot":true,"Bytespider":true,"Meta-ExternalAgent":true,"Amazonbot":true}},"sitemap":{"status":200,"content_type":"text/html","bytes":875841,"sha256":"7c22be9dec5edb2fe9dfc70e193f912b5be36b43e03d95ac59b288d4e4134929","final_host":"www.youtube.com","final_path":"/watch","ms":196,"url":"https://youtu.be/sitemap.xml","declared":false,"xml":false},"files":{"llms_txt":{"status":200,"content_type":"text/html","bytes":874286,"sha256":"6f2ba6dbe97ab297e7d3eb01bd4313a0e3c58a153a768abcb66457b22590212f","final_host":"www.youtube.com","final_path":"/watch","ms":242,"path":"/llms.txt","html":true,"json":null,"present":false},"agents_md":{"status":200,"content_type":"text/html","bytes":870435,"sha256":"3d8e22dd910ca6896f6a1d234a1c9953a0c980154da29ce1c2932fd2cf6982af","final_host":"www.youtube.com","final_path":"/watch","ms":204,"path":"/agents.md","html":true,"json":null,"present":false},"agent_skills":{"status":200,"content_type":"text/html","bytes":871388,"sha256":"b590644e88337ceb05291d744044bc0c1c3be1938ca9024791653468e7fbbb74","final_host":"www.youtube.com","final_path":"/watch","ms":206,"path":"/.well-known/agent-skills/index.json","html":true,"json":null,"present":false},"mcp_card":{"status":200,"content_type":"text/html","bytes":871928,"sha256":"040a288d71b9f2953b9fde69caa5d135ba14b7536dff402f2c96c550ea0acd6c","final_host":"www.youtube.com","final_path":"/watch","ms":212,"path":"/.well-known/mcp/server-card.json","html":true,"json":null,"present":false},"api_catalog":{"status":200,"content_type":"text/html","bytes":871328,"sha256":"a6c9638817d9d3950f18711ac0023c0a8521b61186e239307336cc0ccd07bca0","final_host":"www.youtube.com","final_path":"/watch","ms":146,"path":"/.well-known/api-catalog","html":true,"json":null,"present":false},"a2a_card":{"status":200,"content_type":"text/html","bytes":869532,"sha256":"0c97c426703adf949e1fb4f1580bd155a45ab8db7b6359fa9b291fb85d042fee","final_host":"www.youtube.com","final_path":"/watch","ms":151,"path":"/.well-known/agent-card.json","html":true,"json":null,"present":false}},"rings":{"v":1,"candidates":[],"mcp":[],"a2a":null,"webmcp":{"model_context":false,"bridge":false,"mcp_json":false,"link_rel":false,"any":false},"handshakes":0,"get_path":{"status":200,"content_type":"text/html","ms":240,"error":null},"get_host":{"status":530,"content_type":"text/plain","ms":45,"error":null}},"sf":[1,0,0,0,0,0,2,0,0],"derived":{"reachable":true,"home_class":"answers","robots_served":true,"robots_sitemap":false,"content_signal":false,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":[],"blocks_any_ai":false,"files":{"llms_txt":false,"agents_md":false,"agent_skills":false,"mcp_card":false,"api_catalog":false,"a2a_card":false},"any_capability":false,"ai_refused":[],"ai_thin":[],"aipref":0,"aipref_measured":true,"sf":[1,0,0,0,0,0,2,0,0],"rings":{"mcp_declared":false,"mcp_live":false,"mcp_auth":false,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":3,"fh":"m.youtube.com","fp_other_host":[]},"derive_v":5,"frame":"top1k","shard":6},{"v":1,"kind":"crawlcheck-census-record","run":43,"domain":"x.com","rank":48,"measured_at":"2026-11-13T08:08:20.375Z","home":{"browser":{"status":200,"content_type":"text/html","bytes":46088,"sha256":"ef8360508acba19d92c430cd308e71649088a2341d96e75970c42eddc8fec9b5","final_host":"x.com","final_path":"/","ms":149,"identity":"Mobile browser","html":true,"fp":"8b48d3564ad225ec"},"gptbot":{"status":402,"content_type":"application/json","bytes":55,"sha256":"35e642f0e1bd426b9d8b0ee0284aefe801873dc13c18819fbeeaf2bf47136664","final_host":"x.com","final_path":"/","ms":2,"identity":"GPTBot","html":false},"claudebot":{"status":403,"content_type":null,"bytes":0,"sha256":"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855","final_host":"x.com","final_path":"/","ms":110,"identity":"ClaudeBot","html":null},"oaisearch":{"status":402,"content_type":"application/json","bytes":55,"sha256":"35e642f0e1bd426b9d8b0ee0284aefe801873dc13c18819fbeeaf2bf47136664","final_host":"x.com","final_path":"/","ms":1,"identity":"OAI-SearchBot","html":false},"perplexity":{"status":402,"content_type":"application/json","bytes":55,"sha256":"35e642f0e1bd426b9d8b0ee0284aefe801873dc13c18819fbeeaf2bf47136664","final_host":"x.com","final_path":"/","ms":2,"identity":"PerplexityBot","html":false}},"robots":{"status":200,"content_type":"text/plain","bytes":3039,"sha256":"b54dd4fefb8786b6441dc95ab04db1dbe4c6b9fcb5e7bb60d8381ab6e40989ff","final_host":"x.com","final_path":"/robots.txt","ms":109,"html":false,"text":"# Google / Bing Search Engine Robots\n# ==================================\n# Shared group so Googlebot and Bingbot always get the same rules.\nUser-agent: Googlebot\nUser-agent: Bingbot\n\nAllow: /*?s=\nAllow: /*?t=\nAllow: /*?ref_src=\nAllow: /hashtag/*?src=\nAllow: /search?q=%23\nAllow: /i/api/\n# Same length as the /*/likes and /*/media rules; RFC 9309 breaks the tie in\n# favor of Allow, so /i/api/ fetches stay crawlable.\nAllow: /i/api/*\nDisallow: /*?lang=en-ss\nAllow: /*?lang=\nDisallow: /search/realtime\nDisallow: /search/users\nDisallow: /search/*/grid\n\nDisallow: /*/analytics\nDisallow: /*/followers\nDisallow: /*/following\nDisallow: /*/verified_followers\n\nDisallow: /account/deactivated\nDisallow: /settings/deactivated\n\n# Only `*` and `$` are metacharacters here (RFC 9309); regex classes match\n# literally. /photo is anchored so /status/*/photo/1 media pages stay open.\nDisallow: /*/likes\nDisallow: /*/likes?\nDisallow: /*/retweets\nDisallow: /*/retweets?\nDisallow: /*/media\nDisallow: /*/media?\nDisallow: /*/photo$\nDisallow: /*/photo?\nAllow: /*?\n\nUser-agent: facebookexternalhit\n\nAllow: /*?lang=\nAllow: /*?s=\nAllow: /*&s=\nAllow: /*?t=\nAllow: /*?ref_src=\nAllow: /hashtag/*?src=\nAllow: /search?*cashtagRestId=\nAllow: /i/api/\nAllow: /i/api/*\nDisallow: /search?q=\nDisallow: /search/realtime\nDisallow: /search/users\nDisallow: /search/*/grid\n\nDisallow: /*?\nDisallow: /*/followers\nDisallow: /*/following\nDisallow: /*/verified_followers\n\nDisallow: /account/deactivated\nDisallow: /settings/deactivated\n\nDisallow: /*/likes\nDisallow: /*/likes?\nDisallow: /*/retweets\nDisallow: /*/retweets?\nDisallow: /*/media\nDisallow: /*/media?\nDisallow: /*/photo$\nDisallow: /*/photo?\n\nUser-Agent: Google-Extended\nDisallow: *\n\nUser-Agent: FacebookBot\nDisallow: *\n\nUser-agent: Discordbot\nDisallow: *\n\n# Meta AI / product crawlers (product tokens; versions like /1.1 are ignored\n# under RFC 9309 token matching).\nUser-agent: meta-webindexer\nDisallow: *\n\nUser-agent: meta-externalagent\nDisallow: *\n\nUser-agent: meta-externalads\nDisallow: *\n\nUser-agent: meta-externalfetcher\nDisallow: *\n\n# Shared AI training preference (IETF draft-kelley-aipref-training-preference-signal),\n# honored by Microsoft, Google, OpenAI, and Anthropic. This is a usage policy\n# token, not a crawler: it does not change crawl access, and `User-agent: *`\n# never matches it.\nUser-agent: __usage_ai-training\nDisallow: /\n\n# Every bot that might possibly read and respect this file\n# ========================================================\nUser-agent: *\nDisallow: /\n\n\n# WHAT-4882 - Keep notification-email links (/i/u) out of search results.\n# Named crawlers stay un-blocked on purpose: they must crawl /i/u to see its\n# X-Robots-Tag noindex (the robots.txt Noindex directive died in 2019).\nDisallow: /i/u\n\n# Wait 1 second between successive requests. See ONBOARD-2698 for details.\nCrawl-delay: 1\n\n# Independent of user agent. Links in the sitemap are full URLs using https://\n# and need to match the protocol of the sitemap.\nSitemap: https://x.com/sitemap.xml\nSitemap: https://x.com/sitemaps/finance0\n","agents":{"GPTBot":false,"OAI-SearchBot":false,"ChatGPT-User":false,"ClaudeBot":false,"Claude-SearchBot":false,"PerplexityBot":false,"Google-Extended":false,"Applebot-Extended":false,"CCBot":false,"Bytespider":false,"Meta-ExternalAgent":false,"Amazonbot":false}},"sitemap":{"status":404,"content_type":"text/html","bytes":46084,"sha256":"3c43862a92ba7f8bc05ac8a3fd7bbae447e2f15d570d664b4aac3fe4462ff63a","final_host":"x.com","final_path":"/sitemap.xml","ms":176,"url":"https://x.com/sitemap.xml","declared":true,"xml":false},"files":{"llms_txt":{"status":404,"content_type":"text/html","bytes":46062,"sha256":"637556678d4962ca9dc7cb0e3dc29ccc5177ec8df04fdd6827403c2b7bf39ff2","final_host":"x.com","final_path":"/llms.txt","ms":80,"path":"/llms.txt","html":true,"json":null,"present":false},"agents_md":{"status":404,"content_type":"text/html","bytes":46070,"sha256":"510545f99feb19da0379af9c44c10d0118b696d53cc8dbab42a9c0ec2c5ec33d","final_host":"x.com","final_path":"/agents.md","ms":179,"path":"/agents.md","html":true,"json":null,"present":false},"agent_skills":{"status":404,"content_type":"text/html","bytes":46251,"sha256":"a34c71df76d16713cd9b583655375e67e8818aa9ac10b9202e8bb96c11160f27","final_host":"x.com","final_path":"/.well-known/agent-skills/index.json","ms":186,"path":"/.well-known/agent-skills/index.json","html":true,"json":null,"present":false},"mcp_card":{"status":404,"content_type":"text/html","bytes":46232,"sha256":"690b47bdc90a44f54f67a867f8355b2da0045f5b34e5d60746ab93e2f4d09cfd","final_host":"x.com","final_path":"/.well-known/mcp/server-card.json","ms":180,"path":"/.well-known/mcp/server-card.json","html":true,"json":null,"present":false},"api_catalog":{"status":404,"content_type":"text/html","bytes":46167,"sha256":"88572960e30bd684dfb6907cb2fa36d36b79e8cfe7a1993668619c064c6b8d37","final_host":"x.com","final_path":"/.well-known/api-catalog","ms":466,"path":"/.well-known/api-catalog","html":true,"json":null,"present":false},"a2a_card":{"status":404,"content_type":"text/html","bytes":46191,"sha256":"abc18904b5cee0daa2986ac441df02b8d9a0a01f319bbd7f35c8dc098c4822da","final_host":"x.com","final_path":"/.well-known/agent-card.json","ms":169,"path":"/.well-known/agent-card.json","html":true,"json":null,"present":false}},"rings":{"v":1,"candidates":[],"mcp":[],"a2a":null,"webmcp":{"model_context":false,"bridge":false,"mcp_json":false,"link_rel":false,"any":false},"handshakes":0,"get_path":{"status":200,"content_type":"text/html","ms":307,"error":null},"get_host":{"status":530,"content_type":"text/plain","ms":84,"error":null}},"sf":[4,0,0,0,0,0,1,0,0],"derived":{"reachable":true,"home_class":"answers","robots_served":true,"robots_sitemap":true,"content_signal":false,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":["GPTBot","OAI-SearchBot","ChatGPT-User","ClaudeBot","Claude-SearchBot","PerplexityBot","Google-Extended","Applebot-Extended","CCBot","Bytespider","Meta-ExternalAgent","Amazonbot"],"blocks_any_ai":true,"files":{"llms_txt":false,"agents_md":false,"agent_skills":false,"mcp_card":false,"api_catalog":false,"a2a_card":false},"any_capability":false,"ai_refused":["gptbot","claudebot","oaisearch","perplexity"],"ai_thin":[],"aipref":2,"aipref_measured":true,"sf":[4,0,0,0,0,0,1,0,0],"rings":{"mcp_declared":false,"mcp_live":false,"mcp_auth":false,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":3,"fh":"x.com","fp_other_host":[],"fp":"8b48d3564ad225ec","fp_bits":{}},"derive_v":5,"frame":"top1k","shard":7},{"v":1,"kind":"crawlcheck-census-record","run":43,"domain":"wordpress.org","rank":49,"measured_at":"2026-11-13T07:56:55.238Z","home":{"browser":{"status":200,"content_type":"text/html","bytes":178395,"sha256":"90e3e07bef05a56e1ca849ab78b7c2ec8c6e2a3e9e01474239f1cc0c11a7fbd3","final_host":"wordpress.org","final_path":"/","ms":814,"identity":"Mobile browser","html":true,"fp":"39fbefd5b2739be0"},"gptbot":{"status":200,"content_type":"text/html","bytes":178395,"sha256":"687cbcbcc344c68cdc55ddbf556b0a6f4848b41a2b27dc65f7b24e2531caf135","final_host":"wordpress.org","final_path":"/","ms":380,"identity":"GPTBot","html":true,"fp":"39fbefd5b2739be0"},"claudebot":{"status":200,"content_type":"text/html","bytes":178395,"sha256":"90e3e07bef05a56e1ca849ab78b7c2ec8c6e2a3e9e01474239f1cc0c11a7fbd3","final_host":"wordpress.org","final_path":"/","ms":791,"identity":"ClaudeBot","html":true,"fp":"39fbefd5b2739be0"},"oaisearch":{"status":200,"content_type":"text/html","bytes":178395,"sha256":"25c4fc02d6404d809e2eee2f06a4c456dfc8b04836ec1b510acdabc9d06cec7e","final_host":"wordpress.org","final_path":"/","ms":383,"identity":"OAI-SearchBot","html":true,"fp":"39fbefd5b2739be0"},"perplexity":{"status":200,"content_type":"text/html","bytes":178395,"sha256":"687cbcbcc344c68cdc55ddbf556b0a6f4848b41a2b27dc65f7b24e2531caf135","final_host":"wordpress.org","final_path":"/","ms":211,"identity":"PerplexityBot","html":true,"fp":"39fbefd5b2739be0"}},"robots":{"status":200,"content_type":"text/plain","bytes":976,"sha256":"65185a701db3ab195bcb2f86657300e13153f758af110ac0ffeeea5dc19ab5dc","final_host":"wordpress.org","final_path":"/robots.txt","ms":522,"html":false,"text":"Sitemap: https://wordpress.org/sitemap.xml\nSitemap: https://wordpress.org/news-sitemap.xml\nSitemap: https://wordpress.org/themes/sitemap.xml\nSitemap: https://wordpress.org/plugins/sitemap.xml\nSitemap: https://wordpress.org/news/sitemap.xml\nSitemap: https://wordpress.org/showcase/sitemap.xml\nSitemap: https://wordpress.org/documentation/sitemap.xml\nSitemap: https://wordpress.org/patterns/sitemap.xml\nSitemap: https://wordpress.org/photos/sitemap.xml\nUser-agent: *\nDisallow: /wp-admin/\nAllow: /wp-admin/admin-ajax.php\nAllow: /wp-admin/load-scripts.php\nAllow: /wp-admin/load-styles.php\n\nUser-agent: *\nDisallow: /search\nDisallow: /?s=\n\nUser-agent: GPTBot\nAllow: /\n\nUser-agent: ClaudeBot\nAllow: /\n\nUser-agent: anthropic-ai\nAllow: /\n\nUser-agent: Google-Extended\nAllow: /\n\nUser-agent: Applebot-Extended\nAllow: /\n\nUser-agent: PerplexityBot\nAllow: /\n\nUser-agent: Bytespider\nAllow: /\n\nUser-agent: CCBot\nAllow: /\n\nUser-agent: Copilot\nAllow: /\n\nUser-agent: *\nDisallow: /plugins/search/\n","agents":{"GPTBot":true,"OAI-SearchBot":true,"ChatGPT-User":true,"ClaudeBot":true,"Claude-SearchBot":true,"PerplexityBot":true,"Google-Extended":true,"Applebot-Extended":true,"CCBot":true,"Bytespider":true,"Meta-ExternalAgent":true,"Amazonbot":true}},"sitemap":{"status":200,"content_type":"text/xml","bytes":631,"sha256":"cd23d460235ac050dd80379b4a8b7190bd87b47aef92c07d8f3eb42423d479d8","final_host":"wordpress.org","final_path":"/sitemap.xml","ms":261,"url":"https://wordpress.org/sitemap.xml","declared":true,"xml":true},"files":{"llms_txt":{"status":200,"content_type":"text/plain","bytes":5765,"sha256":"731a4c979c79b5a350375f8b089fc26ed70b20d395de39ad305b716de233b4af","final_host":"wordpress.org","final_path":"/llms.txt","ms":148,"path":"/llms.txt","html":false,"json":null,"present":true},"agents_md":{"status":404,"content_type":"text/html","bytes":124910,"sha256":"273133e6c53a8dbc5b97f0eff7a77dfa844a5f4a4610e7980d62587f75a2e813","final_host":"wordpress.org","final_path":"/agents.md","ms":380,"path":"/agents.md","html":true,"json":null,"present":false},"agent_skills":{"status":404,"content_type":"text/html","bytes":124962,"sha256":"8ef4a1e2261a89f2611089e379c7f7ea6c04de45c6263bcc5b7a5e4d3f4f13c8","final_host":"wordpress.org","final_path":"/.well-known/agent-skills/index.json","ms":413,"path":"/.well-known/agent-skills/index.json","html":true,"json":null,"present":false},"mcp_card":{"status":404,"content_type":"text/html","bytes":124956,"sha256":"571e5e92bac00217ff8ffb6466d2ae138d2d1ba31e305ed4cc027222de3a116d","final_host":"wordpress.org","final_path":"/.well-known/mcp/server-card.json","ms":353,"path":"/.well-known/mcp/server-card.json","html":true,"json":null,"present":false},"api_catalog":{"status":404,"content_type":"text/html","bytes":124938,"sha256":"20dae00f0a500ba9938f20f08a15c1a54eb2586efad856c1ad4d6c5b517f5338","final_host":"wordpress.org","final_path":"/.well-known/api-catalog","ms":434,"path":"/.well-known/api-catalog","html":true,"json":null,"present":false},"a2a_card":{"status":404,"content_type":"text/html","bytes":124946,"sha256":"b6e164cba9d5e843c8dc2bde2526500d9928313533b29347c0ba1786592fe18a","final_host":"wordpress.org","final_path":"/.well-known/agent-card.json","ms":410,"path":"/.well-known/agent-card.json","html":true,"json":null,"present":false}},"rings":{"v":1,"candidates":[],"mcp":[],"a2a":null,"webmcp":{"model_context":false,"bridge":false,"mcp_json":false,"link_rel":false,"any":false},"handshakes":0,"get_path":{"status":404,"content_type":"text/html","ms":394,"error":null},"get_host":{"status":200,"content_type":"text/html","ms":959,"error":null}},"sf":[4,0,0,0,0,0,1,0,0],"derived":{"reachable":true,"home_class":"answers","robots_served":true,"robots_sitemap":true,"content_signal":false,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":[],"blocks_any_ai":false,"files":{"llms_txt":true,"agents_md":false,"agent_skills":false,"mcp_card":false,"api_catalog":false,"a2a_card":false},"any_capability":false,"ai_refused":[],"ai_thin":[],"aipref":0,"aipref_measured":true,"sf":[4,0,0,0,0,0,1,0,0],"rings":{"mcp_declared":false,"mcp_live":false,"mcp_auth":false,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":3,"fh":"wordpress.org","fp_other_host":[],"fp":"39fbefd5b2739be0","fp_bits":{"gptbot":0,"claudebot":0,"oaisearch":0,"perplexity":0}},"derive_v":5,"frame":"top1k","shard":0},{"v":1,"kind":"crawlcheck-census-record","run":43,"domain":"digicert.com","rank":50,"measured_at":"2026-11-13T07:57:55.549Z","home":{"browser":{"status":200,"content_type":"text/html","bytes":69492,"sha256":"b43735cf19993773ea8ba508566b5b4b073be0691a9b750c4970cbef494c91bf","final_host":"www.digicert.com","final_path":"/","ms":847,"identity":"Mobile browser","html":true,"fp":"9119de7a93913e97"},"gptbot":{"status":200,"content_type":"text/html","bytes":69490,"sha256":"7f649366b016ef51df1ea9f9049b6e9c4b6fb9640834e8c2a8ce4b383fba4dd2","final_host":"www.digicert.com","final_path":"/","ms":846,"identity":"GPTBot","html":true,"fp":"9119de7a93913e97"},"claudebot":{"status":200,"content_type":"text/html","bytes":69491,"sha256":"c7b9e1e69b2722b0cb4e32dc598e198ecc87d1ee19c47deea0c7988e13f6df61","final_host":"www.digicert.com","final_path":"/","ms":846,"identity":"ClaudeBot","html":true,"fp":"9119de7a93913e97"},"oaisearch":{"status":200,"content_type":"text/html","bytes":69492,"sha256":"fad2ee6dd56a436ab0f45b3bd616b6f016683fedcd5060cd09c959b6750279e8","final_host":"www.digicert.com","final_path":"/","ms":831,"identity":"OAI-SearchBot","html":true,"fp":"9119de7a93913e97"},"perplexity":{"status":200,"content_type":"text/html","bytes":1038,"sha256":"f5e5c75c5949306c331c3e0b899410c34f8bd1c799a83e0924ac8353ea2ffb4c","final_host":"www.digicert.com","final_path":"/","ms":811,"identity":"PerplexityBot","html":true}},"robots":{"status":200,"content_type":"text/plain","bytes":2605,"sha256":"a54fb2d2729303e9c3fe3b38d8be5112f7794d7423013e38d0234df03ed19bc5","final_host":"www.digicert.com","final_path":"/robots.txt","ms":1163,"html":false,"text":"# Welcome to DigiCert\r\n# Think you have what it takes to join the best team in the world?\r\n# Check out our careers while you're here\r\n# https://www.digicert.com/careers\r\n# Or check out our YouTube channel for our latest\r\n# https://www.youtube.com/user/DigiCertSSL\r\n\r\nUser-agent: *\r\nDisallow: /account/sso/idp\r\nDisallow: */test/\r\nDisallow: */ni/\r\nDisallow: */gated-content-pdfs/\r\nDisallow: /content/dam/digicert/pdfs/ni/\r\nDisallow: /account/saml-guest-request/idp\r\nDisallow: /webinars/*\r\nDisallow: /seals/\r\nDisallow: /link/\r\nDisallow: /secure/*\r\nDisallow: /services/*\r\nDisallow: /in-console-ssm\r\nDisallow: /jp/sid-partner*\r\nDisallow: /ppc/\r\nDisallow: /ru/\r\nDisallow: /ar/\r\nDisallow: /dc/blog/*\r\nDisallow: /dc/news/*\r\nDisallow: /author/*\r\nDisallow: /tag/*\r\nDisallow: /clients/*\r\nDisallow: /voucher/*\r\nDisallow: /bios*\r\nDisallow: /direct/*\r\nDisallow: /*thank\r\nDisallow: /*.xls$\r\nDisallow: /*.docx$\r\nDisallow: /language-masters/*\r\nDisallow: /link*\r\nDisallow: /*feed\r\n\r\nAllow: /*.js\r\nAllow: /*.css\r\n\r\n# AI / answer-engine crawlers\r\n\r\nUser-agent: GPTBot\r\nAllow: /\r\n\r\nUser-agent: OAI-SearchBot\r\nAllow: /\r\n\r\nUser-agent: ChatGPT-User\r\nAllow: /\r\n\r\nUser-agent: PerplexityBot\r\nAllow: /\r\n\r\nUser-agent: Perplexity-User\r\nAllow: /\r\n\r\nUser-agent: ClaudeBot\r\nAllow: /\r\n\r\nUser-agent: Claude-SearchBot\r\nAllow: /\r\n\r\nUser-agent: Claude-User\r\nAllow: /\r\n\r\nUser-agent: Googlebot-Image\r\nAllow: /\r\n\r\nUser-agent: AdsBot-Google\r\nAllow: /\r\n\r\nUser-agent: Google-Extended\r\nAllow: /\r\n\r\nSitemap: https://www.digicert.com/sitemap.xml\r\nSitemap: https://www.digicert.com/cn/sitemap.xml\r\nSitemap: https://www.digicert.com/de/sitemap.xml\r\nSitemap: https://www.digicert.com/es/sitemap.xml\r\nSitemap: https://www.digicert.com/fr/sitemap.xml\r\nSitemap: https://www.digicert.com/it/sitemap.xml\r\nSitemap: https://www.digicert.com/jp/sitemap.xml\r\nSitemap: https://www.digicert.com/kr/sitemap.xml\r\nSitemap: https://www.digicert.com/nl/sitemap.xml\r\nSitemap: https://www.digicert.com/tw/sitemap.xml\r\nSitemap: https://www.digicert.com/pt/sitemap.xml\r\nSitemap: https://www.digicert.com/faq/sitemap.xml\r\nSitemap: https://www.digicert.com/blog/sitemap.xml\r\nSitemap: https://www.digicert.com/eds-sitemap.xml\r\n\r\n# Bot Requests\r\n\r\nUser-agent: XoviBot\r\nDisallow: /\r\n\r\nUser-agent: BLEXBot\r\nDisallow: /\r\n\r\nUser-agent: DotBot\r\nDisallow: /\r\n\r\nUser-agent: MJ12bot\r\nDisallow: /\r\n\r\nUser-agent: Xenu's\r\nDisallow: /\r\n\r\nUser-agent: Xenu's Link Sleuth 1.1c\r\nDisallow: /\r\n\r\nUser-agent: Zeus\r\nDisallow: /\r\n\r\nUser-agent: Zeus 32297 Webster Pro V2.9 Win32\r\nDisallow: /\r\n\r\nUser-agent: Updownerbot\r\nDisallow: /\r\n\r\nUser-agent: Blekkobot\r\nDisallow: /\r\n\r\nUser-agent: 008\r\nDisallow: /","agents":{"GPTBot":true,"OAI-SearchBot":true,"ChatGPT-User":true,"ClaudeBot":true,"Claude-SearchBot":true,"PerplexityBot":true,"Google-Extended":true,"Applebot-Extended":true,"CCBot":true,"Bytespider":true,"Meta-ExternalAgent":true,"Amazonbot":true}},"sitemap":{"status":200,"content_type":"text/html","bytes":1033,"sha256":"33c042f93baa323ad27d2e04c6268f47d55a09b36329ff922e85944b94e636d6","final_host":"www.digicert.com","final_path":"/sitemap.xml","ms":67,"url":"https://www.digicert.com/sitemap.xml","declared":true,"xml":false},"files":{"llms_txt":{"status":200,"content_type":"text/plain","bytes":8395,"sha256":"02169ec84a351a1d16d7d35ae9186f3edaa67dc827c2a7b55f9cea8f32abd439","final_host":"www.digicert.com","final_path":"/llms.txt","ms":1123,"path":"/llms.txt","html":false,"json":null,"present":true},"agents_md":{"status":200,"content_type":"text/html","bytes":1033,"sha256":"bde5c732bf6abd56f757c9aff19ed74972b9ea3c5b07a43b3e09c046c4874866","final_host":"www.digicert.com","final_path":"/agents.md","ms":854,"path":"/agents.md","html":true,"json":null,"present":false},"agent_skills":{"status":200,"content_type":"text/html","bytes":1033,"sha256":"1688d7b53364c188cdb87b74c5d745a1fd94f72777fd7c023d20815b10fa1353","final_host":"www.digicert.com","final_path":"/.well-known/agent-skills/index.json","ms":732,"path":"/.well-known/agent-skills/index.json","html":true,"json":null,"present":false},"mcp_card":{"status":200,"content_type":"text/html","bytes":1033,"sha256":"593010b87122b841827467058bffc39d23b1b5fd2d8ce594be13da594f6be6dc","final_host":"www.digicert.com","final_path":"/.well-known/mcp/server-card.json","ms":234,"path":"/.well-known/mcp/server-card.json","html":true,"json":null,"present":false},"api_catalog":{"status":200,"content_type":"text/html","bytes":1039,"sha256":"0a74cb889a7f9bbe88d72cac8c904fcae5f5ed20d3e8dcb3815376648e453085","final_host":"www.digicert.com","final_path":"/.well-known/api-catalog","ms":222,"path":"/.well-known/api-catalog","html":true,"json":null,"present":false},"a2a_card":{"status":200,"content_type":"text/html","bytes":1040,"sha256":"2de05af7224c3c5a74a6087d1afb750a3c4d5deb62a5d26cc8691f1c637f63b9","final_host":"www.digicert.com","final_path":"/.well-known/agent-card.json","ms":222,"path":"/.well-known/agent-card.json","html":true,"json":null,"present":false}},"rings":{"v":1,"candidates":[],"mcp":[],"a2a":null,"webmcp":{"model_context":false,"bridge":false,"mcp_json":false,"link_rel":false,"any":false},"handshakes":0,"get_path":{"status":200,"content_type":"text/html","ms":230,"error":null},"get_host":{"status":404,"content_type":null,"ms":822,"error":null}},"sf":[1,0,0,0,0,0,0,0,0],"derived":{"reachable":true,"home_class":"answers","robots_served":true,"robots_sitemap":true,"content_signal":false,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":[],"blocks_any_ai":false,"files":{"llms_txt":true,"agents_md":false,"agent_skills":false,"mcp_card":false,"api_catalog":false,"a2a_card":false},"any_capability":false,"ai_refused":[],"ai_thin":["perplexity"],"aipref":0,"aipref_measured":true,"sf":[1,0,0,0,0,0,0,0,0],"rings":{"mcp_declared":false,"mcp_live":false,"mcp_auth":false,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":3,"fh":"www.digicert.com","fp_other_host":[],"fp":"9119de7a93913e97","fp_bits":{"gptbot":0,"claudebot":0,"oaisearch":0}},"derive_v":5,"frame":"top1k","shard":1},{"v":1,"kind":"crawlcheck-census-record","run":43,"domain":"skype.com","rank":51,"measured_at":"2026-11-13T07:58:58.649Z","home":{"browser":{"status":200,"content_type":"text/html","bytes":61966,"sha256":"b144e6fb1ec4ac630a03723cabfe89942b8d6bfb3b11df663c0a4a520b4e1ad9","final_host":"teams.live.com","final_path":"/free","ms":1712,"identity":"Mobile browser","html":true,"fp":"e138e2f4827477b1"},"gptbot":{"status":200,"content_type":"text/html","bytes":61987,"sha256":"31b8a45a43ad47c275471b36a0ce399ba96269bc1e37699361463f1719131f94","final_host":"teams.live.com","final_path":"/free","ms":308,"identity":"GPTBot","html":true,"fp":"e138e2f4827477b1"},"claudebot":{"status":200,"content_type":"text/html","bytes":61967,"sha256":"6f438fab26ef293a51db086dffbc482aa165ae18d27cefefab32d742d5b45205","final_host":"teams.live.com","final_path":"/free","ms":213,"identity":"ClaudeBot","html":true,"fp":"e138e2f4827477b1"},"oaisearch":{"status":200,"content_type":"text/html","bytes":61987,"sha256":"c37ce8177dd281b1d4fe79f0dcba9a64094a5291ff84d726985b1026e740f446","final_host":"teams.live.com","final_path":"/free","ms":98,"identity":"OAI-SearchBot","html":true,"fp":"e138e2f4827477b1"},"perplexity":{"status":200,"content_type":"text/html","bytes":61967,"sha256":"05c7d1dc0ff7a1100dbe9144915adcdf6c83daedccdc2565c6cb515c9313c1fb","final_host":"teams.live.com","final_path":"/free","ms":700,"identity":"PerplexityBot","html":true,"fp":"e138e2f4827477b1"}},"robots":{"status":200,"content_type":"text/html","bytes":61967,"sha256":"70836e91a623d01997d42bccb51efb3314dcc921a80b26aaa99a331f6ebfaade","final_host":"teams.live.com","final_path":"/free","ms":713,"html":true,"text":null,"agents":null},"sitemap":{"status":200,"content_type":"text/html","bytes":61987,"sha256":"fbb88c475e3c5d419c51aa6ef95f13b2a0bb47b5b7a9b16124ddf159377f4379","final_host":"teams.live.com","final_path":"/free","ms":94,"url":"https://skype.com/sitemap.xml","declared":false,"xml":false},"files":{"llms_txt":{"status":200,"content_type":"text/html","bytes":61966,"sha256":"c08f20078e713e0aa6ccf26d13fcbbabf318073361f29f216b9c92a2fcd6bcb3","final_host":"teams.live.com","final_path":"/free","ms":228,"path":"/llms.txt","html":true,"json":null,"present":false},"agents_md":{"status":200,"content_type":"text/html","bytes":61988,"sha256":"76f1942ad00a15d380d03b7262b27a7ae1649a6cea4c4e67ae58ade0c1f5d1f6","final_host":"teams.live.com","final_path":"/free","ms":249,"path":"/agents.md","html":true,"json":null,"present":false},"agent_skills":{"status":200,"content_type":"text/html","bytes":61967,"sha256":"0c926b9e852d789a743479517463415f8a4beed10f98dd006d43e06d40877c91","final_host":"teams.live.com","final_path":"/free","ms":158,"path":"/.well-known/agent-skills/index.json","html":true,"json":null,"present":false},"mcp_card":{"status":200,"content_type":"text/html","bytes":61987,"sha256":"2f2bfb7da567b03a27020ccf43e0856aad3b052f301fb3bbab4e6d5a8abb6e95","final_host":"teams.live.com","final_path":"/free","ms":108,"path":"/.well-known/mcp/server-card.json","html":true,"json":null,"present":false},"api_catalog":{"status":200,"content_type":"text/html","bytes":61987,"sha256":"7f78e45c179ab2150dc69090f7bb701793a1b113b6784d459354316924e78cba","final_host":"teams.live.com","final_path":"/free","ms":127,"path":"/.well-known/api-catalog","html":true,"json":null,"present":false},"a2a_card":{"status":200,"content_type":"text/html","bytes":61966,"sha256":"d573ffac8167133c01d5cfe6f29a37fbf9993ad7a8916fc63f1c9137b2ee56eb","final_host":"teams.live.com","final_path":"/free","ms":281,"path":"/.well-known/agent-card.json","html":true,"json":null,"present":false}},"rings":{"v":1,"candidates":[],"mcp":[],"a2a":null,"webmcp":{"model_context":false,"bridge":false,"mcp_json":false,"link_rel":false,"any":false},"handshakes":0,"get_path":{"status":200,"content_type":"text/html","ms":90,"error":null},"get_host":{"status":530,"content_type":"text/plain","ms":12,"error":null}},"sf":[1,0,0,0,0,0,2,0,0],"derived":{"reachable":true,"home_class":"answers","robots_served":false,"robots_sitemap":false,"content_signal":false,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":[],"blocks_any_ai":false,"files":{"llms_txt":false,"agents_md":false,"agent_skills":false,"mcp_card":false,"api_catalog":false,"a2a_card":false},"any_capability":false,"ai_refused":[],"ai_thin":[],"sf":[1,0,0,0,0,0,2,0,0],"rings":{"mcp_declared":false,"mcp_live":false,"mcp_auth":false,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":3,"fh":"teams.live.com","fp_other_host":[],"fp":"e138e2f4827477b1","fp_bits":{"gptbot":0,"claudebot":0,"oaisearch":0,"perplexity":0}},"derive_v":5,"frame":"top1k","shard":2},{"v":1,"kind":"crawlcheck-census-record","run":43,"domain":"icloud.com","rank":52,"measured_at":"2026-11-13T07:59:19.049Z","home":{"browser":{"status":200,"content_type":"text/html","bytes":31452,"sha256":"c717aece2a7ab8c0b7f3aa9d30bdccac3311b4873c7965bd62fe72041b3ed299","final_host":"www.icloud.com","final_path":"/","ms":99,"identity":"Mobile browser","html":true},"gptbot":{"status":200,"content_type":"text/html","bytes":31452,"sha256":"c717aece2a7ab8c0b7f3aa9d30bdccac3311b4873c7965bd62fe72041b3ed299","final_host":"www.icloud.com","final_path":"/","ms":16,"identity":"GPTBot","html":true},"claudebot":{"status":200,"content_type":"text/html","bytes":31452,"sha256":"c717aece2a7ab8c0b7f3aa9d30bdccac3311b4873c7965bd62fe72041b3ed299","final_host":"www.icloud.com","final_path":"/","ms":25,"identity":"ClaudeBot","html":true},"oaisearch":{"status":200,"content_type":"text/html","bytes":31452,"sha256":"c717aece2a7ab8c0b7f3aa9d30bdccac3311b4873c7965bd62fe72041b3ed299","final_host":"www.icloud.com","final_path":"/","ms":19,"identity":"OAI-SearchBot","html":true},"perplexity":{"status":200,"content_type":"text/html","bytes":31452,"sha256":"c717aece2a7ab8c0b7f3aa9d30bdccac3311b4873c7965bd62fe72041b3ed299","final_host":"www.icloud.com","final_path":"/","ms":16,"identity":"PerplexityBot","html":true}},"robots":{"status":200,"content_type":"text/plain","bytes":1289,"sha256":"7c0da5b622a8703a69e156d3df581d1a2c5d4106a0da2e85c4db8f18c5a9e9c8","final_host":"www.icloud.com","final_path":"/robots.txt","ms":98,"html":false,"text":"\nSitemap: https://www.icloud.com/sitemap.xml\nSitemap: https://www.icloud.com/sitemap-info.xml\n\nUser-agent: *\n\nDisallow: /unsupported_browser\nDisallow: /unsupported_mobile\nDisallow: /enable_cookies\nDisallow: /maintenance\nDisallow: /not_found\nDisallow: /unavailable\nDisallow: /attachment/\nDisallow: /download/\nDisallow: /invites/\nDisallow: /iw/\nDisallow: /bookmarks_extension/\nDisallow: /family_sharing/\nDisallow: /calendar/eventreply/\nDisallow: /fmf_getting_started/\nDisallow: /fmf_location_update/\nDisallow: /fmf_location_update_unsubscribe_undo/\nDisallow: /fmf_unsubscribe/\nDisallow: /fmf_unsubscribe_error/\nDisallow: /fmf_unsubscribe_success/\nDisallow: /fmf_unsubscribe_undo/\nDisallow: /fmf_unsupported_device/\nDisallow: /photos_get_started/\nDisallow: /sharedalbum_unsubscribe/\nDisallow: /sharedalbum_unsubscribe_error/\nDisallow: /sharedalbum_unsubscribe_undo/\nDisallow: /sharedalbum_unsupported_device/\nDisallow: /photostream_unsubscribe/\nDisallow: /iclouddrive/\nDisallow: /notes/\nDisallow: /photos/\nDisallow: /reminders/\nDisallow: /share/\nDisallow: /pages/\nDisallow: /numbers/\nDisallow: /keynote/\nDisallow: /keynote-live/\nDisallow: /sharedalbum/\nDisallow: /ca/ics/\nDisallow: /shared/\n\nUser-agent: Googlebot\nAllow: /\n\nUser-agent: facebookexternalhit\nAllow: /invites/\nAllow: /opengraph/\n","agents":{"GPTBot":true,"OAI-SearchBot":true,"ChatGPT-User":true,"ClaudeBot":true,"Claude-SearchBot":true,"PerplexityBot":true,"Google-Extended":true,"Applebot-Extended":true,"CCBot":true,"Bytespider":true,"Meta-ExternalAgent":true,"Amazonbot":true}},"sitemap":{"status":200,"content_type":"text/xml","bytes":3442,"sha256":"b73212a376f2901d9ff3ec1bb8d59999116437dc44b19691c5ccc0fbcce60430","final_host":"www.icloud.com","final_path":"/sitemap.xml","ms":120,"url":"https://www.icloud.com/sitemap.xml","declared":true,"xml":true},"files":{"llms_txt":{"status":404,"content_type":"text/html","bytes":3250,"sha256":"836485c3395d88777be49942e7f66bdc1a0f1ed982533ff2d0cbeebd52d3f5dc","final_host":"www.icloud.com","final_path":"/llms.txt","ms":101,"path":"/llms.txt","html":true,"json":null,"present":false},"agents_md":{"status":404,"content_type":"text/html","bytes":3250,"sha256":"836485c3395d88777be49942e7f66bdc1a0f1ed982533ff2d0cbeebd52d3f5dc","final_host":"www.icloud.com","final_path":"/agents.md","ms":119,"path":"/agents.md","html":true,"json":null,"present":false},"agent_skills":{"status":404,"content_type":"text/html","bytes":3250,"sha256":"836485c3395d88777be49942e7f66bdc1a0f1ed982533ff2d0cbeebd52d3f5dc","final_host":"www.icloud.com","final_path":"/.well-known/agent-skills/index.json","ms":62,"path":"/.well-known/agent-skills/index.json","html":true,"json":null,"present":false},"mcp_card":{"status":404,"content_type":"text/html","bytes":3250,"sha256":"836485c3395d88777be49942e7f66bdc1a0f1ed982533ff2d0cbeebd52d3f5dc","final_host":"www.icloud.com","final_path":"/.well-known/mcp/server-card.json","ms":96,"path":"/.well-known/mcp/server-card.json","html":true,"json":null,"present":false},"api_catalog":{"status":404,"content_type":"text/html","bytes":3250,"sha256":"836485c3395d88777be49942e7f66bdc1a0f1ed982533ff2d0cbeebd52d3f5dc","final_host":"www.icloud.com","final_path":"/.well-known/api-catalog","ms":86,"path":"/.well-known/api-catalog","html":true,"json":null,"present":false},"a2a_card":{"status":404,"content_type":"text/html","bytes":3250,"sha256":"836485c3395d88777be49942e7f66bdc1a0f1ed982533ff2d0cbeebd52d3f5dc","final_host":"www.icloud.com","final_path":"/.well-known/agent-card.json","ms":64,"path":"/.well-known/agent-card.json","html":true,"json":null,"present":false}},"rings":{"v":1,"candidates":[],"mcp":[],"a2a":null,"webmcp":{"model_context":false,"bridge":false,"mcp_json":false,"link_rel":false,"any":false},"handshakes":0,"get_path":{"status":404,"content_type":"text/html","ms":81,"error":null},"get_host":{"status":530,"content_type":"text/plain","ms":12,"error":null}},"sf":[4,0,0,0,0,0,0,0,1],"derived":{"reachable":true,"home_class":"answers","robots_served":true,"robots_sitemap":true,"content_signal":false,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":[],"blocks_any_ai":false,"files":{"llms_txt":false,"agents_md":false,"agent_skills":false,"mcp_card":false,"api_catalog":false,"a2a_card":false},"any_capability":false,"ai_refused":[],"ai_thin":[],"aipref":0,"aipref_measured":true,"sf":[4,0,0,0,0,0,0,0,1],"rings":{"mcp_declared":false,"mcp_live":false,"mcp_auth":false,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":3,"fh":"www.icloud.com","fp_other_host":[]},"derive_v":5,"frame":"top1k","shard":3},{"v":1,"kind":"crawlcheck-census-record","run":43,"domain":"pinterest.com","rank":53,"measured_at":"2026-11-13T08:05:49.397Z","home":{"browser":{"status":200,"content_type":"text/html","bytes":1305191,"sha256":"4418fae59665fd59d073f67586e9a7629b795959024f310b197f616bfd4af9b4","final_host":"www.pinterest.com","final_path":"/","ms":204,"identity":"Mobile browser","html":true,"fp":"415139f3e79f27eb"},"gptbot":{"status":403,"content_type":"text/html","bytes":2987,"sha256":"24904ffb770e2e4efaddc2822a25172805bf7d496d12cb32e11b052cc1f070de","final_host":"www.pinterest.com","final_path":"/","ms":51,"identity":"GPTBot","html":true},"claudebot":{"status":200,"content_type":"text/html","bytes":1276583,"sha256":"6303207b08000273efdf5dbbf484e3d95e454a3b73feede419697923fffe13f8","final_host":"www.pinterest.com","final_path":"/","ms":312,"identity":"ClaudeBot","html":true},"oaisearch":{"status":403,"content_type":"text/html","bytes":2987,"sha256":"0b78cc8b3e8bf900c9acdca43ac844aba1766bb36ce31b315cda8f447f9ab365","final_host":"www.pinterest.com","final_path":"/","ms":70,"identity":"OAI-SearchBot","html":true},"perplexity":{"status":403,"content_type":"text/html","bytes":2987,"sha256":"76c9756a3825634f6d90c2320a61bbb4145b5cdd83be48a5edc378f7e55a66c3","final_host":"www.pinterest.com","final_path":"/","ms":117,"identity":"PerplexityBot","html":true}},"robots":{"status":200,"content_type":"text/plain","bytes":32989,"sha256":"96f8ce958e3ab15489b925dc10f3e65c1ea4e62851ca6e2d7fdaa4d26124b165","final_host":"www.pinterest.com","final_path":"/robots.txt","ms":166,"html":false,"text":"# Allowlist your bot: https://help.pinterest.com/bot-submission-form\n#\n# Pinterest is hiring!\n#\n# Learn about the SEO work that we're doing at https://medium.com/@Pinterest_Engineering/demystifying-seo-with-experiments-a183b325cf4c\n#\n# Check out some of our available positions at https://www.pinterestcareers.com/\n\nUser-agent: 2checkout\nUser-agent: AASA-Bot\nUser-agent: AccessibleWebBot\nUser-agent: Adagiobot\nUser-agent: adbeat_bot\nUser-agent: ADmantX\nUser-agent: AdometryBot\nUser-agent: Adsense-Snapshot-Google\nUser-agent: AdsTxtCrawler\nUser-agent: Adyen\nUser-agent: AhrefsBot\nUser-agent: AhrefsSiteAudit\nUser-agent: Akismet\nUser-agent: Algolia Crawler\nUser-agent: AmazonProductDiscovery\nUser-agent: Aolbot-News\nUser-agent: APIs-Google\nUser-agent: AppInsights\nUser-agent: Applebot\nUser-agent: AppleNewsBot\nUser-agent: Archive-It\nUser-agent: archive.org_bot\nUser-agent: ArchiveBot\nUser-agent: ArchiveTeam-ArchiveBot\nUser-agent: AspiegelBot\nUser-agent: Audisto Crawler\nUser-agent: Awario\nUser-agent: Baidu-ADS\nUser-agent: Baiduspider\nUser-agent: Barkrowler\nUser-agent: bingbot\nUser-agent: BingPreview\nUser-agent: bitlybot\nUser-agent: bl.uk_ldfc_bot\nUser-agent: BLP_bbot\nUser-agent: bordermanager\nUser-agent: botify\nUser-agent: BrandVerity\nUser-agent: Bravebot\nUser-agent: BrightEdge Crawler\nUser-agent: BrightEdge-Crawler\nUser-agent: BUbiNG\nUser-agent: CaliperBot\nUser-agent: CapitalOneBot\nUser-agent: Catchpoint\nUser-agent: cf-qualys-scanner\nUser-agent: Chrome-Lighthouse\nUser-agent: CJBot\nUser-agent: ClarityBot\nUser-agent: Clickagy Intelligence Bot v2\nUser-agent: Cloudflare-AI-Search\nUser-agent: CloudflareBrowserRenderingCrawler\nUser-agent: coccocbot\nUser-agent: ContextualBot\nUser-agent: CookieReports\nUser-agent: Coveobot\nUser-agent: CriteoBot\nUser-agent: curly-\nUser-agent: cXensebot\nUser-agent: Datadog\nUser-agent: DataForSeoBot\nUser-agent: Daum\nUser-agent: demandbase\nUser-agent: Discordbot\nUser-agent: DotBot\nUser-agent: DoubleVerifyBot\nUser-agent: Doximity-Pipeline\nUser-agent: Dragonbot\nUser-agent: Dratabot\nUser-agent: DuckAssistBot\nUser-agent: DuckDuckBot\nUser-agent: DuckDuckGo-Favicons-Bot\nUser-agent: DVBot\nUser-agent: eContext\nUser-agent: ecurly Webhook\nUser-agent: EngageBDR\nUser-agent: europarchive.org\nUser-agent: evidon\nUser-agent: ExaleadCloudview\nUser-agent: FacebookBot\nUser-agent: facebookcatalog\nUser-agent: facebookexternalhit\nUser-agent: Facebot\nUser-agent: FastDAST\nUser-agent: FeedBurner\nUser-agent: Feedfetcher-Google\nUser-agent: FlipboardProxy\nUser-agent: FlippBot\nUser-agent: Freshping\nUser-agent: GeedoProductSearch\nUser-agent: Google Favicon\nUser-agent: Google News\nUser-agent: Google Page Speed Insights\nUser-agent: Google Wireless Transcoder\nUser-agent: Google-Apps-Script\nUser-agent: Google-Certificates-Bridge\nUser-agent: Google-CWS\nUser-agent: Google-Display-Ads-Bot\nUser-agent: Google-Firebase\nUser-agent: Google-HotelAdsVerifier\nUser-agent: Google-InspectionTool\nUser-agent: Google-NotebookLM\nUser-agent: Google-PageRenderer\nUser-agent: Google-PhysicalWeb\nUser-agent: Google-Pinpoint\nUser-agent: Google-Read-Aloud\nUser-agent: Google-Safety\nUser-agent: Google-Site-Verification\nUser-agent: Google-speakr\nUser-agent: Google-Structured-Data-Testing\nUser-agent: Google-Trust-Services\nUser-agent: google_partner_monitoring\nUser-agent: GoogleAssociationService\nUser-agent: GoogleImageProxy\nUser-agent: GoogleMessages\nUser-agent: GoogleProducer\nUser-agent: GoogleSecurityScanner\nUser-agent: grafana/synthetic-monitoring\nUser-agent: GroovinaAdsbot\nUser-agent: GuestpostsBot\nUser-agent: heritrix\nUser-agent: Hotjar\nUser-agent: HubSpot\nUser-agent: ia_archiver\nUser-agent: IABot\nUser-agent: IAS-Crawler\nUser-agent: ias_crawler\nUser-agent: ias_wombles\nUser-agent: INA dlweb\nUser-agent: infolink\nUser-agent: InteractiveAdvertisingBureau\nUser-agent: ip-label\nUser-agent: J-BRW\nUser-agent: J-DLC\nUser-agent: James BOT\nUser-agent: JobRapido\nUser-agent: Jooblebot\nUser-agent: Jumio\nUser-agent: kakaotalk-scrap\nUser-agent: KargoBot-Artemis\nUser-agent: Leikibot\nUser-agent: LiftoffDroid\nUser-agent: line-poker\nUser-agent: Linguee Bot\nUser-agent: linkchecker.pro\nUser-agent: LOCKSS cache\nUser-agent: LogicMonitor SiteMonitor\nUser-agent: Lyonl\nUser-agent: magpie-crawler\nUser-agent: Mastodon\nUser-agent: MaxPoint Bot\nUser-agent: MBCrawler\nUser-agent: mediaboardbot\nUser-agent: MediaMonitoringBot\nUser-agent: Mediatoolkitbot\nUser-agent: MeltwaterNews\nUser-agent: Metorik\nUser-agent: MgidBot\nUser-agent: MicrosoftPreview\nUser-agent: MixrankBot\nUser-agent: MJ12bot\nUser-agent: moatbot\nUser-agent: MojeekBot\nUser-agent: Monsidobot\nUser-agent: Moreover\nUser-agent: Morningscore Bot\nUser-agent: MSNPTC\nUser-agent: msrbot\nUser-agent: MTRobot\nUser-agent: netarkivindsamling\nUser-agent: NetpeakCheckerBot\nUser-agent: NeumobBot\nUser-agent: Neustar\nUser-agent: Nielsen\nUser-agent: NostoCrawlerBot\nUser-agent: NumeratorBot\nUser-agent: OdklBot\nUser-agent: OKX-dolphin-crawler\nUser-agent: OnCrawl\nUser-agent: OneTrust\nUser-agent: OPBot\nUser-agent: OSZKbot\nUser-agent: outbrain\nUser-agent: Paddle\nUser-agent: PageFreezer\nUser-agent: Pandalytics\nUser-agent: parsely.com\nUser-agent: ParselySharesBot\nUser-agent: PayPal\nUser-agent: peer39_crawler\nUser-agent: Pinterest\nUser-agent: Pinterestbot\nUser-agent: Plukkie\nUser-agent: Prerender\nUser-agent: PriceSpider\nUser-agent: ProjectShield-UrlCheck\nUser-agent: proximic\nUser-agent: QualifiedBot\nUser-agent: Quantcastbot\nUser-agent: Qwantbot\nUser-agent: Qwantify\nUser-agent: Razorpay\nUser-agent: Rigor\nUser-agent: RiverglassScanner\nUser-agent: RKG Url Verifier\nUser-agent: rogerbot-crawler\nUser-agent: RuxitSynthetic\nUser-agent: scanalert\nUser-agent: search.marginalia.nu\nUser-agent: SearchAtlas Bot\nUser-agent: SeekportBot\nUser-agent: SemanticJuice\nUser-agent: SemrushBot\nUser-agent: SemrushBot-BA\nUser-agent: SemrushBot-OCOB\nUser-agent: SemrushBot-SA\nUser-agent: SemrushBot-SI\nUser-agent: SemrushBot-SWA\nUser-agent: sentry\nUser-agent: SentryUptimeBot\nUser-agent: seo-audit-check-bot\nUser-agent: SeobilityBot\nUser-agent: SERankingBacklinksBot\nUser-agent: SeznamBot\nUser-agent: SinceraSyntheticUser\nUser-agent: SISTRIX Optimizer\nUser-agent: SiteAuditBot\nUser-agent: sitebulb\nUser-agent: SiteCheckerBotCrawler\nUser-agent: Siteimprove-SiteCheck\nUser-agent: SiteSearch360\nUser-agent: SkypeUriPreview\nUser-agent: Slack-ImgProxy\nUser-agent: Slackbot\nUser-agent: Slurp\nUser-agent: SmarshBot\nUser-agent: SmartBriefBot\nUser-agent: Snap URL Preview Service\nUser-agent: SnapPreviewBot\nUser-agent: Sogou\nUser-agent: solarwinds\nUser-agent: spbot\nUser-agent: Splunk Synthetics\nUser-agent: SpringBot\nUser-agent: SSL Labs\nUser-agent: ssr.ias-lab.de\nUser-agent: Stape\nUser-agent: SteamChat\nUser-agent: Storebot-Google\nUser-agent: Stripe\nUser-agent: Sucuri\nUser-agent: TagInspector\nUser-agent: TelegramBot\nUser-agent: Teoma\nUser-agent: trendictionbot\nUser-agent: trovitBot\nUser-agent: Tumblr\nUser-agent: twibble.io\nUser-agent: TwinWaveScanner\nUser-agent: um-LN\nUser-agent: VaultPress\nUser-agent: vercel-screenshot\nUser-agent: Viber\nUser-agent: VigLink\nUser-agent: VocusBot\nUser-agent: W3C-checklink\nUser-agent: W3C_Validator\nUser-agent: Wappalyzer\nUser-agent: WhatsApp\nUser-agent: woorankreview\nUser-agent: Y!J-ASR\nUser-agent: Y!J-BRJ/YATS crawler\nUser-agent: Y!J-BRU/VSIDX\nUser-agent: Y!J-BRW\nUser-agent: Y!J-BRY/YATSH crawler\nUser-agent: Y!J-BRZ/YATSHA crawler\nUser-agent: Y!J-MMP/dscv\nUser-agent: Y!J-WSC\nUser-agent: YaDirectFetcher\nUser-agent: Yahoo Ad monitoring\nUser-agent: Yahoo Link Preview\nUser-agent: Yahoo! Slurp\nUser-agent: Yahoo-Japan-ASR\nUser-agent: YahooCacheSystem\nUser-agent: YahooExternalCache\nUser-agent: yahoofeedseeker\nUser-agent: YahooMailProxy\nUser-agent: YaK\nUser-agent: Yandex\nUser-agent: Yeti\nUser-agent: YextBot\nUser-agent: YioopBot\nUser-agent: YJ-SAD\nAllow: /_ngjs/resource/*/get/\nAllow: /business/create/\nAllow: /resource/*/get/\nDisallow: /*/*/*/_tools/*\nDisallow: /*/*/*/more_ideas/\nDisallow: /*/*/_tools/*\nDisallow: /*/*/activity/*\nDisallow: /*/*/group/\nDisallow: /*/*/invite/\nDisallow: /*/*/more_ideas/*\nDisallow: /*/?*amp_client_id*\nDisallow: /*/?z=1\nDisallow: /*/__wishlist__/*\nDisallow: /*/_activities/*\nDisallow: /*/_activity/*\nDisallow: /*/_community/*\nDisallow: /*/_created/*\nDisallow: /*/_followers/*\nDisallow: /*/_following/*\nDisallow: /*/_quick_saves/*\nDisallow: /*/_saved/*\nDisallow: /*/_shop/*\nDisallow: /*/_shopping_list/*\nDisallow: /*/communities/\nDisallow: /*/discovered/*\nDisallow: /*/followers/*\nDisallow: /*/following/*\nDisallow: /*/likes/*\nDisallow: /*/organize/*\nDisallow: /*/pins/\nDisallow: /*/profile-about/\nDisallow: /*/scheduled-pin/*\nDisallow: /*/scheduled-pins/*\nDisallow: /*/story_pins/*\nDisallow: /*/tagged/*\nDisallow: /*/tried/*\nDisallow: /*/video_pins/*\nDisallow: /?show_error=true\nDisallow: /_/_/*\nDisallow: /_/_/pixel/*\nDisallow: /_ngjs/resource/\nDisallow: /about/*\nDisallow: /addboard/\nDisallow: /admin/\nDisallow: /age-appeal/*\nDisallow: /amp/*\nDisallow: /appealed-pin/\nDisallow: /autocomplete/\nDisallow: /board/*\nDisallow: /board_activity/*\nDisallow: /board_activity_comment/*\nDisallow: /bookmarklet/\nDisallow: /branded-pins/\nDisallow: /business/ad_create/\nDisallow: /business/business-access/*\nDisallow: /business/business-hierarchy/*\nDisallow: /business/business-manager/*\nDisallow: /business/catalogs/*\nDisallow: /business/commerce-integrations/*\nDisallow: /business/community-information/\nDisallow: /business/convert/\nDisallow: /business/create-linked-account/\nDisallow: /business/hub/\nDisallow: /business/invite/*\nDisallow: /business/onboarding/\nDisallow: /business/pin_create/\nDisallow: /business/referral/*\nDisallow: /business/site/\nDisallow: /business/switch-account/\nDisallow: /business/top-ads/\nDisallow: /business/update/\nDisallow: /business/verified-merchant/status/\nDisallow: /christmas-spotlight/\nDisallow: /close-account/\nDisallow: /communities/*\nDisallow: /community/*\nDisallow: /community_comment/*\nDisallow: /community_post/*\nDisallow: /connect/*/\nDisallow: /content-claiming/*\nDisallow: /conversation/*\nDisallow: /conversations/*\nDisallow: /convert-business/\nDisallow: /convert-personal/\nDisallow: /copyright-reporting/*\nDisallow: /create_pin/\nDisallow: /creative-studio/\nDisallow: /creative_editor/\nDisallow: /creator-spotlight/*\nDisallow: /creator_hub/_earn\nDisallow: /creator_hub/_partnerships\nDisallow: /csrf_error/\nDisallow: /ct.html\nDisallow: /deactivate-account/\nDisallow: /delete_post/\nDisallow: /discover/*\nDisallow: /edit/*\nDisallow: /editorial/*/*/\nDisallow: /email/*\nDisallow: /email_verification_error/\nDisallow: /engagement/*\nDisallow: /explore/*/?p=*\nDisallow: /facebook/\nDisallow: /fb.html\nDisallow: /follow_board/\nDisallow: /follow_user/\nDisallow: /google_gdn.html\nDisallow: /google_search.html\nDisallow: /hanukkah-spotlight/\nDisallow: /holiday-spotlight/\nDisallow: /holidays-celebrations/\nDisallow: /homefeed/\nDisallow: /homefeed/*\nDisallow: /idea-ads-tool/*\nDisallow: /idea-pin-builder/*\nDisallow: /idea-pin-invite/\nDisallow: /install-shuffles/\nDisallow: /invite/\nDisallow: /invited/\nDisallow: /join/?next=*\nDisallow: /join/signup/?next=*\nDisallow: /kwanzaa-spotlight/\nDisallow: /lens-search/*\nDisallow: /linked/*\nDisallow: /live-session/*\nDisallow: /login/?next=*\nDisallow: /logout/\nDisallow: /m/\nDisallow: /new_invite/\nDisallow: /news_hub/*\nDisallow: /notifications/\nDisallow: /oauth/line/*\nDisallow: /oauth/login/*\nDisallow: /oauth_error/\nDisallow: /offsite/\nDisallow: /parental-consent/confirmation/\nDisallow: /parental-passcode/\nDisallow: /password/reset/\nDisallow: /password/reset/*\nDisallow: /pin-builder/*\nDisallow: /pin-communities/*\nDisallow: /pin-creation-tool/*\nDisallow: /pin-editor/\nDisallow: /pin/*/activities\nDisallow: /pin/*/activity/\nDisallow: /pin/*/analytics\nDisallow: /pin/*/brand-catalog/\nDisallow: /pin/*/brand-catalog/*\nDisallow: /pin/*/comments\nDisallow: /pin/*/email\nDisallow: /pin/*/feedback\nDisallow: /pin/*/idea-pin/related-products/*\nDisallow: /pin/*/likes\nDisallow: /pin/*/promote\nDisallow: /pin/*/reaction\nDisallow: /pin/*/related-products\nDisallow: /pin/*/related-products/*\nDisallow: /pin/*/repin\nDisallow: /pin/*/repins\nDisallow: /pin/*/share\nDisallow: /pin/*/shop-the-look/*\nDisallow: /pin/*/tag\nDisallow: /pin/*/takes\nDisallow: /pin/*/visual-search\nDisallow: /pin/*/visual-shop\nDisallow: /pin/create/*\nDisallow: /pin/find/\nDisallow: /pin/find?url=*\nDisallow: /pin_shop/*\nDisallow: /pinterest-predicts/\nDisallow: /pinterest-wellbeing/*\nDisallow: /pinterestwellbeing/*\nDisallow: /place/*\nDisallow: /prefs/\nDisallow: /product-catalogs/\nDisallow: /product-catalogs/*\nDisallow: /product-feed/\nDisallow: /pw/*\nDisallow: /quiz/*\nDisallow: /recently-viewed/\nDisallow: /refresh_stored_accounts\nDisallow: /refresh_token\nDisallow: /register/\nDisallow: /reports-and-violations/\nDisallow: /request-data/\nDisallow: /resource/\nDisallow: /safe-redirect/\nDisallow: /search/\nDisallow: /search?\nDisallow: /secure/*\nDisallow: /sendHeaderEarly/\nDisallow: /settings/*\nDisallow: /shop_pins/*\nDisallow: /signup/*\nDisallow: /socialmanager/*\nDisallow: /source/*\nDisallow: /sso/*\nDisallow: /story-pin-builder/*\nDisallow: /story-pin-invite/\nDisallow: /story_feed/\nDisallow: /story_pins/*\nDisallow: /suspension-appeal/*\nDisallow: /sw-shell.html\nDisallow: /taggingtool/*\nDisallow: /thanksgiving-spotlight/\nDisallow: /theme/*\nDisallow: /tv/*\nDisallow: /unauth-profile/\nDisallow: /unlink/\nDisallow: /upload-lens-image\nDisallow: /upload-lens-image-history\nDisallow: /upload-shuffle-draft\nDisallow: /v1/urls/count.json\nDisallow: /v3/pidgets/log/\nDisallow: /website/confirm/\nDisallow: /website/confirm/download*\nDisallow: /welcome/\nDisallow: /your-shop/\nDisallow: /your-shop/*\n\nUser-agent: Googlebot\nAllow: /_ngjs/resource/*/get/\nAllow: /business/create/\nAllow: /resource/*/get/\nDisallow: /*/*/*/_tools/*\nDisallow: /*/*/*/more_ideas/\nDisallow: /*/*/_tools/*\nDisallow: /*/*/activity/*\nDisallow: /*/*/group/\nDisallow: /*/*/invite/\nDisallow: /*/*/more_ideas/*\nDisallow: /*/?*amp_client_id*\nDisallow: /*/?z=1\nDisallow: /*/__wishlist__/*\nDisallow: /*/_activities/*\nDisallow: /*/_activity/*\nDisallow: /*/_community/*\nDisallow: /*/_created/*\nDisallow: /*/_followers/*\nDisallow: /*/_following/*\nDisallow: /*/_quick_saves/*\nDisallow: /*/_saved/*\nDisallow: /*/_shop/*\nDisallow: /*/_shopping_list/*\nDisallow: /*/communities/\nDisallow: /*/discovered/*\nDisallow: /*/followers/*\nDisallow: /*/following/*\nDisallow: /*/likes/*\nDisallow: /*/organize/*\nDisallow: /*/pins/\nDisallow: /*/profile-about/\nDisallow: /*/scheduled-pin/*\nDisallow: /*/scheduled-pins/*\nDisallow: /*/story_pins/*\nDisallow: /*/tagged/*\nDisallow: /*/tried/*\nDisallow: /*/video_pins/*\nDisallow: /?show_error=true\nDisallow: /_/_/*\nDisallow: /_/_/pixel/*\nDisallow: /_ngjs/resource/\nDisallow: /about/*\nDisallow: /addboard/\nDisallow: /admin/\nDisallow: /age-appeal/*\nDisallow: /amp/*\nDisallow: /appealed-pin/\nDisallow: /autocomplete/\nDisallow: /board/*\nDisallow: /board_activity/*\nDisallow: /board_activity_comment/*\nDisallow: /bookmarklet/\nDisallow: /branded-pins/\nDisallow: /business/ad_create/\nDisallow: /business/business-access/*\nDisallow: /business/business-hierarchy/*\nDisallow: /business/business-manager/*\nDisallow: /business/catalogs/*\nDisallow: /business/commerce-integrations/*\nDisallow: /business/community-information/\nDisallow: /business/convert/\nDisallow: /business/create-linked-account/\nDisallow: /business/hub/\nDisallow: /business/invite/*\nDisallow: /business/onboarding/\nDisallow: /business/pin_create/\nDisallow: /business/referral/*\nDisallow: /business/site/\nDisallow: /business/switch-account/\nDisallow: /business/top-ads/\nDisallow: /business/update/\nDisallow: /business/verified-merchant/status/\nDisallow: /christmas-spotlight/\nDisallow: /close-account/\nDisallow: /communities/*\nDisallow: /community/*\nDisallow: /community_comment/*\nDisallow: /community_post/*\nDisallow: /connect/*/\nDisallow: /content-claiming/*\nDisallow: /conversation/*\nDisallow: /conversations/*\nDisallow: /convert-business/\nDisallow: /convert-personal/\nDisallow: /copyright-reporting/*\nDisallow: /create_pin/\nDisallow: /creative-studio/\nDisallow: /creative_editor/\nDisallow: /creator-spotlight/*\nDisallow: /creator_hub/_earn\nDisallow: /creator_hub/_partnerships\nDisallow: /csrf_error/\nDisallow: /ct.html\nDisallow: /deactivate-account/\nDisallow: /delete_post/\nDisallow: /discover/*\nDisallow: /edit/*\nDisallow: /editorial/*/*/\nDisallow: /email/*\nDisallow: /email_verification_error/\nDisallow: /engagement/*\nDisallow: /explore/*\nDisallow: /explore/*/?p=*\nDisallow: /facebook/\nDisallow: /fb.html\nDisallow: /follow_board/\nDisallow: /follow_user/\nDisallow: /google_gdn.html\nDisallow: /google_search.html\nDisallow: /hanukkah-spotlight/\nDisallow: /holiday-spotlight/\nDisallow: /holidays-celebrations/\nDisallow: /homefeed/\nDisallow: /homefeed/*\nDisallow: /idea-ads-tool/*\nDisallow: /idea-pin-builder/*\nDisallow: /idea-pin-invite/\nDisallow: /install-shuffles/\nDisallow: /invite/\nDisallow: /invited/\nDisallow: /join/?next=*\nDisallow: /join/signup/?next=*\nDisallow: /kwanzaa-spotlight/\nDisallow: /lens-search/*\nDisallow: /linked/*\nDisallow: /live-session/*\nDisallow: /login/?next=*\nDisallow: /logout/\nDisallow: /m/\nDisallow: /new_invite/\nDisallow: /news_hub/*\nDisallow: /notifications/\nDisallow: /oauth/line/*\nDisallow: /oauth/login/*\nDisallow: /oauth_error/\nDisallow: /offsite/\nDisallow: /parental-consent/confirmation/\nDisallow: /parental-passcode/\nDisallow: /password/reset/\nDisallow: /password/reset/*\nDisallow: /pin-builder/*\nDisallow: /pin-communities/*\nDisallow: /pin-creation-tool/*\nDisallow: /pin-editor/\nDisallow: /pin/*/activities\nDisallow: /pin/*/activity/\nDisallow: /pin/*/analytics\nDisallow: /pin/*/brand-catalog/\nDisallow: /pin/*/brand-catalog/*\nDisallow: /pin/*/comments\nDisallow: /pin/*/email\nDisallow: /pin/*/feedback\nDisallow: /pin/*/idea-pin/related-products/*\nDisallow: /pin/*/likes\nDisallow: /pin/*/promote\nDisallow: /pin/*/reaction\nDisallow: /pin/*/related-products\nDisallow: /pin/*/related-products/*\nDisallow: /pin/*/repin\nDisallow: /pin/*/repins\nDisallow: /pin/*/share\nDisallow: /pin/*/shop-the-look/*\nDisallow: /pin/*/tag\nDisallow: /pin/*/takes\nDisallow: /pin/*/visual-search\nDisallow: /pin/*/visual-shop\nDisallow: /pin/create/*\nDisallow: /pin/find/\nDisallow: /pin/find?url=*\nDisallow: /pin_shop/*\nDisallow: /pinterest-predicts/\nDisallow: /pinterest-wellbeing/*\nDisallow: /pinterestwellbeing/*\nDisallow: /place/*\nDisallow: /prefs/\nDisallow: /product-catalogs/\nDisallow: /product-catalogs/*\nDisallow: /product-feed/\nDisallow: /pw/*\nDisallow: /quiz/*\nDisallow: /recently-viewed/\nDisallow: /refresh_stored_accounts\nDisallow: /refresh_token\nDisallow: /register/\nDisallow: /reports-and-violations/\nDisallow: /request-data/\nDisallow: /resource/\nDisallow: /safe-redirect/\nDisallow: /search/\nDisallow: /search?\nDisallow: /secure/*\nDisallow: /sendHeaderEarly/\nDisallow: /settings/*\nDisallow: /shop_pins/*\nDisallow: /signup/*\nDisallow: /socialmanager/*\nDisallow: /source/*\nDisallow: /sso/*\nDisallow: /story-pin-builder/*\nDisallow: /story-pin-invite/\nDisallow: /story_feed/\nDisallow: /story_pins/*\nDisallow: /suspension-appeal/*\nDisallow: /sw-shell.html\nDisallow: /taggingtool/*\nDisallow: /thanksgiving-spotlight/\nDisallow: /theme/*\nDisallow: /tv/*\nDisallow: /unauth-profile/\nDisallow: /unlink/\nDisallow: /upload-lens-image\nDisallow: /upload-lens-image-history\nDisallow: /upload-shuffle-draft\nDisallow: /v1/urls/count.json\nDisallow: /v3/pidgets/log/\nDisallow: /website/confirm/\nDisallow: /website/confirm/download*\nDisallow: /welcome/\nDisallow: /your-shop/\nDisallow: /your-shop/*\n\nUser-agent: Twitterbot\nAllow: /_ngjs/resource/*/get/\nAllow: /business/create/\nAllow: /pin/*/feedback\nAllow: /pin/*/sent\nAllow: /resource/*/get/\nDisallow: /*/*/*/_tools/*\nDisallow: /*/*/*/more_ideas/\nDisallow: /*/*/_tools/*\nDisallow: /*/*/activity/*\nDisallow: /*/*/group/\nDisallow: /*/*/invite/\nDisallow: /*/*/more_ideas/*\nDisallow: /*/?*amp_client_id*\nDisallow: /*/?z=1\nDisallow: /*/__wishlist__/*\nDisallow: /*/_activities/*\nDisallow: /*/_activity/*\nDisallow: /*/_community/*\nDisallow: /*/_created/*\nDisallow: /*/_followers/*\nDisallow: /*/_following/*\nDisallow: /*/_quick_saves/*\nDisallow: /*/_saved/*\nDisallow: /*/_shop/*\nDisallow: /*/_shopping_list/*\nDisallow: /*/communities/\nDisallow: /*/discovered/*\nDisallow: /*/followers/*\nDisallow: /*/following/*\nDisallow: /*/likes/*\nDisallow: /*/organize/*\nDisallow: /*/pins/\nDisallow: /*/profile-about/\nDisallow: /*/scheduled-pin/*\nDisallow: /*/scheduled-pins/*\nDisallow: /*/story_pins/*\nDisallow: /*/tagged/*\nDisallow: /*/tried/*\nDisallow: /*/video_pins/*\nDisallow: /?show_error=true\nDisallow: /_/_/*\nDisallow: /_/_/pixel/*\nDisallow: /_ngjs/resource/\nDisallow: /about/*\nDisallow: /addboard/\nDisallow: /admin/\nDisallow: /age-appeal/*\nDisallow: /amp/*\nDisallow: /appealed-pin/\nDisallow: /autocomplete/\nDisallow: /board/*\nDisallow: /board_activity/*\nDisallow: /board_activity_comment/*\nDisallow: /bookmarklet/\nDisallow: /branded-pins/\nDisallow: /business/ad_create/\nDisallow: /business/business-access/*\nDisallow: /business/business-hierarchy/*\nDisallow: /business/business-manager/*\nDisallow: /business/catalogs/*\nDisallow: /business/commerce-integrations/*\nDisallow: /business/community-information/\nDisallow: /business/convert/\nDisallow: /business/create-linked-account/\nDisallow: /business/hub/\nDisallow: /business/invite/*\nDisallow: /business/onboarding/\nDisallow: /business/pin_create/\nDisallow: /business/referral/*\nDisallow: /business/site/\nDisallow: /business/switch-account/\nDisallow: /business/top-ads/\nDisallow: /business/update/\nDisallow: /business/verified-merchant/status/\nDisallow: /christmas-spotlight/\nDisallow: /close-account/\nDisallow: /communities/*\nDisallow: /community/*\nDisallow: /community_comment/*\nDisallow: /community_post/*\nDisallow: /connect/*/\nDisallow: /content-claiming/*\nDisallow: /conversation/*\nDisallow: /conversations/*\nDisallow: /convert-business/\nDisallow: /convert-personal/\nDisallow: /copyright-reporting/*\nDisallow: /create_pin/\nDisallow: /creative-studio/\nDisallow: /creative_editor/\nDisallow: /creator-spotlight/*\nDisallow: /creator_hub/_earn\nDisallow: /creator_hub/_partnerships\nDisallow: /csrf_error/\nDisallow: /ct.html\nDisallow: /deactivate-account/\nDisallow: /delete_post/\nDisallow: /discover/*\nDisallow: /edit/*\nDisallow: /editorial/*/*/\nDisallow: /email/*\nDisallow: /email_verification_error/\nDisallow: /engagement/*\nDisallow: /explore/*\nDisallow: /explore/*/?p=*\nDisallow: /facebook/\nDisallow: /fb.html\nDisallow: /follow_board/\nDisallow: /follow_user/\nDisallow: /google_gdn.html\nDisallow: /google_search.html\nDisallow: /hanukkah-spotlight/\nDisallow: /holiday-spotlight/\nDisallow: /holidays-celebrations/\nDisallow: /homefeed/\nDisallow: /homefeed/*\nDisallow: /idea-ads-tool/*\nDisallow: /idea-pin-builder/*\nDisallow: /idea-pin-invite/\nDisallow: /install-shuffles/\nDisallow: /invite/\nDisallow: /invited/\nDisallow: /join/?next=*\nDisallow: /join/signup/?next=*\nDisallow: /kwanzaa-spotlight/\nDisallow: /lens-search/*\nDisallow: /linked/*\nDisallow: /live-session/*\nDisallow: /login/?next=*\nDisallow: /logout/\nDisallow: /m/\nDisallow: /new_invite/\nDisallow: /news_hub/*\nDisallow: /notifications/\nDisallow: /oauth/line/*\nDisallow: /oauth/login/*\nDisallow: /oauth_error/\nDisallow: /offsite/\nDisallow: /parental-consent/confirmation/\nDisallow: /parental-passcode/\nDisallow: /password/reset/\nDisallow: /password/reset/*\nDisallow: /pin-builder/*\nDisallow: /pin-communities/*\nDisallow: /pin-creation-tool/*\nDisallow: /pin-editor/\nDisallow: /pin/*/activities\nDisallow: /pin/*/activity/\nDisallow: /pin/*/analytics\nDisallow: /pin/*/brand-catalog/\nDisallow: /pin/*/brand-catalog/*\nDisallow: /pin/*/comments\nDisallow: /pin/*/email\nDisallow: /pin/*/idea-pin/related-products/*\nDisallow: /pin/*/likes\nDisallow: /pin/*/promote\nDisallow: /pin/*/reaction\nDisallow: /pin/*/related-products\nDisallow: /pin/*/related-products/*\nDisallow: /pin/*/repin\nDisallow: /pin/*/repins\nDisallow: /pin/*/share\nDisallow: /pin/*/shop-the-look/*\nDisallow: /pin/*/tag\nDisallow: /pin/*/takes\nDisallow: /pin/*/visual-search\nDisallow: /pin/*/visual-shop\nDisallow: /pin/create/*\nDisallow: /pin/find/\nDisallow: /pin/find?url=*\nDisallow: /pin_shop/*\nDisallow: /pinterest-predicts/\nDisallow: /pinterest-wellbeing/*\nDisallow: /pinterestwellbeing/*\nDisallow: /place/*\nDisallow: /prefs/\nDisallow: /product-catalogs/\nDisallow: /product-catalogs/*\nDisallow: /product-feed/\nDisallow: /pw/*\nDisallow: /quiz/*\nDisallow: /recently-viewed/\nDisallow: /refresh_stored_accounts\nDisallow: /refresh_token\nDisallow: /register/\nDisallow: /reports-and-violations/\nDisallow: /request-data/\nDisallow: /resource/\nDisallow: /safe-redirect/\nDisallow: /search/\nDisallow: /search?\nDisallow: /secure/*\nDisallow: /sendHeaderEarly/\nDisallow: /settings/*\nDisallow: /shop_pins/*\nDisallow: /signup/*\nDisallow: /socialmanager/*\nDisallow: /source/*\nDisallow: /sso/*\nDisallow: /story-pin-builder/*\nDisallow: /story-pin-invite/\nDisallow: /story_feed/\nDisallow: /story_pins/*\nDisallow: /suspension-appeal/*\nDisallow: /sw-shell.html\nDisallow: /taggingtool/*\nDisallow: /thanksgiving-spotlight/\nDisallow: /theme/*\nDisallow: /tv/*\nDisallow: /unauth-profile/\nDisallow: /unlink/\nDisallow: /upload-lens-image\nDisallow: /upload-lens-image-history\nDisallow: /upload-shuffle-draft\nDisallow: /v1/urls/count.json\nDisallow: /v3/pidgets/log/\nDisallow: /website/confirm/\nDisallow: /website/confirm/download*\nDisallow: /welcome/\nDisallow: /your-shop/\nDisallow: /your-shop/*\n\nUser-agent: Mediapartners-Google\nUser-agent: AmazonAdBot\nAllow: /homefeed/\nAllow: /homefeed/*\nAllow: /pin/\nAllow: /search/*\nDisallow: /\n\nUser-agent: Google-adstxt\nUser-agent: IAB-Tech-Lab\nAllow: /ads.txt\nAllow: /app-ads.txt\nDisallow: /\n\nUser-agent: adidxbot\nUser-agent: AdsBot-Google\nUser-agent: AdsBot-Google-Mobile\nUser-agent: AxonBot\nUser-agent: Google Web Preview\nUser-agent: Google-Ads-Overview\nUser-agent: Google-AdWords-Express\nUser-agent: Google-Adwords-Instant\nUser-agent: LinkedInBot\nUser-agent: meta-externalads\nUser-agent: SnapchatAds\nUser-agent: Taboolabot\nAllow: /\n\nUser-agent: YandexAdditional\nUser-agent: YandexAdditionalBot\nDisallow: /\n\nUser-agent: *\nDisallow: /\n\nSitemap: https://www.pinterest.com/v3_sitemaps/seasonal_events_sitemap_20261206_christmas_season_begins_www.pinterest.com.xml\nSitemap: https://www.pinterest.com/v3_sitemaps/qem_klp_vlm_navboost_sample_medium_www.pinterest.com.xml\nSitemap: https://www.pinterest.com/v3_sitemaps/seasonal_events_sitemap_20261226_national_day_of_thanksgiving_www.pinterest.com.xml\nSitemap: https://www.pinterest.com/v3_sitemaps/qem_klp_vlm_navboost_sample_high_www.pinterest.com.xml\nSitemap: https://www.pinterest.com/v3_sitemaps/ideas_hub_v_expansion_sitemap_batch_2_www.pinterest.com.xml\nSitemap: https://www.pinterest.com/v3_sitemaps/ideas_hub_expansion_v2_sitemap_www.pinterest.com.xml\nSitemap: https://www.pinterest.com/v3_sitemaps/seasonal_events_sitemap_20261206_cayman_thanksgiving_www.pinterest.com.xml\nSitemap: https://www.pinterest.com/v3_sitemaps/today_article_sitemap_www.pinterest.com.xml\nSitemap: https://www.pinterest.com/v3_sitemaps/shopping_sitemap_www.pinterest.com.xml\nSitemap: https://www.pinterest.com/v3_sitemaps/ideas_hub_world_cup_terms_sitemap_www.pinterest.com.xml\nSitemap: https://www.pinterest.com/v3_sitemaps/seasonal_events_sitemap_20261231_western_new_year_s_eve_www.pinterest.com.xml\nSitemap: https://www.pinterest.com/v3_sitemaps/seasonal_events_sitemap_20261224_catholic_protestant_christmas_eve_www.pinterest.com.xml\nSitemap: https://www.pinterest.com/v3_sitemaps/seasonal_events_sitemap_20261129_first_advent_sunday_www.pinterest.com.xml\nSitemap: https://www.pinterest.com/v3_sitemaps/qem_klp_vlm_vase_medium_www.pinterest.com.xml\nSitemap: https://www.pinterest.com/v3_sitemaps/qem_klp_vlm_vase_low_www.pinterest.com.xml\nSitemap: https://www.pinterest.com/v3_sitemaps/ideas_hub_vlm_terms_sitemap_auto_publish_www.pinterest.com.xml\nSitemap: https://www.pinterest.com/v3_sitemaps/seasonal_events_sitemap_20261231_international_new_year_s_eve_www.pinterest.com.xml\nSitemap: https://www.pinterest.com/v3_sitemaps/ideas_hub_v_expansion_sitemap_www.pinterest.com.xml\nSitemap: https://www.pinterest.com/v3_sitemaps/seasonal_events_sitemap_20261129_first_sunday_of_advent_www.pinterest.com.xml\nSitemap: https://www.pinterest.com/v3_sitemaps/qem_klp_vlm_vase_high_www.pinterest.com.xml\nSitemap: https://www.pinterest.com/v3_sitemaps/seasonal_events_sitemap_20261231_new_year_holiday_www.pinterest.com.xml\nSitemap: https://www.pinterest.com/v3_sitemaps/seasonal_events_sitemap_20261225_christmas_www.pinterest.com.xml\nSitemap: https://www.pinterest.com/v3_sitemaps/seasonal_events_sitemap_20261224_christmas_eve_from_2pm_www.pinterest.com.xml\nSitemap: https://www.pinterest.com/v3_sitemaps/seasonal_events_sitemap_20261127_day_after_thanksgiving_www.pinterest.com.xml\nSitemap: https://www.pinterest.com/v3_sitemaps/ideas_hub_gsc_terms_sitemap_www.pinterest.com.xml\nSitemap: https://www.pinterest.com/v3_sitemaps/seasonal_events_sitemap_20261031_halloween_www.pinterest.com.xml\nSitemap: https://www.pinterest.com/v3_sitemaps/ideas_hub_answers_terms_sitemap_www.pinterest.com.xml\nSitemap: https://www.pinterest.com/v3_sitemaps/seasonal_events_sitemap_20261231_new_year_s_eve_from_2pm_www.pinterest.com.xml\nSitemap: https://www.pinterest.com/v3_sitemaps/seasonal_events_sitemap_20261126_thanksgiving_www.pinterest.com.xml\nSitemap: https://www.pinterest.com/v3_sitemaps/seasonal_events_sitemap_20261224_christmas_eve_joint_holiday_www.pinterest.com.xml\nSitemap: https://www.pinterest.com/v3_sitemaps/qem_klp_vlm_navboost_sample_low_www.pinterest.com.xml\nSitemap: https://www.pinterest.com/v3_sitemaps/image_pin_high_quality_tier_0_www.pinterest.com.xml\nSitemap: https://www.pinterest.com/v3_sitemaps/board_link_sitemap_priority_segment_2026q3_www.pinterest.com.xml\nSitemap: https://www.pinterest.com/v3_sitemaps/seasonal_halloween_www.pinterest.com.xml\nSitemap: https://www.pinterest.com/v3_sitemaps/priority_segment_2026q4_www.pinterest.com.xml\nSitemap: https://www.pinterest.com/v3_sitemaps/video_pin_sitemap_www.pinterest.com.xml\nSitemap: https://www.pinterest.com/v3_sitemaps/board_link_sitemap_recent_engagement_tier_0_www.pinterest.com.xml\nSitemap: https://www.pinterest.com/v3_sitemaps/board_link_sitemap_high_cohesion_tier_0_www.pinterest.com.xml\nSitemap: https://www.pinterest.com/v3_sitemaps/image_pin_sitemap_recent_engagement_tier_0_www.pinterest.com.xml\nSitemap: https://www.pinterest.com/v3_sitemaps/board_link_sitemap_www.pinterest.com.xml\nSitemap: https://www.pinterest.com/v3_sitemaps/board_link_sitemap_tier_3_quality_uncrawled_www.pinterest.com.xml\nSitemap: https://www.pinterest.com/v3_sitemaps/board_link_sitemap_priority_segment_2026q4_www.pinterest.com.xml\nSitemap: https://www.pinterest.com/v3_sitemaps/page_quality_model_high_score_www.pinterest.com.xml\nSitemap: https://www.pinterest.com/v3_sitemaps/board_link_sitemap_high_cohesion_tier_1_www.pinterest.com.xml\nSitemap: https://www.pinterest.com/v3_sitemaps/board_link_sitemap_seasonal_halloween_www.pinterest.com.xml\nSitemap: https://www.pinterest.com/v3_sitemaps/image_pin_sitemap_recently_created_www.pinterest.com.xml\nSitemap: https://www.pinterest.com/v3_sitemaps/image_pin_sitemap_high_engagement_tier_1_www.pinterest.com.xml\nSitemap: https://www.pinterest.com/v3_sitemaps/product_pin_sitemap_www.pinterest.com.xml\nSitemap: https://www.pinterest.com/v3_sitemaps/image_pin_high_quality_tier_1_www.pinterest.com.xml\nSitemap: https://www.pinterest.com/v3_sitemaps/priority_segment_2026q3_www.pinterest.com.xml\nSitemap: https://www.pinterest.com/v3_sitemaps/board_link_sitemap_tier_2_quality_uncrawled_www.pinterest.com.xml\nSitemap: https://www.pinterest.com/v3_sitemaps/board_link_sitemap_tier_0_quality_uncrawled_www.pinterest.com.xml\nSitemap: https://www.pinterest.com/v3_sitemaps/board_link_sitemap_tier_1_quality_uncrawled_www.pinterest.com.xml\nSitemap: https://www.pinterest.com/v3_sitemaps/video_product_pin_sitemap_www.pinterest.com.xml\nSitemap: https://www.pinterest.com/v3_sitemaps/board_link_sitemap_recent_engagement_tier_1_www.pinterest.com.xml\nSitemap: https://www.pinterest.com/v3_sitemaps/user_link_sitemap_www.pinterest.com.xml\nSitemap: https://www.pinterest.com/v3_sitemaps/image_pin_sitemap_high_engagement_tier_0_www.pinterest.com.xml\nSitemap: https://www.pinterest.com/v3_sitemaps/gift_board_link_sitemap_www.pinterest.com.xml\nSitemap: https://www.pinterest.com/v3_sitemaps/image_pin_sitemap_recent_engagement_tier_1_www.pinterest.com.xml\nSitemap: https://www.pinterest.com/v3_sitemaps/image_pin_sitemap_www.pinterest.com.xml\nSitemap: https://www.pinterest.com/v3_sitemaps/seasonal_halloween_segment_www.pinterest.com.xml\nSitemap: https://www.pinterest.com/v3_sitemaps/christmas_pin_reinforce_www.pinterest.com.xml\nSitemap: https://www.pinterest.com/v3_sitemaps/new_years_eve_pin_newset_www.pinterest.com.xml\nSitemap: https://www.pinterest.com/v3_sitemaps/christmas_pin_newset_www.pinterest.com.xml\nSitemap: https://www.pinterest.com/v3_sitemaps/new_years_eve_pin_reinforce_www.pinterest.com.xml\nSitemap: https://www.pinterest.com/v3_sitemaps/seasonal_events_sitemap_www.pinterest.com.xml\nSitemap: https://www.pinterest.com/v3_sitemaps/answers_page_sitemap_www.pinterest.com.xml","agents":{"GPTBot":false,"OAI-SearchBot":false,"ChatGPT-User":false,"ClaudeBot":false,"Claude-SearchBot":false,"PerplexityBot":false,"Google-Extended":false,"Applebot-Extended":true,"CCBot":false,"Bytespider":false,"Meta-ExternalAgent":false,"Amazonbot":false}},"sitemap":{"status":404,"content_type":"text/html","bytes":2987,"sha256":"e18183bf0fda3188ca987bfc10821ba58ed98904307d136d648ef1c400a56ffe","final_host":"www.pinterest.com","final_path":"/v3_sitemaps/seasonal_events_sitemap_20261206_christmas_season_begins_www.pinterest.com.xml","ms":77,"url":"https://www.pinterest.com/v3_sitemaps/seasonal_events_sitemap_20261206_christmas_season_begins_www.pinterest.com.xml","declared":true,"xml":false},"files":{"llms_txt":{"status":200,"content_type":"text/html","bytes":1518421,"sha256":"1e6fa3abbce0f6a2f96734d56ad394c1b4db92e3994585445898359be96fe9f0","final_host":"www.pinterest.com","final_path":"/llms.txt","ms":278,"path":"/llms.txt","html":true,"json":null,"present":false},"agents_md":{"status":200,"content_type":"text/html","bytes":1290419,"sha256":"9c21927d001e0df76cbc1e7cc476663cb51f69f128bb4ea8bf266463559b0818","final_host":"www.pinterest.com","final_path":"/agents.md","ms":281,"path":"/agents.md","html":true,"json":null,"present":false},"agent_skills":{"status":200,"content_type":"text/html","bytes":1345043,"sha256":"2ca896d65a5c9ca19d9b4a6a9fdfd050f9686eb088f1ad1cd835d534a41537ee","final_host":"www.pinterest.com","final_path":"/.well-known/agent-skills/index.json","ms":311,"path":"/.well-known/agent-skills/index.json","html":true,"json":null,"present":false},"mcp_card":{"status":200,"content_type":"text/html","bytes":1295912,"sha256":"c0bbf9e02d8bc35f8fef9b4baaa3c4a9addc756ed566761e5da05c7c006fe4d6","final_host":"www.pinterest.com","final_path":"/.well-known/mcp/server-card.json","ms":208,"path":"/.well-known/mcp/server-card.json","html":true,"json":null,"present":false},"api_catalog":{"status":200,"content_type":"text/html","bytes":1293864,"sha256":"a8ca8e7cbec4d992e3dbf176696c74f40395f7c4801bc8586f3763a22403a455","final_host":"www.pinterest.com","final_path":"/.well-known/api-catalog","ms":301,"path":"/.well-known/api-catalog","html":true,"json":null,"present":false},"a2a_card":{"status":200,"content_type":"text/html","bytes":1294035,"sha256":"e102da107155ce37fafbcb8df880605b0d0612b123bfa9a02fe8344e456016b5","final_host":"www.pinterest.com","final_path":"/.well-known/agent-card.json","ms":231,"path":"/.well-known/agent-card.json","html":true,"json":null,"present":false}},"rings":{"v":1,"candidates":[],"mcp":[],"a2a":null,"webmcp":{"model_context":false,"bridge":false,"mcp_json":false,"link_rel":false,"any":false},"handshakes":0,"get_path":{"status":200,"content_type":"text/html","ms":401,"error":null},"get_host":{"status":530,"content_type":"text/plain","ms":19,"error":null}},"sf":[4,0,0,0,0,0,1,0,1],"derived":{"reachable":true,"home_class":"answers","robots_served":true,"robots_sitemap":true,"content_signal":false,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":["GPTBot","OAI-SearchBot","ChatGPT-User","ClaudeBot","Claude-SearchBot","PerplexityBot","Google-Extended","CCBot","Bytespider","Meta-ExternalAgent","Amazonbot"],"blocks_any_ai":true,"files":{"llms_txt":false,"agents_md":false,"agent_skills":false,"mcp_card":false,"api_catalog":false,"a2a_card":false},"any_capability":false,"ai_refused":["gptbot","oaisearch","perplexity"],"ai_thin":[],"aipref":0,"aipref_measured":true,"sf":[4,0,0,0,0,0,1,0,1],"rings":{"mcp_declared":false,"mcp_live":false,"mcp_auth":false,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":3,"fh":"www.pinterest.com","fp_other_host":[],"fp":"415139f3e79f27eb","fp_bits":{}},"derive_v":5,"frame":"top1k","shard":4},{"v":1,"kind":"crawlcheck-census-record","run":43,"domain":"tiktok.com","rank":54,"measured_at":"2026-11-13T08:07:00.657Z","home":{"browser":{"status":200,"content_type":"text/html","bytes":175872,"sha256":"95b4ffca0e52de2bdf16f35744d8e2c642a8be56859b69cf1588bfcd232129cd","final_host":"www.tiktok.com","final_path":"/","ms":130,"identity":"Mobile browser","html":true},"gptbot":{"status":200,"content_type":"text/html","bytes":426979,"sha256":"e9e721ca2bd80b7eec27965025c635c644c93d3c31313e0056214d56c315665e","final_host":"www.tiktok.com","final_path":"/","ms":229,"identity":"GPTBot","html":true},"claudebot":{"status":200,"content_type":"text/html","bytes":378685,"sha256":"bed50f25a88af66cdabe336e283c9ae23fed3815a9f9b6ac0dd36806599d0a71","final_host":"www.tiktok.com","final_path":"/explore","ms":462,"identity":"ClaudeBot","html":true},"oaisearch":{"status":200,"content_type":"text/html","bytes":424523,"sha256":"7f439070ae92d85998689c77625ce23c1e9b24bf89923ed88ec3b1b71b009502","final_host":"www.tiktok.com","final_path":"/","ms":227,"identity":"OAI-SearchBot","html":true},"perplexity":{"status":200,"content_type":"text/html","bytes":346735,"sha256":"dcd882a6b01524a9c5ebea2f66384356ecdd7850ef04bc8d50c31e1ed0229ad1","final_host":"www.tiktok.com","final_path":"/","ms":176,"identity":"PerplexityBot","html":true}},"robots":{"status":200,"content_type":"text/plain","bytes":1567,"sha256":"835705a2ecb862f49a813b3aa4061e0edfefbb8318316b371f8fa2dba4018802","final_host":"www.tiktok.com","final_path":"/robots.txt","ms":73,"html":false,"text":"User-agent: Baiduspider\nUser-agent: 360Spider\nUser-agent: Sogouspider\nUser-agent: Yisouspider\nUser-agent: PetalBot\nUser-agent: GPTBot\nUser-agent: OAI-SearchBot\nUser-agent: anthropic-ai\nUser-agent: ClaudeBot\nUser-agent: PerplexityBot\nUser-agent: Google-Extended\nUser-agent: Applebot-Extended\nUser-agent: meta-externalagent\nUser-agent: DuckAssistBot\nUser-agent: ChatGPT-User\nUser-agent: Gemini-Deep-Research\nUser-agent: GoogleAgent-Mariner\nUser-agent: Google-NotebookLM\nUser-agent: CCBot\nUser-agent: Claude-User\nUser-agent: Claude-SearchBot\nUser-agent: Perplexity-User\nUser-agent: AI2Bot\nUser-agent: MistralAI-User\nUser-agent: Bytespider\nDisallow: /\n\nUser-agent: *\nAllow: /foryou\nAllow: /discover\nAllow: /about\nAllow: /legal\nAllow: /safety\nAllow: /forgood\nAllow: /community-guidelines\nAllow: /tag\nAllow: /amp\nAllow: /transparency\nAllow: /share\nAllow: /music\nDisallow: /inapp\nDisallow: /auth\nDisallow: /embed/@\nDisallow: /embed/v2\nDisallow: /embed/curated\nDisallow: /link\nDisallow: */directory/\nDisallow: /search/video?\nDisallow: /search/user?q=\nDisallow: /shop/view/product/\nDisallow: /sgtm/g/collect\nDisallow: /api/share/settings\nDisallow: /api/recommend/embed_videos\nDisallow: /discover/trending/detail/\nDisallow: /search?\nDisallow: /search/live?\n\nUser-agent: Bingbot\nDisallow: /discover\nDisallow: /inapp\nDisallow: /auth\nDisallow: /embed/@\nDisallow: /embed/v2\nDisallow: /embed/curated\nDisallow: /link\nDisallow: */directory/\nDisallow: /shop/view/product/\nDisallow: /sgtm/g/collect\nDisallow: /api/share/settings\nDisallow: /api/recommend/embed_videos\nDisallow: /search?\n","agents":{"GPTBot":false,"OAI-SearchBot":false,"ChatGPT-User":false,"ClaudeBot":false,"Claude-SearchBot":false,"PerplexityBot":false,"Google-Extended":false,"Applebot-Extended":false,"CCBot":false,"Bytespider":false,"Meta-ExternalAgent":false,"Amazonbot":true}},"sitemap":{"status":200,"content_type":"application/xml","bytes":521367,"sha256":"4600fe62aabaab0b53b4c57ab1cddea537a052ad6b57445ea86d840252e5f603","final_host":"www.tiktok.com","final_path":"/tos/node/sitemaps/sitemap.xml","ms":1579,"url":"https://tiktok.com/sitemap.xml","declared":false,"xml":true},"files":{"llms_txt":{"status":200,"content_type":"text/html","bytes":33737,"sha256":"78336b13b569abc3d4e59c0d897793c12c48449c8521810279f943e932d9c2d5","final_host":"www.tiktok.com","final_path":"/404","ms":76,"path":"/llms.txt","html":true,"json":null,"present":false},"agents_md":{"status":200,"content_type":"text/html","bytes":33737,"sha256":"78336b13b569abc3d4e59c0d897793c12c48449c8521810279f943e932d9c2d5","final_host":"www.tiktok.com","final_path":"/404","ms":71,"path":"/agents.md","html":true,"json":null,"present":false},"agent_skills":{"status":200,"content_type":"text/html","bytes":33737,"sha256":"78336b13b569abc3d4e59c0d897793c12c48449c8521810279f943e932d9c2d5","final_host":"www.tiktok.com","final_path":"/404","ms":77,"path":"/.well-known/agent-skills/index.json","html":true,"json":null,"present":false},"mcp_card":{"status":200,"content_type":"text/html","bytes":33737,"sha256":"78336b13b569abc3d4e59c0d897793c12c48449c8521810279f943e932d9c2d5","final_host":"www.tiktok.com","final_path":"/404","ms":103,"path":"/.well-known/mcp/server-card.json","html":true,"json":null,"present":false},"api_catalog":{"status":200,"content_type":"text/html","bytes":33737,"sha256":"78336b13b569abc3d4e59c0d897793c12c48449c8521810279f943e932d9c2d5","final_host":"www.tiktok.com","final_path":"/404","ms":139,"path":"/.well-known/api-catalog","html":true,"json":null,"present":false},"a2a_card":{"status":200,"content_type":"text/html","bytes":33737,"sha256":"78336b13b569abc3d4e59c0d897793c12c48449c8521810279f943e932d9c2d5","final_host":"www.tiktok.com","final_path":"/404","ms":143,"path":"/.well-known/agent-card.json","html":true,"json":null,"present":false}},"rings":{"v":1,"candidates":[],"mcp":[],"a2a":null,"webmcp":{"model_context":false,"bridge":false,"mcp_json":false,"link_rel":false,"any":false},"handshakes":0,"get_path":{"status":200,"content_type":"text/html","ms":93,"error":null},"get_host":{"status":530,"content_type":"text/plain","ms":7,"error":null}},"sf":[4,0,0,0,0,0,1,0,0],"derived":{"reachable":true,"home_class":"answers","robots_served":true,"robots_sitemap":false,"content_signal":false,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":["GPTBot","OAI-SearchBot","ChatGPT-User","ClaudeBot","Claude-SearchBot","PerplexityBot","Google-Extended","Applebot-Extended","CCBot","Bytespider","Meta-ExternalAgent"],"blocks_any_ai":true,"files":{"llms_txt":false,"agents_md":false,"agent_skills":false,"mcp_card":false,"api_catalog":false,"a2a_card":false},"any_capability":false,"ai_refused":[],"ai_thin":[],"aipref":0,"aipref_measured":true,"sf":[4,0,0,0,0,0,1,0,0],"rings":{"mcp_declared":false,"mcp_live":false,"mcp_auth":false,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":3,"fh":"www.tiktok.com","fp_other_host":[]},"derive_v":5,"frame":"top1k","shard":5},{"v":1,"kind":"crawlcheck-census-record","run":43,"domain":"whatsapp.com","rank":55,"measured_at":"2026-11-13T08:07:15.177Z","home":{"browser":{"status":400,"content_type":"text/html","bytes":6278,"sha256":"5ebbd8e6c02a58e8cd7c348665c804d6a90915348c861e093b159b365c1cbff7","final_host":"www.whatsapp.com","final_path":"/","ms":250,"identity":"Mobile browser","html":true},"gptbot":{"status":200,"content_type":"text/html","bytes":277980,"sha256":"b475e1744d2edce6b7a7c4a4a59922cdd0337752fc3117079a8916abff2b8782","final_host":"www.whatsapp.com","final_path":"/","ms":833,"identity":"GPTBot","html":true,"fp":"035ecaf7bb074b16"},"claudebot":{"status":200,"content_type":"text/html","bytes":277927,"sha256":"2ca169a7908eaa515717487a6e1d79fa105a6890992469e2a1401335a085c498","final_host":"www.whatsapp.com","final_path":"/","ms":286,"identity":"ClaudeBot","html":true,"fp":"035ecaf7bb074b16"},"oaisearch":{"status":200,"content_type":"text/html","bytes":277984,"sha256":"94f30167ef1508f1eaf2daa5a37e90829fbbe2aac654303c5d700b8efc791f6b","final_host":"www.whatsapp.com","final_path":"/","ms":232,"identity":"OAI-SearchBot","html":true,"fp":"035ecaf7bb074b16"},"perplexity":{"status":200,"content_type":"text/html","bytes":288100,"sha256":"06e2c0e472b028c22c5bd7114fe386203ae5e6a293ee86b12e6fe6346d9af88c","final_host":"www.whatsapp.com","final_path":"/","ms":676,"identity":"PerplexityBot","html":true,"fp":"015ecbb7a3070b56"}},"robots":{"status":200,"content_type":"text/plain","bytes":1225,"sha256":"6d6febc5de572ecaca22e0169030eb4e4cd41ecaf12af30d64f2cdd9c000d0cb","final_host":"www.whatsapp.com","final_path":"/robots.txt","ms":189,"html":false,"text":"# Notice: Collection of data on Facebook through automated means is\n# prohibited unless you have express written permission from Facebook\n# and may only be conducted for the limited purpose contained in said\n# permission.\n# See: http://www.facebook.com/apps/site_scraping_tos_terms.php\n\nUser-agent: __usage_ai-training\nDisallow: /\n\nUser-agent: Amazonbot\nDisallow: /\n\nUser-agent: Applebot-Extended\nDisallow: /\n\nUser-agent: Brightbot\nDisallow: /\n\nUser-agent: ClaudeBot\nDisallow: /\n\nUser-agent: Google-Extended\nDisallow: /\n\nUser-agent: GPTBot\nDisallow: /\n\nUser-agent: PerplexityBot\nDisallow: /\n\nUser-agent: PetalBot\nDisallow: /\n\nUser-agent: Scrapy\nDisallow: /\n\nUser-agent: uptimerobot\nDisallow: /\n\nUser-agent: viberbot\nDisallow: /\n\nUser-agent: YaK\nDisallow: /\n\nUser-agent: Yandex\nDisallow: /\n\nUser-agent: Yeti\nDisallow: /\n\nUser-agent: facebookexternalhit\nAllow: *\n\nUser-agent: meta-externalads\nAllow: *\n\nUser-agent: *\nDisallow: /*cursor=\nDisallow: /*fb_comment_id=\nDisallow: /ajax/\nDisallow: /tealium/\nDisallow: /intern/\nDisallow: /internal/\nDisallow: /login/\nDisallow: /oidc/callback/\nDisallow: /*.php\nDisallow: /android/third_party\nDisallow: /settings\n\nSitemap: https://www.whatsapp.com/sitemap/www_whatsapp_com_sitemap.xml.gz","agents":{"GPTBot":false,"OAI-SearchBot":true,"ChatGPT-User":true,"ClaudeBot":false,"Claude-SearchBot":true,"PerplexityBot":false,"Google-Extended":false,"Applebot-Extended":false,"CCBot":true,"Bytespider":true,"Meta-ExternalAgent":true,"Amazonbot":false}},"sitemap":null,"files":null,"derived":{"reachable":false,"home_class":"refused_us","robots_served":false,"robots_sitemap":false,"content_signal":false,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":[],"blocks_any_ai":false,"files":{"llms_txt":false,"agents_md":false,"agent_skills":false,"mcp_card":false,"api_catalog":false,"a2a_card":false},"any_capability":false,"ai_refused":[],"ai_thin":[],"rings":{"mcp_declared":false,"mcp_live":false,"mcp_auth":false,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":3,"fh":"www.whatsapp.com","fp_other_host":[]},"derive_v":5,"frame":"top1k","shard":6},{"v":1,"kind":"crawlcheck-census-record","run":43,"domain":"googledomains.com","rank":56,"measured_at":"2026-11-13T08:08:20.375Z","home":{"browser":{"status":200,"content_type":"text/html","bytes":5354,"sha256":"ad6f90b7571c6a7ef0b13f5aeece4b6fe246851c2ae36b105fd30ff42b80bbb6","final_host":"domains.google","final_path":"/","ms":666,"identity":"Mobile browser","html":true,"fp":"e51811cd1864f4d8"},"gptbot":{"status":200,"content_type":"text/html","bytes":5354,"sha256":"e695b1d8af62a6e91ec2b007f244014e4ace673391a354748a6b8af2bfe4337c","final_host":"domains.google","final_path":"/","ms":624,"identity":"GPTBot","html":true,"fp":"e51811cd1864f4d8"},"claudebot":{"status":200,"content_type":"text/html","bytes":5354,"sha256":"55f7b4669420b0d5c7830bacaf51b987dc03287620cf272f7293d747ee0ee054","final_host":"domains.google","final_path":"/","ms":586,"identity":"ClaudeBot","html":true,"fp":"e51811cd1864f4d8"},"oaisearch":{"status":200,"content_type":"text/html","bytes":5354,"sha256":"d3d56216e100a0cfffc22e6775dd38ded6def97bb84de2438583b5bc1d50adfe","final_host":"domains.google","final_path":"/","ms":555,"identity":"OAI-SearchBot","html":true,"fp":"e51811cd1864f4d8"},"perplexity":{"status":200,"content_type":"text/html","bytes":5354,"sha256":"ab62b1d7486ff1458c8fb8f3f74bb6f58c703dea88c8d877d7db110a7fb77de5","final_host":"domains.google","final_path":"/","ms":553,"identity":"PerplexityBot","html":true,"fp":"e51811cd1864f4d8"}},"robots":{"status":200,"content_type":"text/html","bytes":5354,"sha256":"036002394d4625f0e7db8c50b062c82e3585631df038adac14da9c1a5a572c7a","final_host":"domains.google","final_path":"/","ms":523,"html":true,"text":null,"agents":null},"sitemap":{"status":200,"content_type":"text/html","bytes":5354,"sha256":"276d7a28b34625e1a3467d02519e2fb52ca7a7c095ca5040f903a8e1949523d8","final_host":"domains.google","final_path":"/","ms":61,"url":"https://googledomains.com/sitemap.xml","declared":false,"xml":false},"files":{"llms_txt":{"status":200,"content_type":"text/html","bytes":5354,"sha256":"98c512bddd6ee0f47288bdda311c2ce0919b090a7a8441acf598797731e74120","final_host":"domains.google","final_path":"/","ms":61,"path":"/llms.txt","html":true,"json":null,"present":false},"agents_md":{"status":200,"content_type":"text/html","bytes":5354,"sha256":"f52277db53fca44d1624aaffe8b2d2ffe85454d65d74aaf6cf24066e00411a0c","final_host":"domains.google","final_path":"/","ms":60,"path":"/agents.md","html":true,"json":null,"present":false},"agent_skills":{"status":200,"content_type":"text/html","bytes":5354,"sha256":"0c79b2f0532ec9d5588c33f0a8c699457672b92a051823c8dced2b2911933433","final_host":"domains.google","final_path":"/","ms":57,"path":"/.well-known/agent-skills/index.json","html":true,"json":null,"present":false},"mcp_card":{"status":200,"content_type":"text/html","bytes":5354,"sha256":"028521b20f49dfdf30a806b642cf6ec2d39c98e5e38fc65ca77f6eca393ba315","final_host":"domains.google","final_path":"/","ms":59,"path":"/.well-known/mcp/server-card.json","html":true,"json":null,"present":false},"api_catalog":{"status":200,"content_type":"text/html","bytes":5354,"sha256":"7be2310741a028cdb23a3e4f4f5146f1eee62fa7dd1774f57ec2b2aeb70c094e","final_host":"domains.google","final_path":"/","ms":57,"path":"/.well-known/api-catalog","html":true,"json":null,"present":false},"a2a_card":{"status":200,"content_type":"text/html","bytes":5354,"sha256":"4af22ad96eef7c25e9ca4c87ca987641b627a3c02dad3870a323f5ef08639226","final_host":"domains.google","final_path":"/","ms":62,"path":"/.well-known/agent-card.json","html":true,"json":null,"present":false}},"rings":{"v":1,"candidates":[],"mcp":[],"a2a":null,"webmcp":{"model_context":false,"bridge":false,"mcp_json":false,"link_rel":false,"any":false},"handshakes":0,"get_path":{"status":200,"content_type":"text/html","ms":52,"error":null},"get_host":{"status":530,"content_type":"text/plain","ms":81,"error":null}},"sf":[1,0,0,0,0,0,1,0,0],"derived":{"reachable":true,"home_class":"answers","robots_served":false,"robots_sitemap":false,"content_signal":false,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":[],"blocks_any_ai":false,"files":{"llms_txt":false,"agents_md":false,"agent_skills":false,"mcp_card":false,"api_catalog":false,"a2a_card":false},"any_capability":false,"ai_refused":[],"ai_thin":[],"sf":[1,0,0,0,0,0,1,0,0],"rings":{"mcp_declared":false,"mcp_live":false,"mcp_auth":false,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":3,"fh":"domains.google","fp_other_host":[],"fp":"e51811cd1864f4d8","fp_bits":{"gptbot":0,"claudebot":0,"oaisearch":0,"perplexity":0}},"derive_v":5,"frame":"top1k","shard":7},{"v":1,"kind":"crawlcheck-census-record","run":43,"domain":"cloudflare.net","rank":57,"measured_at":"2026-11-13T07:56:55.238Z","home":{"browser":{"status":200,"content_type":"text/html","bytes":70943,"sha256":"ccbf170b00251fc3a9a4e89e4ddf9f10ee8ec90598e10e1cfb3b3c6b1a3f3c30","final_host":"cloudflare.net","final_path":"/home/default.aspx","ms":300,"identity":"Mobile browser","html":true,"fp":"c118113eb6f046ff"},"gptbot":{"status":403,"content_type":"text/html","bytes":5585,"sha256":"61d403558c86e5497f8faf288776b2d3f6a6b9a70343053c0c51831eb5c129e4","final_host":"cloudflare.net","final_path":"/","ms":7,"identity":"GPTBot","html":true},"claudebot":{"status":200,"content_type":"text/html","bytes":70943,"sha256":"833c889216505bab12c98db636f116b613fefdfacf5f2b634bf0ef009e5c4164","final_host":"cloudflare.net","final_path":"/home/default.aspx","ms":290,"identity":"ClaudeBot","html":true,"fp":"c118113eb6f046ff"},"oaisearch":{"status":200,"content_type":"text/html","bytes":70943,"sha256":"53824df78e0b4e40a714bd2cb8f0e605efd656d7d4a80bc7aaf1c4753c086c30","final_host":"cloudflare.net","final_path":"/home/default.aspx","ms":199,"identity":"OAI-SearchBot","html":true,"fp":"c118113eb6f046ff"},"perplexity":{"status":200,"content_type":"text/html","bytes":70943,"sha256":"e1a5c9eab539cb6629f3ad0bb66de5198d7f3994f97dbb419c5e18c709efe04c","final_host":"cloudflare.net","final_path":"/home/default.aspx","ms":28,"identity":"PerplexityBot","html":true,"fp":"c118113eb6f046ff"}},"robots":{"status":200,"content_type":"text/plain","bytes":207,"sha256":"de73d32142f114b8d18bbfd7fa5779ccee5f7a1753873402cfdab59444834ccd","final_host":"cloudflare.net","final_path":"/robots.txt","ms":126,"html":false,"text":"User-agent: *\r\nCrawl-delay: 10\r\nAllow: /\r\n\r\nUser-agent: Eluta.ca\r\nCrawl-delay: 60\r\nAllow: /\r\n\r\nUser-agent: MS Search 6.0 Robot\r\nCrawl-delay: 60\r\nAllow: /\r\n\r\nUser-agent: gsa-crawler\r\nCrawl-delay: 60\r\nAllow: /","agents":{"GPTBot":true,"OAI-SearchBot":true,"ChatGPT-User":true,"ClaudeBot":true,"Claude-SearchBot":true,"PerplexityBot":true,"Google-Extended":true,"Applebot-Extended":true,"CCBot":true,"Bytespider":true,"Meta-ExternalAgent":true,"Amazonbot":true}},"sitemap":{"status":404,"content_type":"text/html","bytes":1286,"sha256":"049e1b638af15c4f3525aed1b4be5d3fd6fe361d999601cbc978dfb34a8c3285","final_host":"cloudflare.net","final_path":"/sitemap.xml","ms":132,"url":"https://cloudflare.net/sitemap.xml","declared":false,"xml":false},"files":{"llms_txt":{"status":404,"content_type":"text/html","bytes":1286,"sha256":"049e1b638af15c4f3525aed1b4be5d3fd6fe361d999601cbc978dfb34a8c3285","final_host":"cloudflare.net","final_path":"/llms.txt","ms":134,"path":"/llms.txt","html":true,"json":null,"present":false},"agents_md":{"status":404,"content_type":"text/html","bytes":1286,"sha256":"049e1b638af15c4f3525aed1b4be5d3fd6fe361d999601cbc978dfb34a8c3285","final_host":"cloudflare.net","final_path":"/agents.md","ms":197,"path":"/agents.md","html":true,"json":null,"present":false},"agent_skills":{"status":200,"content_type":"text/html","bytes":11,"sha256":"ee172faa9c17e8e5f4b63ef56d6d6863a48c223de152be43781305c99e143200","final_host":"cloudflare.net","final_path":"/.well-known/agent-skills/index.json","ms":160,"path":"/.well-known/agent-skills/index.json","html":true,"json":null,"present":false},"mcp_card":{"status":200,"content_type":"text/html","bytes":11,"sha256":"ee172faa9c17e8e5f4b63ef56d6d6863a48c223de152be43781305c99e143200","final_host":"cloudflare.net","final_path":"/.well-known/mcp/server-card.json","ms":160,"path":"/.well-known/mcp/server-card.json","html":true,"json":null,"present":false},"api_catalog":{"status":200,"content_type":"text/html","bytes":11,"sha256":"ee172faa9c17e8e5f4b63ef56d6d6863a48c223de152be43781305c99e143200","final_host":"cloudflare.net","final_path":"/.well-known/api-catalog","ms":316,"path":"/.well-known/api-catalog","html":true,"json":null,"present":false},"a2a_card":{"status":200,"content_type":"text/html","bytes":11,"sha256":"ee172faa9c17e8e5f4b63ef56d6d6863a48c223de152be43781305c99e143200","final_host":"cloudflare.net","final_path":"/.well-known/agent-card.json","ms":168,"path":"/.well-known/agent-card.json","html":true,"json":null,"present":false}},"rings":{"v":1,"candidates":[],"mcp":[],"a2a":null,"webmcp":{"model_context":false,"bridge":false,"mcp_json":false,"link_rel":false,"any":false},"handshakes":0,"get_path":{"status":404,"content_type":"text/html","ms":287,"error":null},"get_host":{"status":530,"content_type":"text/plain","ms":8,"error":null}},"sf":[4,0,0,0,0,0,2,0,0],"derived":{"reachable":true,"home_class":"answers","robots_served":true,"robots_sitemap":false,"content_signal":false,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":[],"blocks_any_ai":false,"files":{"llms_txt":false,"agents_md":false,"agent_skills":false,"mcp_card":false,"api_catalog":false,"a2a_card":false},"any_capability":false,"ai_refused":["gptbot"],"ai_thin":[],"aipref":0,"aipref_measured":true,"sf":[4,0,0,0,0,0,2,0,0],"rings":{"mcp_declared":false,"mcp_live":false,"mcp_auth":false,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":3,"fh":"cloudflare.net","fp_other_host":[],"fp":"c118113eb6f046ff","fp_bits":{"claudebot":0,"oaisearch":0,"perplexity":0}},"derive_v":5,"frame":"top1k","shard":0},{"v":1,"kind":"crawlcheck-census-record","run":43,"domain":"goo.gl","rank":58,"measured_at":"2026-11-13T07:57:55.549Z","home":{"browser":{"status":400,"content_type":"text/html","bytes":9233,"sha256":"467f85f08ed95e4d207f06ae2f39969cbf6c9c25a18a0929a0835ee367404d81","final_host":"goo.gl","final_path":"/","ms":1181,"identity":"Mobile browser","html":true},"gptbot":{"status":400,"content_type":"text/html","bytes":9233,"sha256":"467f85f08ed95e4d207f06ae2f39969cbf6c9c25a18a0929a0835ee367404d81","final_host":"goo.gl","final_path":"/","ms":1186,"identity":"GPTBot","html":true},"claudebot":{"status":400,"content_type":"text/html","bytes":9233,"sha256":"467f85f08ed95e4d207f06ae2f39969cbf6c9c25a18a0929a0835ee367404d81","final_host":"goo.gl","final_path":"/","ms":1029,"identity":"ClaudeBot","html":true},"oaisearch":{"status":400,"content_type":"text/html","bytes":9233,"sha256":"467f85f08ed95e4d207f06ae2f39969cbf6c9c25a18a0929a0835ee367404d81","final_host":"goo.gl","final_path":"/","ms":1175,"identity":"OAI-SearchBot","html":true},"perplexity":{"status":400,"content_type":"text/html","bytes":9233,"sha256":"467f85f08ed95e4d207f06ae2f39969cbf6c9c25a18a0929a0835ee367404d81","final_host":"goo.gl","final_path":"/","ms":1075,"identity":"PerplexityBot","html":true}},"robots":{"status":400,"content_type":"text/html","bytes":9258,"sha256":"bae5d66fbe371f66a934935346906d25bb35df6fb4cd5504a359cd324e89ea11","final_host":"goo.gl","final_path":"/robots.txt","ms":392,"html":true,"text":null,"agents":null},"sitemap":null,"files":null,"derived":{"reachable":false,"home_class":"refused_us","robots_served":false,"robots_sitemap":false,"content_signal":false,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":[],"blocks_any_ai":false,"files":{"llms_txt":false,"agents_md":false,"agent_skills":false,"mcp_card":false,"api_catalog":false,"a2a_card":false},"any_capability":false,"ai_refused":[],"ai_thin":[],"rings":{"mcp_declared":false,"mcp_live":false,"mcp_auth":false,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":3,"fh":"goo.gl","fp_other_host":[]},"derive_v":5,"frame":"top1k","shard":1},{"v":1,"kind":"crawlcheck-census-record","run":43,"domain":"yahoo.com","rank":59,"measured_at":"2026-11-13T07:58:58.649Z","home":{"browser":{"status":200,"content_type":"text/html","bytes":615779,"sha256":"796ccd5d17e7d222199f14e32779936d5cb3b0f29e1d6604cb636d79a2ac0e39","final_host":"www.yahoo.com","final_path":"/","ms":498,"identity":"Mobile browser","html":true,"fp":"36cd10ca4537a8a8"},"gptbot":{"status":429,"content_type":"text/html","bytes":23,"sha256":"0d24c98db98d3b4a87b9626a026e7d72c99754ab6bd88211a17bc372350b7d89","final_host":"www.yahoo.com","final_path":"/","ms":72,"identity":"GPTBot","html":true},"claudebot":{"status":404,"content_type":"text/html","bytes":1616,"sha256":"17ea8d8c4f070d0b6fd4c08935fcaae7553667f43acfdd8cb587a090d867e581","final_host":"yahoo.com","final_path":"/","ms":43,"identity":"ClaudeBot","html":true},"oaisearch":{"status":404,"content_type":"text/html","bytes":1616,"sha256":"17ea8d8c4f070d0b6fd4c08935fcaae7553667f43acfdd8cb587a090d867e581","final_host":"yahoo.com","final_path":"/","ms":167,"identity":"OAI-SearchBot","html":true},"perplexity":{"status":404,"content_type":"text/html","bytes":1616,"sha256":"17ea8d8c4f070d0b6fd4c08935fcaae7553667f43acfdd8cb587a090d867e581","final_host":"yahoo.com","final_path":"/","ms":151,"identity":"PerplexityBot","html":true}},"robots":{"status":200,"content_type":"text/plain","bytes":1464,"sha256":"dfcd18f101783bde40c3bdca97bb90dbea1653321fd8b57f81cf5a6a6dc00b4b","final_host":"www.yahoo.com","final_path":"/robots.txt","ms":196,"html":false,"text":"User-agent: *\nDisallow: /info/p.gif\nDisallow: /p/\nDisallow: /r/\nDisallow: /bin/\nDisallow: /caas/\nDisallow: /blank.html\nDisallow: /includes/\nDisallow: /_td_api\nDisallow: /tdv2_fp\nDisallow: /nel_ms\nDisallow: /fp_ms\nDisallow: /sports_fp_ms\nDisallow: /search_ms\nDisallow: /_tdpp_api\nDisallow: /_remote\nDisallow: /_multiremote\nDisallow: /_tdhl_api\nDisallow: /digest\nDisallow: /fpjs\nDisallow: /myjs\nDisallow: /news/m/\n\nUser-agent: ADmantX\nUser-agent: AlphaBot\nUser-agent: anthropic-ai\nUser-agent: AwarioRssBot\nUser-agent: AwarioSmartBot\nUser-agent: BLEXBot\nUser-agent: Buzzbot\nUser-agent: Bytespider\nUser-agent: CCBot\nUser-agent: ChatGPT-User\nUser-agent: claritybot\nUser-agent: Claude-Web\nUser-agent: ClaudeBot\nUser-agent: cohere-ai\nUser-agent: Diffbot\nUser-agent: FacebookBot\nUser-agent: FriendlyCrawler\nUser-agent: Google-Extended\nUser-agent: GPTBot\nUser-agent: huggingface\nUser-agent: ImagesiftBot\nUser-agent: img2dataset\nUser-agent: magpie-crawler\nUser-agent: Meltwater\nUser-agent: Neevabot\nUser-agent: news-please\nUser-agent: NewsNow\nUser-agent: Nutch\nUser-agent: omgili\nUser-agent: omgilibot\nUser-agent: panscient.com\nUser-agent: Perplexity-ai\nUser-agent: PerplexityBot\nUser-agent: PetalBot\nUser-agent: PiplBot\nUser-agent: scoop.it\nUser-agent: Scrapy\nUser-agent: Seekr\nUser-agent: SentiBot\nUser-agent: SeznamBot\nUser-agent: TurnitinBot\nUser-agent: YouBot\nUser-agent: ZumBot\nDisallow: /\n\nUser-agent: Claude-SearchBot\nUser-agent: OAI-SearchBot\nDisallow: */articles/\n","agents":{"GPTBot":false,"OAI-SearchBot":true,"ChatGPT-User":false,"ClaudeBot":false,"Claude-SearchBot":true,"PerplexityBot":false,"Google-Extended":false,"Applebot-Extended":true,"CCBot":false,"Bytespider":false,"Meta-ExternalAgent":true,"Amazonbot":true}},"sitemap":{"status":404,"content_type":"text/html","bytes":4594,"sha256":"06b53af9dc98c6045eff330c00260c3224017da44d5f8d6b1085e0b5c25876ec","final_host":"www.yahoo.com","final_path":"/sitemap.xml","ms":85,"url":"https://yahoo.com/sitemap.xml","declared":false,"xml":false},"files":{"llms_txt":{"status":404,"content_type":"text/html","bytes":4591,"sha256":"c214c99ec6c735a57c40c762ea5754f56638aa73d42cfdfe7c75bcfbe0ae37ba","final_host":"www.yahoo.com","final_path":"/llms.txt","ms":90,"path":"/llms.txt","html":true,"json":null,"present":false},"agents_md":{"status":404,"content_type":"text/html","bytes":4592,"sha256":"d20e026d56c837de35f626f519cda2450f3f2217ab7d5b3f70f85f4fb0681514","final_host":"www.yahoo.com","final_path":"/agents.md","ms":114,"path":"/agents.md","html":true,"json":null,"present":false},"agent_skills":{"status":404,"content_type":"text/html","bytes":4618,"sha256":"310295c6520067d9b40d2046743ab67d12eceab2f640da7777c9a1ac1923f13e","final_host":"www.yahoo.com","final_path":"/.well-known/agent-skills/index.json","ms":105,"path":"/.well-known/agent-skills/index.json","html":true,"json":null,"present":false},"mcp_card":{"status":404,"content_type":"text/html","bytes":4615,"sha256":"5216945394a448c781d0f530e3866d8e1a995c5701b43a2517c13a407dd0d344","final_host":"www.yahoo.com","final_path":"/.well-known/mcp/server-card.json","ms":89,"path":"/.well-known/mcp/server-card.json","html":true,"json":null,"present":false},"api_catalog":{"status":404,"content_type":"text/html","bytes":4606,"sha256":"7b946ed214e3848d9b5acadd0f8da0996a15f57e0492a612d9c0767a851cdd46","final_host":"www.yahoo.com","final_path":"/.well-known/api-catalog","ms":52,"path":"/.well-known/api-catalog","html":true,"json":null,"present":false},"a2a_card":{"status":404,"content_type":"text/html","bytes":4610,"sha256":"65eda19e119b8fc45e62af8717938deca9f5b10ade113afe439be081e27ddad9","final_host":"www.yahoo.com","final_path":"/.well-known/agent-card.json","ms":84,"path":"/.well-known/agent-card.json","html":true,"json":null,"present":false}},"rings":{"v":1,"candidates":[],"mcp":[],"a2a":null,"webmcp":{"model_context":false,"bridge":false,"mcp_json":false,"link_rel":false,"any":false},"handshakes":0,"get_path":{"status":404,"content_type":"text/html","ms":66,"error":null},"get_host":{"status":530,"content_type":"text/plain","ms":8,"error":null}},"sf":[4,0,0,0,0,0,0,0,0],"derived":{"reachable":true,"home_class":"answers","robots_served":true,"robots_sitemap":false,"content_signal":false,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":["GPTBot","ChatGPT-User","ClaudeBot","PerplexityBot","Google-Extended","CCBot","Bytespider"],"blocks_any_ai":true,"files":{"llms_txt":false,"agents_md":false,"agent_skills":false,"mcp_card":false,"api_catalog":false,"a2a_card":false},"any_capability":false,"ai_refused":["gptbot","claudebot","oaisearch","perplexity"],"ai_thin":[],"aipref":0,"aipref_measured":true,"sf":[4,0,0,0,0,0,0,0,0],"rings":{"mcp_declared":false,"mcp_live":false,"mcp_auth":false,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":3,"fh":"www.yahoo.com","fp_other_host":[],"fp":"36cd10ca4537a8a8","fp_bits":{}},"derive_v":5,"frame":"top1k","shard":2},{"v":1,"kind":"crawlcheck-census-record","run":43,"domain":"googlesyndication.com","rank":60,"measured_at":"2026-11-13T07:59:19.049Z","home":{"browser":{"status":200,"content_type":"text/html","bytes":206802,"sha256":"c84c6576be123e3a6651ec265dbf52104572effd452b3b2bf56d70257ec21795","final_host":"www.google.com","final_path":"/","ms":142,"identity":"Mobile browser","html":true},"gptbot":{"status":200,"content_type":"text/html","bytes":31492,"sha256":"7e347a40c70f7cbe3cde7f3c639609e30795f4f5b84e8cfb45d9c9c420c77b54","final_host":"www.google.com","final_path":"/","ms":84,"identity":"GPTBot","html":true},"claudebot":{"status":200,"content_type":"text/html","bytes":90844,"sha256":"c864eeaa1dd1bb97f8019f69a1c442fd4ccbf3d89943a24fe273d5348c6534b3","final_host":"www.google.com","final_path":"/","ms":91,"identity":"ClaudeBot","html":true},"oaisearch":{"status":200,"content_type":"text/html","bytes":31529,"sha256":"a40478d54bf60800a55d7be82ce852a0fa5b33590d4a6d353f748afffa9d8134","final_host":"www.google.com","final_path":"/","ms":68,"identity":"OAI-SearchBot","html":true},"perplexity":{"status":200,"content_type":"text/html","bytes":89947,"sha256":"480746ed9e375526070ed64abe72c866f27e655357c65f00b186a45e0dffec9c","final_host":"www.google.com","final_path":"/","ms":69,"identity":"PerplexityBot","html":true}},"robots":{"status":404,"content_type":"text/html","bytes":1571,"sha256":"6adee83a3eba87d30fdca3997d27097697f2767805f222a0f78c3f04392004d2","final_host":"googlesyndication.com","final_path":"/robots.txt","ms":132,"html":true,"text":null,"agents":null},"sitemap":{"status":404,"content_type":"text/html","bytes":1572,"sha256":"ab724c219c1a70a2f94a80452f31fa802b754912b60e32c40cd39666dd23910a","final_host":"googlesyndication.com","final_path":"/sitemap.xml","ms":107,"url":"https://googlesyndication.com/sitemap.xml","declared":false,"xml":false},"files":{"llms_txt":{"status":404,"content_type":"text/html","bytes":1569,"sha256":"47d71ed470dfe1ca13eb2af8fa019b95871a1bd1381a518ed71359e69b9c3cc7","final_host":"googlesyndication.com","final_path":"/llms.txt","ms":112,"path":"/llms.txt","html":true,"json":null,"present":false},"agents_md":{"status":404,"content_type":"text/html","bytes":1570,"sha256":"f08e316602488d86581ce85b3076a13c82fc3f97cdac3deba23801fb66ca908d","final_host":"googlesyndication.com","final_path":"/agents.md","ms":119,"path":"/agents.md","html":true,"json":null,"present":false},"agent_skills":{"status":404,"content_type":"text/html","bytes":1596,"sha256":"eee9f4f975e761645c6c385ec645439385500d0062bff7170ed09a8835d6f92b","final_host":"googlesyndication.com","final_path":"/.well-known/agent-skills/index.json","ms":122,"path":"/.well-known/agent-skills/index.json","html":true,"json":null,"present":false},"mcp_card":{"status":404,"content_type":"text/html","bytes":1593,"sha256":"e5db4a1d5f7596a7d62e54e192ab48bbf195116fa33a2c7e160fdaf6131945a8","final_host":"googlesyndication.com","final_path":"/.well-known/mcp/server-card.json","ms":122,"path":"/.well-known/mcp/server-card.json","html":true,"json":null,"present":false},"api_catalog":{"status":404,"content_type":"text/html","bytes":1584,"sha256":"7de25ab2e3ce288cf29d538f80ed48bf0d5759f87b613e29ed345db4675698aa","final_host":"googlesyndication.com","final_path":"/.well-known/api-catalog","ms":113,"path":"/.well-known/api-catalog","html":true,"json":null,"present":false},"a2a_card":{"status":404,"content_type":"text/html","bytes":1588,"sha256":"64d2189fbe1531e7fb6ab2e6040a932d87ae1feb811d4781df85ea7ef6b4d651","final_host":"googlesyndication.com","final_path":"/.well-known/agent-card.json","ms":107,"path":"/.well-known/agent-card.json","html":true,"json":null,"present":false}},"rings":{"v":1,"candidates":[],"mcp":[],"a2a":null,"webmcp":{"model_context":false,"bridge":false,"mcp_json":false,"link_rel":false,"any":false},"handshakes":0,"get_path":{"status":404,"content_type":"text/html","ms":99,"error":null},"get_host":{"status":530,"content_type":"text/plain","ms":6,"error":null}},"sf":[1,0,0,0,0,0,0,0,0],"derived":{"reachable":true,"home_class":"answers","robots_served":false,"robots_sitemap":false,"content_signal":false,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":[],"blocks_any_ai":false,"files":{"llms_txt":false,"agents_md":false,"agent_skills":false,"mcp_card":false,"api_catalog":false,"a2a_card":false},"any_capability":false,"ai_refused":[],"ai_thin":["gptbot","claudebot","oaisearch","perplexity"],"sf":[1,0,0,0,0,0,0,0,0],"rings":{"mcp_declared":false,"mcp_live":false,"mcp_auth":false,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":3,"fh":"www.google.com","fp_other_host":[]},"derive_v":5,"frame":"top1k","shard":3},{"v":1,"kind":"crawlcheck-census-record","run":43,"domain":"gvt1.com","rank":61,"measured_at":"2026-11-13T08:05:49.397Z","home":{"browser":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"gvt1.com","final_path":"/","ms":65,"identity":"Mobile browser","html":false},"gptbot":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"gvt1.com","final_path":"/","ms":28,"identity":"GPTBot","html":false},"claudebot":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"gvt1.com","final_path":"/","ms":8,"identity":"ClaudeBot","html":false},"oaisearch":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"gvt1.com","final_path":"/","ms":2,"identity":"OAI-SearchBot","html":false},"perplexity":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"gvt1.com","final_path":"/","ms":1,"identity":"PerplexityBot","html":false}},"robots":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"gvt1.com","final_path":"/robots.txt","ms":1,"html":false,"text":null,"agents":null},"sitemap":null,"files":null,"derived":{"reachable":false,"home_class":"no_site","robots_served":false,"robots_sitemap":false,"content_signal":false,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":[],"blocks_any_ai":false,"files":{"llms_txt":false,"agents_md":false,"agent_skills":false,"mcp_card":false,"api_catalog":false,"a2a_card":false},"any_capability":false,"ai_refused":[],"ai_thin":[],"rings":{"mcp_declared":false,"mcp_live":false,"mcp_auth":false,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":3,"fh":"gvt1.com","fp_other_host":[]},"derive_v":5,"frame":"top1k","shard":4},{"v":1,"kind":"crawlcheck-census-record","run":43,"domain":"roblox.com","rank":62,"measured_at":"2026-11-13T08:07:00.657Z","home":{"browser":{"status":200,"content_type":"text/html","bytes":62044,"sha256":"7c7c5c2d46b5060c237cb218313752486e926f7eb7421c9bcd770dabe83fc728","final_host":"www.roblox.com","final_path":"/","ms":92,"identity":"Mobile browser","html":true},"gptbot":{"status":200,"content_type":"text/html","bytes":109963,"sha256":"587369d7a3c070619a54be549263b7fd52ffa20580dde4f314a58e9c741597ff","final_host":"www.roblox.com","final_path":"/","ms":82,"identity":"GPTBot","html":true},"claudebot":{"status":200,"content_type":"text/html","bytes":109963,"sha256":"f2042ad218b1b08855ce5a7674ac08dec286d7ffe7c88f05747e59655b4f7fe0","final_host":"www.roblox.com","final_path":"/","ms":62,"identity":"ClaudeBot","html":true},"oaisearch":{"status":200,"content_type":"text/html","bytes":109963,"sha256":"f0db2f2932d9366b5786e977579f44332400d2d72df00205317bef389755015b","final_host":"www.roblox.com","final_path":"/","ms":67,"identity":"OAI-SearchBot","html":true},"perplexity":{"status":200,"content_type":"text/html","bytes":109959,"sha256":"de8aca0f9265da332bcd27c2b8fd9c160b04fd1703ad3fb4320081e1b3e353d3","final_host":"www.roblox.com","final_path":"/","ms":67,"identity":"PerplexityBot","html":true}},"robots":{"status":200,"content_type":"text/plain","bytes":2281,"sha256":"f5c52b75ce949831a002a1572f68cc23e8a22145c0685a41c95f87c6205f3dab","final_host":"www.roblox.com","final_path":"/robots.txt","ms":62,"html":false,"text":"user-agent: *\n\ndisallow: *.ashx\ndisallow: /abusereport/\ndisallow: /*/abusereport/\ndisallow: /admi/\ndisallow: /ads/\ndisallow: /catalog/contents\ndisallow: /catalog/html\ndisallow: /client-status\ndisallow: /data/\ndisallow: /error/\ndisallow: /forum/\ndisallow: /Forum/\ndisallow: /game/report-event\ndisallow: /game/report-stats\ndisallow: /ide/clienttoolbox\ndisallow: /javascript/\ndisallow: /reports/\ndisallow: /thumbnail/resolve-hash\ndisallow: /thumbnail_holder/g\ndisallow: /viewapp/\ndisallow: /error.html\ndisallow: /offline.html\ndisallow: /badges/\ndisallow: /*/badges/\ndisallow: /*.aspx$\ndisallow: /upgrades/checkout/success?giftPurchaseId=*\ndisallow: /report-abuse\ndisallow: /*/report-abuse\n\nSitemap: https://www.roblox.com/sitemap.xml\nSitemap: https://www.roblox.com/sitemap-games.xml\nSitemap: https://www.roblox.com/sitemap-ar.xml\nSitemap: https://www.roblox.com/sitemap-games-ar.xml\nSitemap: https://www.roblox.com/sitemap-de.xml\nSitemap: https://www.roblox.com/sitemap-games-de.xml\nSitemap: https://www.roblox.com/sitemap-es.xml\nSitemap: https://www.roblox.com/sitemap-games-es.xml\nSitemap: https://www.roblox.com/sitemap-fr.xml\nSitemap: https://www.roblox.com/sitemap-games-fr.xml\nSitemap: https://www.roblox.com/sitemap-hi.xml\nSitemap: https://www.roblox.com/sitemap-games-hi.xml\nSitemap: https://www.roblox.com/sitemap-id.xml\nSitemap: https://www.roblox.com/sitemap-games-id.xml\nSitemap: https://www.roblox.com/sitemap-it.xml\nSitemap: https://www.roblox.com/sitemap-games-it.xml\nSitemap: https://www.roblox.com/sitemap-ja.xml\nSitemap: https://www.roblox.com/sitemap-games-ja.xml\nSitemap: https://www.roblox.com/sitemap-ko.xml\nSitemap: https://www.roblox.com/sitemap-games-ko.xml\nSitemap: https://www.roblox.com/sitemap-nl.xml\nSitemap: https://www.roblox.com/sitemap-games-nl.xml\nSitemap: https://www.roblox.com/sitemap-pl.xml\nSitemap: https://www.roblox.com/sitemap-games-pl.xml\nSitemap: https://www.roblox.com/sitemap-pt.xml\nSitemap: https://www.roblox.com/sitemap-games-pt.xml\nSitemap: https://www.roblox.com/sitemap-th.xml\nSitemap: https://www.roblox.com/sitemap-games-th.xml\nSitemap: https://www.roblox.com/sitemap-tr.xml\nSitemap: https://www.roblox.com/sitemap-games-tr.xml\nSitemap: https://www.roblox.com/sitemap-vi.xml\nSitemap: https://www.roblox.com/sitemap-games-vi.xml\n","agents":{"GPTBot":true,"OAI-SearchBot":true,"ChatGPT-User":true,"ClaudeBot":true,"Claude-SearchBot":true,"PerplexityBot":true,"Google-Extended":true,"Applebot-Extended":true,"CCBot":true,"Bytespider":true,"Meta-ExternalAgent":true,"Amazonbot":true}},"sitemap":{"status":200,"content_type":"application/xml","bytes":77436,"sha256":"77db6a61f44a7c10fc36693646cc8e44b34b11998db51c11a8202bc17ee0a2b3","final_host":"www.roblox.com","final_path":"/sitemap.xml","ms":33,"url":"https://www.roblox.com/sitemap.xml","declared":true,"xml":true},"files":{"llms_txt":{"status":404,"content_type":"text/html","bytes":67371,"sha256":"c2dd72b31d3ce69db83377a0780dc45197bd83dae4c591ab9f1c65d376b3fbb6","final_host":"www.roblox.com","final_path":"/llms.txt","ms":51,"path":"/llms.txt","html":true,"json":null,"present":false},"agents_md":{"status":404,"content_type":"text/html","bytes":67371,"sha256":"c3d4b61b5643463a23d05f997f17c53d60dffb1ac99690024a8251944a2206bc","final_host":"www.roblox.com","final_path":"/agents.md","ms":47,"path":"/agents.md","html":true,"json":null,"present":false},"agent_skills":{"status":404,"content_type":"application/json","bytes":40,"sha256":"6ed55387e8fd77ee283840993d928721a8c3779847c65e57e058082802c75eff","final_host":"www.roblox.com","final_path":"/.well-known/agent-skills/index.json","ms":37,"path":"/.well-known/agent-skills/index.json","html":false,"json":null,"present":false},"mcp_card":{"status":404,"content_type":"application/json","bytes":40,"sha256":"6ed55387e8fd77ee283840993d928721a8c3779847c65e57e058082802c75eff","final_host":"www.roblox.com","final_path":"/.well-known/mcp/server-card.json","ms":38,"path":"/.well-known/mcp/server-card.json","html":false,"json":null,"present":false},"api_catalog":{"status":404,"content_type":"application/json","bytes":40,"sha256":"6ed55387e8fd77ee283840993d928721a8c3779847c65e57e058082802c75eff","final_host":"www.roblox.com","final_path":"/.well-known/api-catalog","ms":39,"path":"/.well-known/api-catalog","html":false,"json":null,"present":false},"a2a_card":{"status":404,"content_type":"application/json","bytes":40,"sha256":"6ed55387e8fd77ee283840993d928721a8c3779847c65e57e058082802c75eff","final_host":"www.roblox.com","final_path":"/.well-known/agent-card.json","ms":39,"path":"/.well-known/agent-card.json","html":false,"json":null,"present":false}},"rings":{"v":1,"candidates":[],"mcp":[],"a2a":null,"webmcp":{"model_context":false,"bridge":false,"mcp_json":false,"link_rel":false,"any":false},"handshakes":0,"get_path":{"status":404,"content_type":"application/json","ms":39,"error":null},"get_host":{"status":530,"content_type":"text/plain","ms":9,"error":null}},"sf":[4,0,0,0,0,0,3,0,1],"derived":{"reachable":true,"home_class":"answers","robots_served":true,"robots_sitemap":true,"content_signal":false,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":[],"blocks_any_ai":false,"files":{"llms_txt":false,"agents_md":false,"agent_skills":false,"mcp_card":false,"api_catalog":false,"a2a_card":false},"any_capability":false,"ai_refused":[],"ai_thin":[],"aipref":0,"aipref_measured":true,"sf":[4,0,0,0,0,0,3,0,1],"rings":{"mcp_declared":false,"mcp_live":false,"mcp_auth":false,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":3,"fh":"www.roblox.com","fp_other_host":[]},"derive_v":5,"frame":"top1k","shard":5},{"v":1,"kind":"crawlcheck-census-record","run":43,"domain":"endpoints.news","rank":63,"measured_at":"2026-11-13T08:07:15.177Z","home":{"browser":{"status":200,"content_type":"text/html","bytes":118124,"sha256":"72a827712302b6aae6f0d943417e71903c9a34f7843d2cc615176ba03ce590f9","final_host":"endpoints.news","final_path":"/","ms":49,"identity":"Mobile browser","html":true,"fp":"474e1b9defb0133d"},"gptbot":{"status":403,"content_type":"text/html","bytes":919,"sha256":"23c51918ed19795fe9fef15126200edaff989f7ddc2be1ba8a5ec5b5d0211dcf","final_host":"endpoints.news","final_path":"/","ms":9,"identity":"GPTBot","html":true},"claudebot":{"status":403,"content_type":"text/html","bytes":919,"sha256":"73bdbcf003e0efb116ae972fdc658846e3b840660ab30e0ddaa30ffcde20289a","final_host":"endpoints.news","final_path":"/","ms":10,"identity":"ClaudeBot","html":true},"oaisearch":{"status":403,"content_type":"text/html","bytes":919,"sha256":"eb10ec134f2888f236d4329fb718c0bc8a4c8f7c0c9a51b7aaef28936de5ea26","final_host":"endpoints.news","final_path":"/","ms":12,"identity":"OAI-SearchBot","html":true},"perplexity":{"status":403,"content_type":"text/html","bytes":919,"sha256":"2a19d4ad7cc677c30c9fdef81541b3f3fc4bbcb6ef0f6ef9bdd99f277e7a5272","final_host":"endpoints.news","final_path":"/","ms":18,"identity":"PerplexityBot","html":true}},"robots":{"status":403,"content_type":"text/html","bytes":919,"sha256":"1685c899f769a6fedd1460c92b7cd9e0175e06c90db19ec88f73b318f6d5b387","final_host":"endpoints.news","final_path":"/robots.txt","ms":32,"html":true,"text":null,"agents":null},"sitemap":{"status":403,"content_type":"text/html","bytes":919,"sha256":"9e68d95e007a7437d418adb9182511b3f4d3ce226d266326a5413e8481435993","final_host":"endpoints.news","final_path":"/sitemap.xml","ms":6,"url":"https://endpoints.news/sitemap.xml","declared":false,"xml":false},"files":{"llms_txt":{"status":403,"content_type":"text/html","bytes":919,"sha256":"8e6b80b737ea4890fd43e2db07dc93838c2e05dcdbcdb009982c6f88b15c4f5c","final_host":"endpoints.news","final_path":"/llms.txt","ms":16,"path":"/llms.txt","html":true,"json":null,"present":false},"agents_md":{"status":403,"content_type":"text/html","bytes":919,"sha256":"919077762ef1499f80a890ce9d353978dc55e86f40a24a8e7ff16eb3fa68b6e0","final_host":"endpoints.news","final_path":"/agents.md","ms":8,"path":"/agents.md","html":true,"json":null,"present":false},"agent_skills":{"status":403,"content_type":"text/html","bytes":919,"sha256":"389ee8a8ade7d256a10708d7849f7aedd5c4bb05774b44a1cf8b33c879d44c4f","final_host":"endpoints.news","final_path":"/.well-known/agent-skills/index.json","ms":8,"path":"/.well-known/agent-skills/index.json","html":true,"json":null,"present":false},"mcp_card":{"status":403,"content_type":"text/html","bytes":919,"sha256":"552518a9777e2aab59ce45429d00b2d8bbe8d4516d378026bf2872fac3213a72","final_host":"endpoints.news","final_path":"/.well-known/mcp/server-card.json","ms":8,"path":"/.well-known/mcp/server-card.json","html":true,"json":null,"present":false},"api_catalog":{"status":403,"content_type":"text/html","bytes":919,"sha256":"f1d48a5a7cb9f0d7c62449c95d79633e46d00d60e3bc884733fa6024f2a6a0ee","final_host":"endpoints.news","final_path":"/.well-known/api-catalog","ms":13,"path":"/.well-known/api-catalog","html":true,"json":null,"present":false},"a2a_card":{"status":403,"content_type":"text/html","bytes":919,"sha256":"5952fbc8bb8a4ae9ccd2613c235b6b52da66b44fb5ec90bd738122e36ab450e8","final_host":"endpoints.news","final_path":"/.well-known/agent-card.json","ms":7,"path":"/.well-known/agent-card.json","html":true,"json":null,"present":false}},"rings":{"v":1,"candidates":[],"mcp":[],"a2a":null,"webmcp":{"model_context":false,"bridge":false,"mcp_json":false,"link_rel":false,"any":false},"handshakes":0,"get_path":{"status":403,"content_type":"text/html","ms":9,"error":null},"get_host":{"status":530,"content_type":"text/plain","ms":31,"error":null}},"sf":[1,0,0,0,0,0,2,0,0],"derived":{"reachable":true,"home_class":"answers","robots_served":false,"robots_sitemap":false,"content_signal":false,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":[],"blocks_any_ai":false,"files":{"llms_txt":false,"agents_md":false,"agent_skills":false,"mcp_card":false,"api_catalog":false,"a2a_card":false},"any_capability":false,"ai_refused":["gptbot","claudebot","oaisearch","perplexity"],"ai_thin":[],"sf":[1,0,0,0,0,0,2,0,0],"rings":{"mcp_declared":false,"mcp_live":false,"mcp_auth":false,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":3,"fh":"endpoints.news","fp_other_host":[],"fp":"474e1b9defb0133d","fp_bits":{}},"derive_v":5,"frame":"top1k","shard":6},{"v":1,"kind":"crawlcheck-census-record","run":43,"domain":"windows.net","rank":64,"measured_at":"2026-11-13T08:08:20.375Z","home":{"browser":{"status":526,"content_type":"text/plain","bytes":16,"sha256":"4c543fe8a4d739c6f622b04303ac9e9d9fafbf00da950fc916ad826da2c0cdac","final_host":"windows.net","final_path":"/","ms":1012,"identity":"Mobile browser","html":false},"gptbot":{"status":526,"content_type":"text/plain","bytes":16,"sha256":"4c543fe8a4d739c6f622b04303ac9e9d9fafbf00da950fc916ad826da2c0cdac","final_host":"windows.net","final_path":"/","ms":1925,"identity":"GPTBot","html":false},"claudebot":{"status":526,"content_type":"text/plain","bytes":16,"sha256":"4c543fe8a4d739c6f622b04303ac9e9d9fafbf00da950fc916ad826da2c0cdac","final_host":"windows.net","final_path":"/","ms":277,"identity":"ClaudeBot","html":false},"oaisearch":{"status":526,"content_type":"text/plain","bytes":16,"sha256":"4c543fe8a4d739c6f622b04303ac9e9d9fafbf00da950fc916ad826da2c0cdac","final_host":"windows.net","final_path":"/","ms":569,"identity":"OAI-SearchBot","html":false},"perplexity":{"status":526,"content_type":"text/plain","bytes":16,"sha256":"4c543fe8a4d739c6f622b04303ac9e9d9fafbf00da950fc916ad826da2c0cdac","final_host":"windows.net","final_path":"/","ms":460,"identity":"PerplexityBot","html":false}},"robots":{"status":526,"content_type":"text/plain","bytes":16,"sha256":"4c543fe8a4d739c6f622b04303ac9e9d9fafbf00da950fc916ad826da2c0cdac","final_host":"windows.net","final_path":"/robots.txt","ms":420,"html":false,"text":null,"agents":null},"sitemap":null,"files":null,"derived":{"reachable":false,"home_class":"no_site","robots_served":false,"robots_sitemap":false,"content_signal":false,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":[],"blocks_any_ai":false,"files":{"llms_txt":false,"agents_md":false,"agent_skills":false,"mcp_card":false,"api_catalog":false,"a2a_card":false},"any_capability":false,"ai_refused":[],"ai_thin":[],"rings":{"mcp_declared":false,"mcp_live":false,"mcp_auth":false,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":3,"fh":"windows.net","fp_other_host":[]},"derive_v":5,"frame":"top1k","shard":7},{"v":1,"kind":"crawlcheck-census-record","run":43,"domain":"adobe.com","rank":65,"measured_at":"2026-11-13T07:56:55.238Z","home":{"browser":{"status":200,"content_type":"text/html","bytes":51346,"sha256":"20ce11ad16966cb5e0a1ed3d9b9d268a06771b5d40f28a81ac18c94c03d064d4","final_host":"www.adobe.com","final_path":"/","ms":68,"identity":"Mobile browser","html":true,"fp":"cf1259fe84823eca"},"gptbot":{"status":403,"content_type":"text/html","bytes":367,"sha256":"5e618ee8ed8dbf6895cfce10c9d3ba6f58db88065176c42e3759f75408102684","final_host":"www.adobe.com","final_path":"/","ms":39,"identity":"GPTBot","html":true},"claudebot":{"status":403,"content_type":"text/html","bytes":367,"sha256":"d20a515ec3b28758494edc72ca73f32d66ca11e848f92b6c6061bb1426c30f0f","final_host":"www.adobe.com","final_path":"/","ms":19,"identity":"ClaudeBot","html":true},"oaisearch":{"status":403,"content_type":"text/html","bytes":367,"sha256":"2bb60d0265463f97d6bfc21c139bccc1b740e16c112750be4021b786478f05bb","final_host":"www.adobe.com","final_path":"/","ms":35,"identity":"OAI-SearchBot","html":true},"perplexity":{"status":403,"content_type":"text/html","bytes":367,"sha256":"9b0cf6a20c1a6197c9b0be3cbec519ddeda90e25495b1fbcabef6ece9af7b422","final_host":"www.adobe.com","final_path":"/","ms":22,"identity":"PerplexityBot","html":true}},"robots":{"status":200,"content_type":"text/plain","bytes":10199,"sha256":"41081faa0fd323774e225ab51e8a73262055b24ecc7732557229fb93938f9d84","final_host":"www.adobe.com","final_path":"/robots.txt","ms":364,"html":false,"text":"# The use of robots or other automated means to access the Adobe site\n# without the express permission of Adobe is strictly prohibited.\n# Notwithstanding the foregoing, Adobe may permit automated access to\n# access certain Adobe pages but solely for the limited purpose of\n# including content in publicly available search engines. Any other\n# use of robots or failure to obey the robots exclusion standards set\n# forth at http://www.robotstxt.org/ is strictly prohibited.\n\n# Details about Googlebot available at: http://www.google.com/bot.html\n# The Google search engine can see everything\nUser-agent: gsa-crawler-www \nAllow: /assets/globalnav/v1.1.4/css/globalnav.css\nAllow: /assets/globalnav/v1.1.4/*/globalnav.js\nAllow: /au/special/symposium\nAllow: /sea/special/symposium\nAllow: /*special/products/marketing-cloud/*.css$\nAllow: /*special/products/marketing-cloud/*.png$\nAllow: /*special/products/marketing-cloud/*.js$\nAllow: /www.adobe.com/special/ri/prod/assets/*.css$\nAllow: /www.adobe.com/special/ri/prod/assets/*.jpg$\nAllow: /www.adobe.com/special/ri/prod/assets/*.js$\nAllow: /www.adobe.com/special/ri/prod/assets/*.png$\nAllow: /www.adobe.com/special/retail-shopping-insights/\nAllow: /uk/special/landing/student/digitaledgeaward/\nAllow: /uk/special/landing/student/digitaledgeaward/\nAllow: /uk/special/landing/student/lovethejourney\nAllow: /fr/special/landing/student/lovethejourney\nDisallow: /assets/globalnav/\nDisallow: /*creativesuite/bridge\nDisallow: /cfusion/gpr/\nDisallow: /cfusion/search/\nDisallow: /cfusion/partnerportal/\nDisallow: /globalsearch\nDisallow: /cgi-local/\nDisallow: /de/products/index-notabs.html\nDisallow: /de/products/index-tabs.html\nDisallow: /devnet-archive/\nDisallow: /devnet/archive\nDisallow: /*devnet/tagsearch\nDisallow: /emea/eemeachannel/\nDisallow: /events/ciocouncil/\nDisallow: /events/executivecouncil/\nDisallow: /google_indexing/\nDisallow: /*.html_\nDisallow: /licensing/distribution/strategies/\nDisallow: /*limited/\nDisallow: /livedocs/\nDisallow: /*Ljava.lang.Object*\nDisallow: /products/acrobat/readstep2.html;JMICROSITE2SESSION*\nDisallow: /*products/flashplayer/distribution3.html\nDisallow: /*products/flashplayer/distribution4.html\nDisallow: /*special/\nDisallow: /*.sql$\nDisallow: /*supportservice/\nDisallow: /*.translate.html\nDisallow: /*products/adobeconnect/enterprise/trial.html\nDisallow: /*products/adobeconnect/enterprise/trial-check.html\nDisallow: /*products/adobeconnect/enterprise/trial-thankyou.html\nDisallow: /*products/adobeconnect/enterprise/trial-error.html\nDisallow: /*products/photoshop/selector\nDisallow: /*unsubscribe.html\nDisallow: /content/dam/acom/en/marketing-cloud/offers/pdf\nDisallow: /content/dam/www/us/en/adi/2020/pdfs/*\nDisallow: /content/dotcom/*.html$\nDisallow: /content/dam/www/us/en/offer/digital-trends-2020/digital-trends-2020-full-report.pdf\nDisallow: /plans-fragments/\nDisallow: /plans.modals*\n\n# The Omniture search engine can see everything\nUser-agent: Atomz/1.0\nDisallow: /devnet-archive/\nDisallow: /devnet/archive\nDisallow: /events/ciocouncil/\nDisallow: /events/executivecouncil/\nDisallow: /licensing/distribution/strategies/\nDisallow: /*limited/\nDisallow: /livedocs/\nDisallow: /products/flashplayer/distribution3.html\nDisallow: /products/flashplayer/distribution4.html\nDisallow: /*special/\nDisallow: /*.sql$\nDisallow: /*supportservice/\n\nUser-agent: *\nAllow: /assets/globalnav/v1.1.4/css/globalnav.css\nAllow: /assets/globalnav/v1.1.4/*globalnav.js\nAllow: /au/special/symposium\nAllow: /sea/special/symposium\nAllow: /*special/products/marketing-cloud/*.css$\nAllow: /*special/products/marketing-cloud/*.js$\nAllow: /*special/products/marketing-cloud/*.png$\nAllow: /www.adobe.com/special/ri/prod/assets/*.css$\nAllow: /www.adobe.com/special/ri/prod/assets/*.jpg$\nAllow: /www.adobe.com/special/ri/prod/assets/*.js$\nAllow: /www.adobe.com/special/ri/prod/assets/*.png$\nAllow: /www.adobe.com/special/retail-shopping-insights/\nAllow: /uk/special/landing/student/digitaledgeaward/\nAllow: /uk/special/landing/student/lovethejourney\nAllow: /fr/special/landing/student/lovethejourney\nAllow: /community/gallery$\nAllow: /community/gallery/assets$\nAllow: /community/gallery/challenges$\nDisallow: /assets/globalnav/\nDisallow: /*creativesuite/bridge\nDisallow: /cfusion/gpr/\nDisallow: /cfusion/search/\nDisallow: /cfusion/partnerportal/\nDisallow: /globalsearch\nDisallow: /cgi-local/\nDisallow: /de/products/index-notabs.html\nDisallow: /de/products/index-tabs.html\nDisallow: /devnet-archive/\nDisallow: /devnet/archive\nDisallow: /*devnet/tagsearch\nDisallow: /emea/eemeachannel/\nDisallow: /events/ciocouncil/\nDisallow: /events/executivecouncil/\nDisallow: /google_indexing/\nDisallow: /*.html_\nDisallow: /licensing/distribution/strategies/\nDisallow: /*limited/\nDisallow: /livedocs/\nDisallow: /*Ljava.lang.Object*\nDisallow: /products/acrobat/readstep2.html;JMICROSITE2SESSION*\nDisallow: /*products/flashplayer/distribution3.html\nDisallow: /*products/flashplayer/distribution4.html\nDisallow: /*special/\nDisallow: /*.sql$\nDisallow: /*supportservice/\nDisallow: /*.translate.html\nDisallow: /*products/adobeconnect/enterprise/trial.html\nDisallow: /*products/adobeconnect/enterprise/trial-check.html\nDisallow: /*products/adobeconnect/enterprise/trial-thankyou.html\nDisallow: /*products/adobeconnect/enterprise/trial-error.html\nDisallow: /*products/photoshop/selector\nDisallow: /*unsubscribe.html\nDisallow: /content/dam/acom/en/marketing-cloud/offers/pdf\nDisallow: /content/dotcom/*.html$\nDisallow: /plans-fragments/\nDisallow: /content/dam/www/us/en/offer/digital-trends-2020-in-media-and-entertainment-html/digital-trends-2020-in-media-and-entertainment.pdf\nDisallow: /content/dam/www/us/en/offer/digital-trends-2020-in-media-and-entertainment-html/digital-trends-2020-in-media-and-entertainment-DE.pdf\nDisallow: /content/dam/www/us/en/offer/digital-trends-2020-in-media-and-entertainment-html/digital-trends-2020-in-media-and-entertainment-EN-US.pdf\nDisallow: /content/dam/www/us/en/offer/digital-trends-2020-in-media-and-entertainment-html/digital-trends-2020-in-media-and-entertainment-EN.pdf\nDisallow: /content/dam/www/us/en/offer/digital-trends-2020-in-media-and-entertainment-html/digital-trends-2020-in-media-and-entertainment-ES.pdf\nDisallow: /content/dam/www/us/en/offer/digital-trends-2020-in-media-and-entertainment-html/digital-trends-2020-in-media-and-entertainment-FR.pdf\nDisallow: /content/dam/www/us/en/offer/digital-trends-2020-in-media-and-entertainment-html/digital-trends-2020-in-media-and-entertainment-IT.pdf\nDisallow: /content/dam/www/us/en/offer/digital-trends-2020-in-media-and-entertainment-html/digital-trends-2020-in-media-and-entertainment-JA.pdf\nDisallow: /content/dam/www/us/en/offer/digital-trends-2020-in-media-and-entertainment-html/digital-trends-2020-in-media-and-entertainment-KR.pdf\nDisallow: /content/dam/www/us/en/offer/digital-trends-2020-in-media-and-entertainment-html/digital-trends-2020-in-media-and-entertainment-zh-Hans.pdf\nDisallow: /content/dam/www/us/en/offer/digital-trends-2020-in-media-and-entertainment-html/digital-trends-2020-in-media-and-entertainment-zh-Hant.pdf\nDisallow: /content/dam/www/us/en/offer/digital-trends-2020-in-media-and-entertainment-html/digital_trends_21_core_report_TC_FA.pdf\nDisallow: /content/dam/dx/us/en/resources/reports/digital-trends-2021-in-high-tech/2021_Digital_Trends_High_Tech_in_Focus.pdf\nDisallow: /content/dam/dx/us/en/resources/digital-trends-2021-in-telecommunications-media-and-entertainment/Digital_Trends_2021_Media_Entertainment_and_Telecoms_Report.pdf\nDisallow: /content/dam/dx/us/en/resources/digital-trends-2021-in-retail/2021_Digital_Trends_Retail_in_Focus.pdf\nDisallow: /content/dam/dx/us/en/resources/digital-trends-2021-in-manufacturing/2021_Digital_Trends_Manufacturing_in_Focus.pdf\nDisallow: /content/dam/www/us/en/offer/technology-trends-2021/eConsultancy_Digital_Trends_2021_IT_Report.pdf\nDisallow: /content/dam/www/us/en/offer/pdf/digital-trends-2020-in-financial-services.pdf\nDisallow: /content/dam/www/us/en/offer/pdf/digital-trends-2020-in-retail.pdf\nDisallow: /content/dam/www/us/en/offer/pdf/digital-trends-2020-in-media-and-entertainment.pdf\nDisallow: /content/dam/www/us/en/offer/pdf/digital-trends-2020-in-travel-and-hospitality.pdf\nDisallow: /content/dam/www/us/en/offer/pdf/digital-trends-2020-in-telecommunications.pdf\nDisallow: /content/dam/www/us/en/offer/pdf/digital-trends-2020-in-advertising.pdf\nDisallow: /content/dam/www/us/en/offer/pdf/digital-trends-2020-in-high-tech.pdf\nDisallow: /content/dam/www/us/en/offer/pdf/digital-trends-2020-in-IT.pdf\nDisallow: /content/dam/www/us/en/adi/2020/pdfs/*\nDisallow: /content/dam/www/us/en/offer/digital-trends-2021-core/digital-trends-2021-full-report-*.pdf\nDisallow: /*langmaster/\nDisallow: /*express/*.mp4$\nDisallow: /en-AE/learn/\nDisallow: /en-BE/learn/\nDisallow: /en-CY/learn/\nDisallow: /en-GR/learn/\nDisallow: /en-HK/learn/\nDisallow: /en-ID/learn/\nDisallow: /en-IL/learn/\nDisallow: /en-LU/learn/\nDisallow: /en-MT/learn/\nDisallow: /en-MY/learn/\nDisallow: /en-PH/learn/\nDisallow: /en-SA/learn/\nDisallow: /en-SG/learn/\nDisallow: /en-TH/learn/\nDisallow: /en-VN/learn/\nDisallow: /en-x-MenA/learn/\nDisallow: /en-ZA/learn/\nDisallow: /*?searchId=\nDisallow: /*?tasks=\nDisallow: /*?campaignid=\nDisallow: /*?skwcid=\nDisallow: /products/photoshop/photoshop-brushes.h*\nDisallow: /community/gallery\n\nUser-agent: Googlebot\nAllow: /?promoid=RTQCN3LX\nDisallow: /*promoid=\nDisallow: /*trackingid=\nDisallow: /*?searchId=\nDisallow: /*?tasks=\nDisallow: /*?campaignid=\nDisallow: /*?skwcid=\nDisallow: /products/photoshop/photoshop-brushes.h*\n\nUser-agent: AdsBot-Google\nAllow: /\n\nUser-agent: AdsBot-Google-Mobile\nAllow: /\n\nUser-agent: SearchmetricsBot\nDisallow: /\n\n# XML sitemaps\nSitemap: https://www.adobe.com/cc-product.index.xml\nSitemap: https://www.adobe.com/creativecloud/sitemap-index.xml\nSitemap: https://www.adobe.com/acrobat/sitemap-index.xml\nSitemap: https://www.adobe.com/home-sitemap.xml\nSitemap: https://www.adobe.com/plans-catalog-sitemap.xml\nSitemap: https://www.adobe.com/learn/sitemap_index.xml\n\n# CC Express \nSitemap: https://www.adobe.com/express/sitemap-index.xml\n\n#Naver\nSitemap: https://www.adobe.com/naver.index.xml\n\n#Education\nSitemap: https://www.adobe.com/edu/sitemap-index.xml\n","agents":{"GPTBot":true,"OAI-SearchBot":true,"ChatGPT-User":true,"ClaudeBot":true,"Claude-SearchBot":true,"PerplexityBot":true,"Google-Extended":true,"Applebot-Extended":true,"CCBot":true,"Bytespider":true,"Meta-ExternalAgent":true,"Amazonbot":true}},"sitemap":{"status":200,"content_type":"application/xml","bytes":8649,"sha256":"8f5710016d9dce1daf3d0d0cbe81f62c1154d28961484d0fb8c518a22a5d638d","final_host":"www.adobe.com","final_path":"/cc-product.index.xml","ms":174,"url":"https://www.adobe.com/cc-product.index.xml","declared":true,"xml":true},"files":{"llms_txt":{"status":200,"content_type":"text/plain","bytes":10499,"sha256":"811353cb603ff604ecd257b7e24643f1c7e157837015221a951d9d2db0c494d8","final_host":"www.adobe.com","final_path":"/llms.txt","ms":119,"path":"/llms.txt","html":false,"json":null,"present":true},"agents_md":{"status":404,"content_type":"text/html","bytes":1323,"sha256":"2c9dd78b24598e2cff988fcf6e809931ee5450e5d0a5a57d7e97a91092e018b5","final_host":"www.adobe.com","final_path":"/agents.md","ms":76,"path":"/agents.md","html":true,"json":null,"present":false},"agent_skills":{"status":404,"content_type":"text/html","bytes":1323,"sha256":"2c9dd78b24598e2cff988fcf6e809931ee5450e5d0a5a57d7e97a91092e018b5","final_host":"www.adobe.com","final_path":"/.well-known/agent-skills/index.json","ms":89,"path":"/.well-known/agent-skills/index.json","html":true,"json":null,"present":false},"mcp_card":{"status":404,"content_type":"text/html","bytes":1323,"sha256":"2c9dd78b24598e2cff988fcf6e809931ee5450e5d0a5a57d7e97a91092e018b5","final_host":"www.adobe.com","final_path":"/.well-known/mcp/server-card.json","ms":81,"path":"/.well-known/mcp/server-card.json","html":true,"json":null,"present":false},"api_catalog":{"status":404,"content_type":"text/html","bytes":1323,"sha256":"2c9dd78b24598e2cff988fcf6e809931ee5450e5d0a5a57d7e97a91092e018b5","final_host":"www.adobe.com","final_path":"/.well-known/api-catalog","ms":97,"path":"/.well-known/api-catalog","html":true,"json":null,"present":false},"a2a_card":{"status":404,"content_type":"text/html","bytes":1323,"sha256":"2c9dd78b24598e2cff988fcf6e809931ee5450e5d0a5a57d7e97a91092e018b5","final_host":"www.adobe.com","final_path":"/.well-known/agent-card.json","ms":73,"path":"/.well-known/agent-card.json","html":true,"json":null,"present":false}},"rings":{"v":1,"candidates":[],"mcp":[],"a2a":null,"webmcp":{"model_context":false,"bridge":false,"mcp_json":false,"link_rel":false,"any":false},"handshakes":0,"get_path":{"status":404,"content_type":"text/html","ms":89,"error":null},"get_host":{"status":530,"content_type":"text/plain","ms":8,"error":null}},"sf":[-1,0,0,0,0,0,0,0,0],"derived":{"reachable":true,"home_class":"answers","robots_served":true,"robots_sitemap":true,"content_signal":false,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":[],"blocks_any_ai":false,"files":{"llms_txt":true,"agents_md":false,"agent_skills":false,"mcp_card":false,"api_catalog":false,"a2a_card":false},"any_capability":false,"ai_refused":["gptbot","claudebot","oaisearch","perplexity"],"ai_thin":[],"aipref":0,"aipref_measured":true,"sf":[-1,0,0,0,0,0,0,0,0],"rings":{"mcp_declared":false,"mcp_live":false,"mcp_auth":false,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":3,"fh":"www.adobe.com","fp_other_host":[],"fp":"cf1259fe84823eca","fp_bits":{}},"derive_v":5,"frame":"top1k","shard":0},{"v":1,"kind":"crawlcheck-census-record","run":43,"domain":"pv-cdn.net","rank":66,"measured_at":"2026-11-13T07:57:55.549Z","home":{"browser":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"pv-cdn.net","final_path":"/","ms":257,"identity":"Mobile browser","html":false},"gptbot":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"pv-cdn.net","final_path":"/","ms":172,"identity":"GPTBot","html":false},"claudebot":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"pv-cdn.net","final_path":"/","ms":166,"identity":"ClaudeBot","html":false},"oaisearch":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"pv-cdn.net","final_path":"/","ms":161,"identity":"OAI-SearchBot","html":false},"perplexity":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"pv-cdn.net","final_path":"/","ms":82,"identity":"PerplexityBot","html":false}},"robots":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"pv-cdn.net","final_path":"/robots.txt","ms":0,"html":false,"text":null,"agents":null},"sitemap":null,"files":null,"derived":{"reachable":false,"home_class":"no_site","robots_served":false,"robots_sitemap":false,"content_signal":false,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":[],"blocks_any_ai":false,"files":{"llms_txt":false,"agents_md":false,"agent_skills":false,"mcp_card":false,"api_catalog":false,"a2a_card":false},"any_capability":false,"ai_refused":[],"ai_thin":[],"rings":{"mcp_declared":false,"mcp_live":false,"mcp_auth":false,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":3,"fh":"pv-cdn.net","fp_other_host":[]},"derive_v":5,"frame":"top1k","shard":1},{"v":1,"kind":"crawlcheck-census-record","run":43,"domain":"windowsupdate.com","rank":67,"measured_at":"2026-11-13T07:58:58.649Z","home":{"browser":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"windowsupdate.com","final_path":"/","ms":13,"identity":"Mobile browser","html":false},"gptbot":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"windowsupdate.com","final_path":"/","ms":1,"identity":"GPTBot","html":false},"claudebot":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"windowsupdate.com","final_path":"/","ms":1,"identity":"ClaudeBot","html":false},"oaisearch":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"windowsupdate.com","final_path":"/","ms":1,"identity":"OAI-SearchBot","html":false},"perplexity":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"windowsupdate.com","final_path":"/","ms":0,"identity":"PerplexityBot","html":false}},"robots":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"windowsupdate.com","final_path":"/robots.txt","ms":1,"html":false,"text":null,"agents":null},"sitemap":null,"files":null,"derived":{"reachable":false,"home_class":"no_site","robots_served":false,"robots_sitemap":false,"content_signal":false,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":[],"blocks_any_ai":false,"files":{"llms_txt":false,"agents_md":false,"agent_skills":false,"mcp_card":false,"api_catalog":false,"a2a_card":false},"any_capability":false,"ai_refused":[],"ai_thin":[],"rings":{"mcp_declared":false,"mcp_live":false,"mcp_auth":false,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":3,"fh":"windowsupdate.com","fp_other_host":[]},"derive_v":5,"frame":"top1k","shard":2},{"v":1,"kind":"crawlcheck-census-record","run":43,"domain":"spotify.com","rank":68,"measured_at":"2026-11-13T07:59:19.049Z","home":{"browser":{"status":200,"content_type":"text/html","bytes":315751,"sha256":"3c06c2abab218a350d342c24f8d601d80365f8d28fce02e47996977c07fe6122","final_host":"open.spotify.com","final_path":"/","ms":306,"identity":"Mobile browser","html":true,"fp":"3b6526edd21d3901"},"gptbot":{"status":200,"content_type":"text/html","bytes":172398,"sha256":"ed93243ccd3e82519b47ee9d3ac571f85a5139d07e9e163c6ee9f4b70839001b","final_host":"open.spotify.com","final_path":"/","ms":294,"identity":"GPTBot","html":true},"claudebot":{"status":200,"content_type":"text/html","bytes":172398,"sha256":"7a9fc5bd7cb9cd9b696f461fe18b27da53e5f9e5f00f58acc2e3f214d08a7c5d","final_host":"open.spotify.com","final_path":"/","ms":145,"identity":"ClaudeBot","html":true},"oaisearch":{"status":200,"content_type":"text/html","bytes":172398,"sha256":"9a256ca3051282b2f45f4100d12fb75a3d276949627cfa654396b8d7284c7014","final_host":"open.spotify.com","final_path":"/","ms":136,"identity":"OAI-SearchBot","html":true},"perplexity":{"status":200,"content_type":"text/html","bytes":172398,"sha256":"97fd623c855345204fdeac5f108d4127da10e87dc0c398866908c9f6f73d79fa","final_host":"open.spotify.com","final_path":"/","ms":119,"identity":"PerplexityBot","html":true}},"robots":{"status":200,"content_type":"text/plain","bytes":635,"sha256":"b68c89548f813d5c465c13051353b9452dba066f14146cafabb46dafd239b738","final_host":"www.spotify.com","final_path":"/robots.txt","ms":125,"html":false,"text":"User-agent: *\nDisallow: /*/about-us/contact/contact-spotify-password/\nDisallow: /*/about-us/contact/contact-spotify-account/\nDisallow: /*/get-spotify/*\nDisallow: /*/xhr/*\nDisallow: /*/external/*\nDisallow: /*/legal/*?ets=\nDisallow: /*/legal/advertiser-terms-and-conditions/\nDisallow: /*/legal/gdpr-article-15-information/\nDisallow: /*/legal/spotify-controller-data-processing-terms/\nDisallow: /*/legal/podcast-api-terms/\nDisallow: /*/legal/distribution-api-terms/\nDisallow: /*/account/cls/*\nDisallow: /*/starbuckspartners\nDisallow: /starbuckspartners\nDisallow: /ppt/*?\nDisallow: /partner/*?\nSitemap: https://www.spotify.com/sitemap.xml\n","agents":{"GPTBot":true,"OAI-SearchBot":true,"ChatGPT-User":true,"ClaudeBot":true,"Claude-SearchBot":true,"PerplexityBot":true,"Google-Extended":true,"Applebot-Extended":true,"CCBot":true,"Bytespider":true,"Meta-ExternalAgent":true,"Amazonbot":true}},"sitemap":{"status":200,"content_type":"application/xml","bytes":1114,"sha256":"2ae8c8fbf612a2977db25fba637d68380b5855470af21c83312aed14e1339258","final_host":"www.spotify.com","final_path":"/sitemap.xml","ms":31,"url":"https://www.spotify.com/sitemap.xml","declared":true,"xml":true},"files":{"llms_txt":{"status":200,"content_type":"text/html","bytes":50829,"sha256":"b1ab81c229a5a7370c7fa02e90c4811d69f594afbc406dbf9f890508e5f435eb","final_host":"www.spotify.com","final_path":"/us/llms.txt","ms":151,"path":"/llms.txt","html":true,"json":null,"present":false},"agents_md":{"status":200,"content_type":"text/html","bytes":50829,"sha256":"b1ab81c229a5a7370c7fa02e90c4811d69f594afbc406dbf9f890508e5f435eb","final_host":"www.spotify.com","final_path":"/us/agents.md","ms":152,"path":"/agents.md","html":true,"json":null,"present":false},"agent_skills":{"status":404,"content_type":"text/html","bytes":3310,"sha256":"554b0c3a6ad268de5b3c8b3deb4205936ce1f3dd80303ec31af3b906d4fa9cf5","final_host":"www.spotify.com","final_path":"/.well-known/agent-skills/index.json","ms":64,"path":"/.well-known/agent-skills/index.json","html":true,"json":null,"present":false},"mcp_card":{"status":404,"content_type":"text/html","bytes":3310,"sha256":"554b0c3a6ad268de5b3c8b3deb4205936ce1f3dd80303ec31af3b906d4fa9cf5","final_host":"www.spotify.com","final_path":"/.well-known/mcp/server-card.json","ms":65,"path":"/.well-known/mcp/server-card.json","html":true,"json":null,"present":false},"api_catalog":{"status":404,"content_type":"text/html","bytes":3310,"sha256":"554b0c3a6ad268de5b3c8b3deb4205936ce1f3dd80303ec31af3b906d4fa9cf5","final_host":"www.spotify.com","final_path":"/.well-known/api-catalog","ms":62,"path":"/.well-known/api-catalog","html":true,"json":null,"present":false},"a2a_card":{"status":404,"content_type":"text/html","bytes":3310,"sha256":"554b0c3a6ad268de5b3c8b3deb4205936ce1f3dd80303ec31af3b906d4fa9cf5","final_host":"www.spotify.com","final_path":"/.well-known/agent-card.json","ms":78,"path":"/.well-known/agent-card.json","html":true,"json":null,"present":false}},"rings":{"v":1,"candidates":[],"mcp":[],"a2a":null,"webmcp":{"model_context":false,"bridge":false,"mcp_json":false,"link_rel":false,"any":false},"handshakes":0,"get_path":{"status":200,"content_type":"text/html","ms":130,"error":null},"get_host":{"status":530,"content_type":"text/plain","ms":131,"error":null}},"sf":[4,0,0,0,0,0,1,0,0],"derived":{"reachable":true,"home_class":"answers","robots_served":true,"robots_sitemap":true,"content_signal":false,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":[],"blocks_any_ai":false,"files":{"llms_txt":false,"agents_md":false,"agent_skills":false,"mcp_card":false,"api_catalog":false,"a2a_card":false},"any_capability":false,"ai_refused":[],"ai_thin":["gptbot","claudebot","oaisearch","perplexity"],"aipref":0,"aipref_measured":true,"sf":[4,0,0,0,0,0,1,0,0],"rings":{"mcp_declared":false,"mcp_live":false,"mcp_auth":false,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":3,"fh":"open.spotify.com","fp_other_host":[],"fp":"3b6526edd21d3901","fp_bits":{}},"derive_v":5,"frame":"top1k","shard":3},{"v":1,"kind":"crawlcheck-census-record","run":43,"domain":"wa.me","rank":69,"measured_at":"2026-11-13T08:05:49.397Z","home":{"browser":{"status":400,"content_type":"text/html","bytes":6278,"sha256":"de28c052cf150b3605632f9f4b12f33008c4a0a7cae153942d2a0bcc65c5f990","final_host":"api.whatsapp.com","final_path":"/","ms":338,"identity":"Mobile browser","html":true},"gptbot":{"status":200,"content_type":"text/html","bytes":187365,"sha256":"5c0a202d81ab39eaa4967ce8903cff3ff8517c4d8f7de6c195e704f4248cf4d6","final_host":"api.whatsapp.com","final_path":"/","ms":371,"identity":"GPTBot","html":true,"fp":"75c68d3fb9077b36"},"claudebot":{"status":200,"content_type":"text/html","bytes":187315,"sha256":"457f73e3da67ac1b03655339bac5b48752780dd384cb4f5abbb2b48c496779c7","final_host":"api.whatsapp.com","final_path":"/","ms":410,"identity":"ClaudeBot","html":true,"fp":"75c68d3fb9077b36"},"oaisearch":{"status":200,"content_type":"text/html","bytes":187366,"sha256":"f8544b1fa0c1946bc946e9b5ca4fa25295eff9006bcbc3a19b298e1003782226","final_host":"api.whatsapp.com","final_path":"/","ms":406,"identity":"OAI-SearchBot","html":true,"fp":"75c68d3fb9077b36"},"perplexity":{"status":200,"content_type":"text/html","bytes":197588,"sha256":"116f102d08691ee9f239b9a9a196d11e5ed8ab36393f6a58ca5e4afaf4e09a8a","final_host":"api.whatsapp.com","final_path":"/","ms":275,"identity":"PerplexityBot","html":true,"fp":"6516893f3b077b3e"}},"robots":{"status":200,"content_type":"text/plain","bytes":1525,"sha256":"6ceebd7d44206c973321a46e46b94102a8edcc37eea87394b2dd94ffe713df8d","final_host":"wa.me","final_path":"/robots.txt","ms":165,"html":false,"text":"# Notice: Collection of data on WhatsApp through automated means is prohibited\n# See: https://www.whatsapp.com/legal/terms-of-service\nuser-agent: *\ndisallow: /usephone/\ndisallow: /message/\ndisallow: /qr/\ndisallow: /pay/\ndisallow: /stickerpack/\ndisallow: /biztools/\ndisallow: /settings/\ndisallow: /disappearing_messages/\ndisallow: /download/\ndisallow: /archive_settings/\ndisallow: /tos/\ndisallow: /support/\ndisallow: /bizsearch/\ndisallow: /directory/\ndisallow: /guia/\ndisallow: /profile/\ndisallow: /status/\ndisallow: /premium/\ndisallow: /community/\ndisallow: /advertise/\ndisallow: /biztab/\ndisallow: /biz-edit-profile/\ndisallow: /biz-edit-description/\ndisallow: /biz-edit-catalog/\ndisallow: /biz-location/\ndisallow: /biz-hours/\ndisallow: /biz-linked-accounts/\ndisallow: /privacy/\ndisallow: /biz-website/\ndisallow: /proxy/\ndisallow: /biz-price-tier/\ndisallow: /chat-transfer/\ndisallow: /calling/\ndisallow: /ais/\ndisallow: /marketing-message/\ndisallow: /invite/\ndisallow: /contacts/\ndisallow: /edit-profile-picture/\ndisallow: /wabai/\ndisallow: /notifications/\ndisallow: /favorites/\ndisallow: /meta_verified/\ndisallow: /biz-agents-onboarding/\ndisallow: /subscription_hub/\ndisallow: /share-whatsapp-web/\ndisallow: /biz-ai-learning-summary/\ndisallow: /waffle/\ndisallow: /family/\ndisallow: /daily-ads-summary/\ndisallow: /tools/\ndisallow: /join_nova_waitlist/\ndisallow: /man/\ndisallow: /hatch/\ndisallow: /subscriptions/\ndisallow: /biz-broadcast-home/\ndisallow: /business-broadcast/\nsitemap: https://wa.me/smp/wa_smb_custom_url.xml.gz","agents":{"GPTBot":true,"OAI-SearchBot":true,"ChatGPT-User":true,"ClaudeBot":true,"Claude-SearchBot":true,"PerplexityBot":true,"Google-Extended":true,"Applebot-Extended":true,"CCBot":true,"Bytespider":true,"Meta-ExternalAgent":true,"Amazonbot":true}},"sitemap":null,"files":null,"derived":{"reachable":false,"home_class":"refused_us","robots_served":false,"robots_sitemap":false,"content_signal":false,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":[],"blocks_any_ai":false,"files":{"llms_txt":false,"agents_md":false,"agent_skills":false,"mcp_card":false,"api_catalog":false,"a2a_card":false},"any_capability":false,"ai_refused":[],"ai_thin":[],"rings":{"mcp_declared":false,"mcp_live":false,"mcp_auth":false,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":3,"fh":"api.whatsapp.com","fp_other_host":[]},"derive_v":5,"frame":"top1k","shard":4},{"v":1,"kind":"crawlcheck-census-record","run":43,"domain":"myfritz.net","rank":70,"measured_at":"2026-11-13T08:07:00.657Z","home":{"browser":{"status":200,"content_type":"text/html","bytes":6885,"sha256":"0ae8e018e72bcac840700b49e691dd87595c3c6fe0bf94aa6ca9e47030b2fe66","final_host":"www.myfritz.net","final_path":"/devices","ms":1238,"identity":"Mobile browser","html":true},"gptbot":{"status":200,"content_type":"text/html","bytes":6885,"sha256":"0ae8e018e72bcac840700b49e691dd87595c3c6fe0bf94aa6ca9e47030b2fe66","final_host":"www.myfritz.net","final_path":"/devices","ms":1229,"identity":"GPTBot","html":true},"claudebot":{"status":200,"content_type":"text/html","bytes":6885,"sha256":"0ae8e018e72bcac840700b49e691dd87595c3c6fe0bf94aa6ca9e47030b2fe66","final_host":"www.myfritz.net","final_path":"/devices","ms":1205,"identity":"ClaudeBot","html":true},"oaisearch":{"status":200,"content_type":"text/html","bytes":6885,"sha256":"0ae8e018e72bcac840700b49e691dd87595c3c6fe0bf94aa6ca9e47030b2fe66","final_host":"www.myfritz.net","final_path":"/devices","ms":1209,"identity":"OAI-SearchBot","html":true},"perplexity":{"status":200,"content_type":"text/html","bytes":6885,"sha256":"0ae8e018e72bcac840700b49e691dd87595c3c6fe0bf94aa6ca9e47030b2fe66","final_host":"www.myfritz.net","final_path":"/devices","ms":1173,"identity":"PerplexityBot","html":true}},"robots":{"status":200,"content_type":"text/plain","bytes":396,"sha256":"b3bcd97fea71d357a13505629dbefa700279b72c572479a6c9ec8def9814fc16","final_host":"myfritz.net","final_path":"/robots.txt","ms":385,"html":false,"text":"User-agent: Yandex \nDisallow: /\n\nUser-agent: meta-externalagent\nDisallow: /\n\nUser-agent: meta-externalads\nDisallow: /\n\nUser-agent: meta-webindexer\nDisallow: /\n\nUser-agent: *\nDisallow: /\nAllow: /$\nAllow: /devices$\nAllow: /devices/$\nAllow: /devices/index.html$\nAllow: /sitemap.xml\nAllow: /legal-notice\nAllow: /terms\nAllow: /about\nHost: www.myfritz.net\n\nSitemap: https://www.myfritz.net/sitemap.xml\n","agents":{"GPTBot":true,"OAI-SearchBot":true,"ChatGPT-User":true,"ClaudeBot":true,"Claude-SearchBot":true,"PerplexityBot":true,"Google-Extended":true,"Applebot-Extended":true,"CCBot":true,"Bytespider":true,"Meta-ExternalAgent":false,"Amazonbot":true}},"sitemap":{"status":200,"content_type":"text/xml","bytes":2903,"sha256":"b83c75a3e8aa820ed2f4297239776d397858323c41e1edc4fcb339d16e973040","final_host":"www.myfritz.net","final_path":"/sitemap.xml","ms":89,"url":"https://www.myfritz.net/sitemap.xml","declared":true,"xml":true},"files":{"llms_txt":{"status":200,"content_type":"text/html","bytes":6885,"sha256":"0ae8e018e72bcac840700b49e691dd87595c3c6fe0bf94aa6ca9e47030b2fe66","final_host":"www.myfritz.net","final_path":"/devices","ms":189,"path":"/llms.txt","html":true,"json":null,"present":false},"agents_md":{"status":200,"content_type":"text/html","bytes":6885,"sha256":"0ae8e018e72bcac840700b49e691dd87595c3c6fe0bf94aa6ca9e47030b2fe66","final_host":"www.myfritz.net","final_path":"/devices","ms":189,"path":"/agents.md","html":true,"json":null,"present":false},"agent_skills":{"status":200,"content_type":"text/html","bytes":6885,"sha256":"0ae8e018e72bcac840700b49e691dd87595c3c6fe0bf94aa6ca9e47030b2fe66","final_host":"www.myfritz.net","final_path":"/devices","ms":190,"path":"/.well-known/agent-skills/index.json","html":true,"json":null,"present":false},"mcp_card":{"status":200,"content_type":"text/html","bytes":6885,"sha256":"0ae8e018e72bcac840700b49e691dd87595c3c6fe0bf94aa6ca9e47030b2fe66","final_host":"www.myfritz.net","final_path":"/devices","ms":188,"path":"/.well-known/mcp/server-card.json","html":true,"json":null,"present":false},"api_catalog":{"status":200,"content_type":"text/html","bytes":6885,"sha256":"0ae8e018e72bcac840700b49e691dd87595c3c6fe0bf94aa6ca9e47030b2fe66","final_host":"www.myfritz.net","final_path":"/devices","ms":184,"path":"/.well-known/api-catalog","html":true,"json":null,"present":false},"a2a_card":{"status":200,"content_type":"text/html","bytes":6885,"sha256":"0ae8e018e72bcac840700b49e691dd87595c3c6fe0bf94aa6ca9e47030b2fe66","final_host":"www.myfritz.net","final_path":"/devices","ms":179,"path":"/.well-known/agent-card.json","html":true,"json":null,"present":false}},"rings":{"v":1,"candidates":[],"mcp":[],"a2a":null,"webmcp":{"model_context":false,"bridge":false,"mcp_json":false,"link_rel":false,"any":false},"handshakes":0,"get_path":{"status":200,"content_type":"text/html","ms":182,"error":null},"get_host":{"status":530,"content_type":"text/plain","ms":188,"error":null}},"sf":[1,0,0,0,0,0,0,0,0],"derived":{"reachable":true,"home_class":"answers","robots_served":true,"robots_sitemap":true,"content_signal":false,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":["Meta-ExternalAgent"],"blocks_any_ai":true,"files":{"llms_txt":false,"agents_md":false,"agent_skills":false,"mcp_card":false,"api_catalog":false,"a2a_card":false},"any_capability":false,"ai_refused":[],"ai_thin":[],"aipref":0,"aipref_measured":true,"sf":[1,0,0,0,0,0,0,0,0],"rings":{"mcp_declared":false,"mcp_live":false,"mcp_auth":false,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":3,"fh":"www.myfritz.net","fp_other_host":[]},"derive_v":5,"frame":"top1k","shard":5},{"v":1,"kind":"crawlcheck-census-record","run":43,"domain":"msn.com","rank":71,"measured_at":"2026-11-13T08:07:15.177Z","home":{"browser":{"status":200,"content_type":"text/html","bytes":330557,"sha256":"cbaec7aa015c95a5af68f3c27d12bd770bcc3019fe09766704e178cf8914dff4","final_host":"www.msn.com","final_path":"/","ms":266,"identity":"Mobile browser","html":true},"gptbot":{"status":200,"content_type":"text/html","bytes":48781,"sha256":"037684139fdc97787527a8ae25767c474f357b74c4a7ff69529f01024d282dda","final_host":"www.msn.com","final_path":"/","ms":107,"identity":"GPTBot","html":true},"claudebot":{"status":200,"content_type":"text/html","bytes":48984,"sha256":"78a02ff33b186f96105d342f52e7b0b2034cd49eee6cf4c080deb85b8d9025b8","final_host":"www.msn.com","final_path":"/","ms":163,"identity":"ClaudeBot","html":true},"oaisearch":{"status":200,"content_type":"text/html","bytes":44802,"sha256":"49801b50b61ac79a8ac4a14f92acc71fd8deaa864638c0685ae0dcde5c30cb0e","final_host":"www.msn.com","final_path":"/","ms":117,"identity":"OAI-SearchBot","html":true},"perplexity":{"status":200,"content_type":"text/html","bytes":49000,"sha256":"00a8eb16fedee2132d3e3e182679fedd08e8284440dc97b5f6a5409aa3685217","final_host":"www.msn.com","final_path":"/","ms":131,"identity":"PerplexityBot","html":true}},"robots":{"status":200,"content_type":"text/plain","bytes":5245,"sha256":"aefbffa0a0a083e30637972046b42afc1569f7813c5687739a5ecdee0a6be4dd","final_host":"www.msn.com","final_path":"/robots.txt","ms":136,"html":false,"text":"User-agent: *\r\nDisallow: /*/health/search/filter\r\nDisallow: /spartan\r\nDisallow: /pt-ao\r\nDisallow: /*preview=\r\nDisallow: /*/autos/marketplace/product/\r\nDisallow: /*/cars/marketplace/product/\r\nDisallow: /*?item=\r\nDisallow: /*&item=\r\nDisallow: /codeCoverage\r\nDisallow: /vpBaselines\r\nDisallow: /msnBaselines\r\nDisallow: /*/studio\r\nDisallow: /*/indexnowdelete/\r\nAllow: /*/shopping/*/ar-\r\nAllow: /*/shopping/*/ss-\r\nAllow: /*/shopping/*/vi-\r\nDisallow: /*/shopping/\r\n\r\nUser-agent: AdsBot-Google\r\nAllow: /\r\nDisallow: /*/health/search/filter\r\n\r\nUser-agent: Verity\r\nCrawl-delay: 3\r\nDisallow: /*/health/search/filter\r\nDisallow: /spartan\r\nDisallow: /pt-ao\r\nDisallow: /*preview=\r\nDisallow: /*/autos/marketplace/product/\r\nDisallow: /*/cars/marketplace/product/\r\nDisallow: /*?item=\r\nDisallow: /*&item=\r\nDisallow: /codeCoverage\r\nDisallow: /vpBaselines\r\nDisallow: /msnBaselines\r\nDisallow: /*/studio\r\nDisallow: /*/indexnowdelete/\r\nAllow: /*/shopping/*/ar-\r\nAllow: /*/shopping/*/ss-\r\nAllow: /*/shopping/*/vi-\r\nDisallow: /*/shopping/\r\n\r\nUser-agent: ias_crawler\r\nCrawl-delay: 0.1\r\nDisallow: /*/health/search/filter\r\nDisallow: /spartan\r\nDisallow: /pt-ao\r\nDisallow: /*preview=\r\nDisallow: /*/autos/marketplace/product/\r\nDisallow: /*/cars/marketplace/product/\r\nDisallow: /*?item=\r\nDisallow: /*&item=\r\nDisallow: /codeCoverage\r\nDisallow: /vpBaselines\r\nDisallow: /msnBaselines\r\nDisallow: /*/studio\r\nDisallow: /*/indexnowdelete/\r\nAllow: /*/shopping/*/ar-\r\nAllow: /*/shopping/*/ss-\r\nAllow: /*/shopping/*/vi-\r\nDisallow: /*/shopping/\r\n\r\nUser-agent: Mediapartners-Google\r\nCrawl-delay: 0.5\r\nDisallow: /*/health/search/filter\r\nDisallow: /spartan\r\nDisallow: /pt-ao\r\nDisallow: /*preview=\r\nDisallow: /*/autos/marketplace/product/\r\nDisallow: /*/cars/marketplace/product/\r\nDisallow: /*?item=\r\nDisallow: /*&item=\r\nDisallow: /codeCoverage\r\nDisallow: /vpBaselines\r\nDisallow: /msnBaselines\r\nDisallow: /*/studio\r\nDisallow: /*/indexnowdelete/\r\nAllow: /*/shopping/*/ar-\r\nAllow: /*/shopping/*/ss-\r\nAllow: /*/shopping/*/vi-\r\nDisallow: /*/shopping/\r\n\r\nUser-agent: CriteoBot\r\nCrawl-delay: 0.5\r\nDisallow: /*/health/search/filter\r\nDisallow: /spartan\r\nDisallow: /pt-ao\r\nDisallow: /*preview=\r\nDisallow: /*/autos/marketplace/product/\r\nDisallow: /*/cars/marketplace/product/\r\nDisallow: /*?item=\r\nDisallow: /*&item=\r\nDisallow: /codeCoverage\r\nDisallow: /vpBaselines\r\nDisallow: /msnBaselines\r\nDisallow: /*/studio\r\nDisallow: /*/indexnowdelete/\r\nAllow: /*/shopping/*/ar-\r\nAllow: /*/shopping/*/ss-\r\nAllow: /*/shopping/*/vi-\r\nDisallow: /*/shopping/\r\n\r\nUser-agent: yandex.com/bot\r\nCrawl-delay: 1\r\nDisallow: /*/weather/\r\nDisallow: /*/clima/\r\nDisallow: /*/cuaca/\r\nDisallow: /*/eltiempo/\r\nDisallow: /*/el-tiempo/\r\nDisallow: /*/havadurumu/\r\nDisallow: /*/idojaras/\r\nDisallow: /*/meteo/\r\nDisallow: /*/meteorologia/\r\nDisallow: /*/pocasi/\r\nDisallow: /*/pogoda/\r\nDisallow: /*/saa/\r\nDisallow: /*/vader/\r\nDisallow: /*/vejr/\r\nDisallow: /*/weer/\r\nDisallow: /*/wetter/\r\nAllow: /*/shopping/*/ar-\r\nAllow: /*/shopping/*/ss-\r\nAllow: /*/shopping/*/vi-\r\nDisallow: /*/shopping/\r\n\r\nUser-agent: admantx-ussy04/3.2\r\nAllow: /\r\n\r\nUser-agent: TTD-Content\r\nAllow: /\r\n\r\nUser-agent: AmazonAdBot\r\nAllow: /\r\n\r\nUser-agent: AI2Bot\r\nUser-agent: Ai2Bot-Dolma\r\nUser-agent: Amazonbot\r\nUser-agent: anthropic-ai\r\nUser-agent: Applebot\r\nUser-agent: Applebot-Extended\r\nUser-agent: Bedrockbot\r\nUser-agent: Bytespider\r\nUser-agent: CCBot\r\nUser-agent: ChatGLM\r\nUser-agent: ChatGPT-User\r\nUser-agent: ClaudeBot\r\nUser-agent: Claude-User\r\nUser-agent: Claude-SearchBot \r\nUser-agent: Claude-Web\r\nUser-agent: cohere-ai\r\nUser-agent: cohere-training-data-crawler\r\nUser-agent: Cotoyogi\r\nUser-agent: DeepSeekBot\r\nUser-agent: Diffbot\r\nUser-agent: DuckAssistBot\r\nUser-agent: FacebookBot\r\nUser-agent: FriendlyCrawler\r\nUser-agent: Gensparkbot \r\nUser-agent: Google-CloudVertexBot\r\nUser-agent: Google-Extended\r\nUser-agent: GoogleOther\r\nUser-agent: GoogleOther-Image\r\nUser-agent: GoogleOther-Video\r\nUser-agent: GPTBot\r\nUser-agent: Grok\r\nUser-agent: iaskspider/2.0\r\nUser-agent: ICC-Crawler\r\nUser-agent: ImagesiftBot\r\nUser-agent: Img2dataset\r\nUser-agent: ISSCyberRiskCrawler\r\nUser-agent: Kangaroo Bot\r\nUser-agent: KunatoCrawler\r\nUser-agent: magpie-crawler\r\nUser-agent: Meltwater\r\nUser-agent: Meta-ExternalAgent\r\nUser-agent: Meta-ExternalFetcher\r\nUser-agent: MistralAI\r\nUser-agent: MistralAI-User\r\nUser-agent: Momentumbot\r\nUser-agent: OAI-SearchBot\r\nUser-agent: omgili\r\nUser-agent: Omgilibot\r\nUser-agent: Other\r\nUser-agent: PanguBot\r\nUser-agent: Perplexity-ai\r\nUser-agent: PerplexityBot\r\nUser-agent: Perplexity-User\r\nUser-agent: QualifiedBot\r\nUser-agent: Scrapy\r\nUser-agent: Seekr\r\nUser-agent: Sidetrade indexer bot\r\nUser-agent: TikTokSpider\r\nUser-agent: Timpibot\r\nUser-agent: Twitterbot\r\nUser-agent: VelenPublicWebCrawler\r\nUser-agent: WARDBot\r\nUser-agent: Webzio-Extended\r\nUser-agent: WRTNBot\r\nUser-agent: YouBot\r\nDisallow: /\r\n\r\nSitemap: https://www.msn.com/sitemaps/casualgames/sitemap-index.xml\r\nSitemap: https://www.msn.com/sitemaps/consumption/index.xml\r\nSitemap: https://www.msn.com/sitemaps/health/health-sitemap-index.xml\r\nSitemap: https://www.msn.com/staticsb/statics/latest/0/casualgames/sitemaps/sitemap-index.xml\r\nSitemap: https://www.msn.com/staticsb/statics/latest/0/finance/sitemaps/sitemap-index.xml\r\nSitemap: https://www.msn.com/sitemaps/weather/sitemap-index.xml","agents":{"GPTBot":false,"OAI-SearchBot":false,"ChatGPT-User":false,"ClaudeBot":false,"Claude-SearchBot":false,"PerplexityBot":false,"Google-Extended":false,"Applebot-Extended":false,"CCBot":false,"Bytespider":false,"Meta-ExternalAgent":false,"Amazonbot":false}},"sitemap":{"status":200,"content_type":"text/xml","bytes":5402,"sha256":"6357cf8b4653e26a07eca3573c30413dbf7e64d2e58f3b9ee45505f3f96607c8","final_host":"www.msn.com","final_path":"/sitemaps/casualgames/sitemap-index.xml","ms":54,"url":"https://www.msn.com/sitemaps/casualgames/sitemap-index.xml","declared":true,"xml":true},"files":{"llms_txt":{"status":404,"content_type":"text/html","bytes":19143,"sha256":"10fcf682b17971154f37ba43feed59f3f054c12d46e8d9dab193696175e960ff","final_host":"www.msn.com","final_path":"/llms.txt","ms":137,"path":"/llms.txt","html":true,"json":null,"present":false},"agents_md":{"status":404,"content_type":"text/html","bytes":19143,"sha256":"10fcf682b17971154f37ba43feed59f3f054c12d46e8d9dab193696175e960ff","final_host":"www.msn.com","final_path":"/agents.md","ms":132,"path":"/agents.md","html":true,"json":null,"present":false},"agent_skills":{"status":404,"content_type":"text/html","bytes":19143,"sha256":"10fcf682b17971154f37ba43feed59f3f054c12d46e8d9dab193696175e960ff","final_host":"www.msn.com","final_path":"/.well-known/agent-skills/index.json","ms":254,"path":"/.well-known/agent-skills/index.json","html":true,"json":null,"present":false},"mcp_card":{"status":404,"content_type":"text/html","bytes":19143,"sha256":"10fcf682b17971154f37ba43feed59f3f054c12d46e8d9dab193696175e960ff","final_host":"www.msn.com","final_path":"/.well-known/mcp/server-card.json","ms":205,"path":"/.well-known/mcp/server-card.json","html":true,"json":null,"present":false},"api_catalog":{"status":404,"content_type":"text/html","bytes":19143,"sha256":"10fcf682b17971154f37ba43feed59f3f054c12d46e8d9dab193696175e960ff","final_host":"www.msn.com","final_path":"/.well-known/api-catalog","ms":145,"path":"/.well-known/api-catalog","html":true,"json":null,"present":false},"a2a_card":{"status":404,"content_type":"text/html","bytes":19143,"sha256":"10fcf682b17971154f37ba43feed59f3f054c12d46e8d9dab193696175e960ff","final_host":"www.msn.com","final_path":"/.well-known/agent-card.json","ms":134,"path":"/.well-known/agent-card.json","html":true,"json":null,"present":false}},"rings":{"v":1,"candidates":[],"mcp":[],"a2a":null,"webmcp":{"model_context":false,"bridge":false,"mcp_json":false,"link_rel":false,"any":false},"handshakes":0,"get_path":{"status":404,"content_type":"text/html","ms":115,"error":null},"get_host":{"status":404,"content_type":"text/html","ms":295,"error":null}},"sf":[4,0,0,0,0,0,0,0,1],"derived":{"reachable":true,"home_class":"answers","robots_served":true,"robots_sitemap":true,"content_signal":false,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":["GPTBot","OAI-SearchBot","ChatGPT-User","ClaudeBot","Claude-SearchBot","PerplexityBot","Google-Extended","Applebot-Extended","CCBot","Bytespider","Meta-ExternalAgent","Amazonbot"],"blocks_any_ai":true,"files":{"llms_txt":false,"agents_md":false,"agent_skills":false,"mcp_card":false,"api_catalog":false,"a2a_card":false},"any_capability":false,"ai_refused":[],"ai_thin":["gptbot","claudebot","oaisearch","perplexity"],"aipref":0,"aipref_measured":true,"sf":[4,0,0,0,0,0,0,0,1],"rings":{"mcp_declared":false,"mcp_live":false,"mcp_auth":false,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":3,"fh":"www.msn.com","fp_other_host":[]},"derive_v":5,"frame":"top1k","shard":6},{"v":1,"kind":"crawlcheck-census-record","run":43,"domain":"zoom.us","rank":72,"measured_at":"2026-11-13T08:08:20.375Z","home":{"browser":{"status":200,"content_type":"text/html","bytes":321055,"sha256":"315259bc12d0bdf08de1349caab81bd10081e70d68dbb42719bf8be880f17e8c","final_host":"www.zoom.com","final_path":"/","ms":68,"identity":"Mobile browser","html":true,"fp":"b1bead5010e26a2d"},"gptbot":{"status":200,"content_type":"text/html","bytes":321055,"sha256":"5712934c39c24a81e8a4fa3034ec1a4391ea0766f93bffb3f4ebf40b286e413d","final_host":"www.zoom.com","final_path":"/","ms":43,"identity":"GPTBot","html":true,"fp":"b1bead5010e26a2d"},"claudebot":{"status":200,"content_type":"text/html","bytes":321055,"sha256":"af7c9d22f5a86e2ff6e357a64405c576c1d78b66768bebd2f0889cbfddaa9f02","final_host":"www.zoom.com","final_path":"/","ms":20,"identity":"ClaudeBot","html":true,"fp":"b1bead5010e26a2d"},"oaisearch":{"status":200,"content_type":"text/html","bytes":321055,"sha256":"3c2a592c57c9ee455daefc70d01ce27cb6e953f2c2646d7ecaa1ea21ef9f08a7","final_host":"www.zoom.com","final_path":"/","ms":54,"identity":"OAI-SearchBot","html":true,"fp":"b1bead5010e26a2d"},"perplexity":{"status":200,"content_type":"text/html","bytes":321055,"sha256":"994f5ab2744c5258dd9502defd5c8ea942d5f8e55cd45c316ebc00b8d904991d","final_host":"www.zoom.com","final_path":"/","ms":39,"identity":"PerplexityBot","html":true,"fp":"b1bead5010e26a2d"}},"robots":{"status":200,"content_type":"text/plain","bytes":2005,"sha256":"b3ed9f53f38c480b046b4d5a648aa7a879c6720957301506dc282b32fb2f5593","final_host":"zoom.us","final_path":"/robots.txt","ms":85,"html":false,"text":"User-Agent: *\nDisallow: /download/*/Zoom_launcher.exe\nDisallow: /download/*/zoomusLauncher.zip\nDisallow: /docs/image/new/brand/outdated/\nDisallow: /docs/doc/2-Page-All-Products.pdf\nDisallow: /docs/doc/zoom-apps-security-privacy-faq.pdf\nDisallow: /docs/doc/zoom-apps-data-management.pdf\nDisallow: /wc/*\nDisallow: /web/*\nDisallow: /share*\nDisallow: /support*\nDisallow: /j/*\nDisallow: /z/*\nDisallow: /s/*\nDisallow: /u/*\nDisallow: /w/*\nDisallow: /e/*\nDisallow: /account*\nDisallow: /start*\nDisallow: /postattendee*\nDisallow: /rec/*\nDisallow: /register*\nDisallow: /webinar/register/*\nDisallow: /webinar/registerevent/*\nDisallow: /meeting/register/*\nDisallow: /*oauth_signin\nDisallow: /my/*\nDisallow: /download2*\nDisallow: /*?\n\nUser-agent: Googlebot\nAllow: /join\n\nUser-agent: LinkedInBot\nAllow: /webinar/register/*\nAllow: /webinar/registerevent/*\nAllow: /meeting/register/*\n\nUser-agent: facebookexternalhit\nAllow: /webinar/register/*\nAllow: /webinar/registerevent/*\nAllow: /meeting/register/*\n\nUser-agent: Twitterbot\nAllow: /webinar/register/*\nAllow: /webinar/registerevent/*\nAllow: /meeting/register/*\n\nUser-agent: PinterestBot\nAllow: /webinar/register/*\nAllow: /webinar/registerevent/*\nAllow: /meeting/register/*\n\nUser-agent: InstagramBot\nAllow: /webinar/register/*\nAllow: /webinar/registerevent/*\nAllow: /meeting/register/*\n\nUser-agent: Slackbot\nAllow: /webinar/register/*\nAllow: /webinar/registerevent/*\nAllow: /meeting/register/*\n\nUser-agent: WhatsApp\nAllow: /webinar/register/*\nAllow: /webinar/registerevent/*\nAllow: /meeting/register/*\n\nUser-agent: redditbot\nAllow: /webinar/register/*\nAllow: /webinar/registerevent/*\nAllow: /meeting/register/*\n\nUser-agent: Tumblr\nAllow: /webinar/register/*\nAllow: /webinar/registerevent/*\nAllow: /meeting/register/*\n\nUser-agent: TikTokBot\nAllow: /webinar/register/*\nAllow: /webinar/registerevent/*\nAllow: /meeting/register/*\n\nUser-agent: Snapchat\nAllow: /webinar/register/*\nAllow: /webinar/registerevent/*\nAllow: /meeting/register/*\n\nSitemap: https://zoom.us/sitemap.xml","agents":{"GPTBot":true,"OAI-SearchBot":true,"ChatGPT-User":true,"ClaudeBot":true,"Claude-SearchBot":true,"PerplexityBot":true,"Google-Extended":true,"Applebot-Extended":true,"CCBot":true,"Bytespider":true,"Meta-ExternalAgent":true,"Amazonbot":true}},"sitemap":{"status":200,"content_type":"text/xml","bytes":432921,"sha256":"fc46be69d0c7b7529bb15d13a177daf4e337d6ec8f0b3e0875e0706c138489b8","final_host":"zoom.us","final_path":"/sitemap.xml","ms":134,"url":"https://zoom.us/sitemap.xml","declared":true,"xml":true},"files":{"llms_txt":{"status":404,"content_type":"text/html","bytes":526089,"sha256":"c6fc080e0062acb6874172eae1ce4dfea1dd43f9b40935d31d6b51c27336e74c","final_host":"zoom.us","final_path":"/llms.txt","ms":163,"path":"/llms.txt","html":true,"json":null,"present":false},"agents_md":{"status":404,"content_type":"text/html","bytes":526089,"sha256":"b333059de2e0f6959c4d94e0b1f4232178d613c95e344f8155ad8f279dc82292","final_host":"zoom.us","final_path":"/agents.md","ms":173,"path":"/agents.md","html":true,"json":null,"present":false},"agent_skills":{"status":200,"content_type":"application/json","bytes":28503,"sha256":"5f91ddfd957cda871aa39b74ea88144863f150eb7deb6a81129ef8c68315349a","final_host":"zoom.us","final_path":"/.well-known/agent-skills/index.json","ms":222,"path":"/.well-known/agent-skills/index.json","html":false,"json":true,"present":true},"mcp_card":{"status":200,"content_type":"application/json","bytes":10008,"sha256":"558897a06ccbaeec16bd6b82dbb87c046b56786987d5c9d95ff709b41df5fe20","final_host":"zoom.us","final_path":"/.well-known/mcp/server-card.json","ms":219,"path":"/.well-known/mcp/server-card.json","html":false,"json":true,"present":true},"api_catalog":{"status":200,"content_type":"application/linkset+json","bytes":34738,"sha256":"8fcefa04c48ae2f36da3b608b8726456e8a032255dff83e36fb3fb4129bad22f","final_host":"zoom.us","final_path":"/.well-known/api-catalog","ms":217,"path":"/.well-known/api-catalog","html":false,"json":true,"present":true},"a2a_card":{"status":404,"content_type":"text/html","bytes":526089,"sha256":"29927f7aef93815bd905e6284453a664858f12c6980ab890517a8af259779aa9","final_host":"zoom.us","final_path":"/.well-known/agent-card.json","ms":176,"path":"/.well-known/agent-card.json","html":true,"json":null,"present":false}},"rings":{"v":1,"candidates":[{"url":"https://mcp.zoom.us/mcp/zoom/streamable","source":"mcp_card"},{"url":"https://mcp.zoom.us/mcp/productivity/streamable","source":"mcp_card"},{"url":"https://mcp.zoom.us/mcp/canvas/streamable","source":"mcp_card"},{"url":"https://mcp.zoom.us/mcp/tasks/streamable","source":"mcp_card"},{"url":"https://mcp.zoom.us/mcp/workflow/streamable","source":"mcp_card"},{"url":"https://mcp.zoom.us/mcp/whiteboard/streamable","source":"mcp_card"}],"mcp":[{"url":"https://mcp.zoom.us/mcp/zoom/streamable","source":"mcp_card","outcome":"auth_required","status":401,"ms":113,"initialized":false,"protocol_version":null,"server":null,"tools":null,"tools_sha256":null,"auth_header":true},{"url":"https://mcp.zoom.us/mcp/productivity/streamable","source":"mcp_card","outcome":"auth_required","status":401,"ms":114,"initialized":false,"protocol_version":null,"server":null,"tools":null,"tools_sha256":null,"auth_header":true},{"url":"https://mcp.zoom.us/mcp/canvas/streamable","source":"mcp_card","outcome":"auth_required","status":401,"ms":113,"initialized":false,"protocol_version":null,"server":null,"tools":null,"tools_sha256":null,"auth_header":true}],"a2a":null,"webmcp":{"model_context":false,"bridge":false,"mcp_json":false,"link_rel":false,"any":false},"handshakes":3,"get_path":{"status":404,"content_type":"text/html","ms":187,"error":null},"get_host":{"status":404,"content_type":"text/html","ms":261,"error":null}},"sf":[-1,0,0,0,0,0,3,0,0],"derived":{"reachable":true,"home_class":"answers","robots_served":true,"robots_sitemap":true,"content_signal":false,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":[],"blocks_any_ai":false,"files":{"llms_txt":false,"agents_md":false,"agent_skills":true,"mcp_card":true,"api_catalog":true,"a2a_card":false},"any_capability":true,"ai_refused":[],"ai_thin":[],"aipref":0,"aipref_measured":true,"sf":[-1,0,0,0,0,0,3,0,0],"rings":{"mcp_declared":true,"mcp_live":false,"mcp_auth":true,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":3,"fh":"www.zoom.com","fp_other_host":[],"fp":"b1bead5010e26a2d","fp_bits":{"gptbot":0,"claudebot":0,"oaisearch":0,"perplexity":0}},"derive_v":5,"frame":"top1k","shard":7},{"v":1,"kind":"crawlcheck-census-record","run":43,"domain":"ntp.org","rank":73,"measured_at":"2026-11-13T07:56:55.238Z","home":{"browser":{"status":200,"content_type":"text/html","bytes":9114,"sha256":"9812869b85080fba2b26749348b535e391dfdc861753788aefc52b86262e3c41","final_host":"www.ntp.org","final_path":"/","ms":1911,"identity":"Mobile browser","html":true,"fp":"4d02f33496f546f0"},"gptbot":{"status":200,"content_type":"text/html","bytes":9114,"sha256":"9812869b85080fba2b26749348b535e391dfdc861753788aefc52b86262e3c41","final_host":"www.ntp.org","final_path":"/","ms":1532,"identity":"GPTBot","html":true,"fp":"4d02f33496f546f0"},"claudebot":{"status":200,"content_type":"text/html","bytes":9114,"sha256":"9812869b85080fba2b26749348b535e391dfdc861753788aefc52b86262e3c41","final_host":"www.ntp.org","final_path":"/","ms":1864,"identity":"ClaudeBot","html":true,"fp":"4d02f33496f546f0"},"oaisearch":{"status":200,"content_type":"text/html","bytes":9114,"sha256":"9812869b85080fba2b26749348b535e391dfdc861753788aefc52b86262e3c41","final_host":"www.ntp.org","final_path":"/","ms":1788,"identity":"OAI-SearchBot","html":true,"fp":"4d02f33496f546f0"},"perplexity":{"status":200,"content_type":"text/html","bytes":9114,"sha256":"9812869b85080fba2b26749348b535e391dfdc861753788aefc52b86262e3c41","final_host":"www.ntp.org","final_path":"/","ms":1546,"identity":"PerplexityBot","html":true,"fp":"4d02f33496f546f0"}},"robots":{"status":200,"content_type":"text/plain","bytes":16,"sha256":"5a5a4434f108ef034c256b9d3e169e516fb806aa6f6353e53eff9655732fbb79","final_host":"www.ntp.org","final_path":"/robots.txt","ms":1939,"html":false,"text":"\nUser-agent: *\n\n","agents":{"GPTBot":true,"OAI-SearchBot":true,"ChatGPT-User":true,"ClaudeBot":true,"Claude-SearchBot":true,"PerplexityBot":true,"Google-Extended":true,"Applebot-Extended":true,"CCBot":true,"Bytespider":true,"Meta-ExternalAgent":true,"Amazonbot":true}},"sitemap":{"status":200,"content_type":"application/xml","bytes":420,"sha256":"2e06093ff2fdc0a20aab23a1b62a673884891c2b7db081ac3846c375598cf4a9","final_host":"www.ntp.org","final_path":"/sitemap.xml","ms":344,"url":"https://ntp.org/sitemap.xml","declared":false,"xml":true},"files":{"llms_txt":{"status":404,"content_type":"text/html","bytes":6601,"sha256":"ce78ad119af14e88e0bd97553a54a70a036626840ba00e22b9aebc2022165394","final_host":"www.ntp.org","final_path":"/llms.txt","ms":487,"path":"/llms.txt","html":true,"json":null,"present":false},"agents_md":{"status":404,"content_type":"text/html","bytes":6601,"sha256":"ce78ad119af14e88e0bd97553a54a70a036626840ba00e22b9aebc2022165394","final_host":"www.ntp.org","final_path":"/agents.md","ms":543,"path":"/agents.md","html":true,"json":null,"present":false},"agent_skills":{"status":404,"content_type":"text/html","bytes":6601,"sha256":"ce78ad119af14e88e0bd97553a54a70a036626840ba00e22b9aebc2022165394","final_host":"www.ntp.org","final_path":"/.well-known/agent-skills/index.json","ms":821,"path":"/.well-known/agent-skills/index.json","html":true,"json":null,"present":false},"mcp_card":{"status":404,"content_type":"text/html","bytes":6601,"sha256":"ce78ad119af14e88e0bd97553a54a70a036626840ba00e22b9aebc2022165394","final_host":"www.ntp.org","final_path":"/.well-known/mcp/server-card.json","ms":301,"path":"/.well-known/mcp/server-card.json","html":true,"json":null,"present":false},"api_catalog":{"status":404,"content_type":"text/html","bytes":6601,"sha256":"ce78ad119af14e88e0bd97553a54a70a036626840ba00e22b9aebc2022165394","final_host":"www.ntp.org","final_path":"/.well-known/api-catalog","ms":562,"path":"/.well-known/api-catalog","html":true,"json":null,"present":false},"a2a_card":{"status":404,"content_type":"text/html","bytes":6601,"sha256":"ce78ad119af14e88e0bd97553a54a70a036626840ba00e22b9aebc2022165394","final_host":"www.ntp.org","final_path":"/.well-known/agent-card.json","ms":665,"path":"/.well-known/agent-card.json","html":true,"json":null,"present":false}},"rings":{"v":1,"candidates":[],"mcp":[],"a2a":null,"webmcp":{"model_context":false,"bridge":false,"mcp_json":false,"link_rel":false,"any":false},"handshakes":0,"get_path":{"status":404,"content_type":"text/html","ms":299,"error":null},"get_host":{"status":530,"content_type":"text/plain","ms":34,"error":null}},"sf":[1,0,0,0,0,0,3,0,0],"derived":{"reachable":true,"home_class":"answers","robots_served":true,"robots_sitemap":false,"content_signal":false,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":[],"blocks_any_ai":false,"files":{"llms_txt":false,"agents_md":false,"agent_skills":false,"mcp_card":false,"api_catalog":false,"a2a_card":false},"any_capability":false,"ai_refused":[],"ai_thin":[],"aipref":0,"aipref_measured":true,"sf":[1,0,0,0,0,0,3,0,0],"rings":{"mcp_declared":false,"mcp_live":false,"mcp_auth":false,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":3,"fh":"www.ntp.org","fp_other_host":[],"fp":"4d02f33496f546f0","fp_bits":{"gptbot":0,"claudebot":0,"oaisearch":0,"perplexity":0}},"derive_v":5,"frame":"top1k","shard":0},{"v":1,"kind":"crawlcheck-census-record","run":43,"domain":"qq.com","rank":74,"measured_at":"2026-11-13T07:57:55.549Z","home":{"browser":{"status":200,"content_type":"text/html","bytes":133979,"sha256":"57e7418a0a4a2b7ee2cea755d2dcc921d06fc23f08fd4a32902b2c0d14c2d6be","final_host":"www.qq.com","final_path":"/","ms":1893,"identity":"Mobile browser","html":true,"fp":"0373645632e76ad2"},"gptbot":{"status":200,"content_type":"text/html","bytes":133953,"sha256":"c27cd080cf48c9ec5e2df4e784aa9efda51d5175160f35712e808c9c8d861cf6","final_host":"www.qq.com","final_path":"/","ms":1448,"identity":"GPTBot","html":true,"fp":"0373645632e76ad2"},"claudebot":{"status":200,"content_type":"text/html","bytes":134016,"sha256":"ec823ce43392a030807d4ac71f29b3f7074dc3c4593dd50382071cb0f74757d9","final_host":"www.qq.com","final_path":"/","ms":1688,"identity":"ClaudeBot","html":true,"fp":"0373645632e76ad2"},"oaisearch":{"status":200,"content_type":"text/html","bytes":134016,"sha256":"21d51adf7f55b3528a0e7a4d4e940043f5c20d9a1f0d9c09b97b7e2c29d8d557","final_host":"www.qq.com","final_path":"/","ms":4051,"identity":"OAI-SearchBot","html":true,"fp":"0373645632e76ad2"},"perplexity":{"status":200,"content_type":"text/html","bytes":134007,"sha256":"9bd2ea6fe32d998cabbecf03512477dbb8762a8811c08b36db4fdc6700140aad","final_host":"www.qq.com","final_path":"/","ms":4763,"identity":"PerplexityBot","html":true,"fp":"0373645632e76ad2"}},"robots":{"status":200,"content_type":"text/plain","bytes":71,"sha256":"1f5d4ddc6513b31998acadbb56fe02e8b9eb244a3eb64d36f446f146e0bc3d5f","final_host":"www.qq.com","final_path":"/robots.txt","ms":6314,"html":false,"text":"User-agent: *\nDisallow: \nSitemap: https://news.qq.com/sitemap/index.xml","agents":{"GPTBot":true,"OAI-SearchBot":true,"ChatGPT-User":true,"ClaudeBot":true,"Claude-SearchBot":true,"PerplexityBot":true,"Google-Extended":true,"Applebot-Extended":true,"CCBot":true,"Bytespider":true,"Meta-ExternalAgent":true,"Amazonbot":true}},"sitemap":{"status":200,"content_type":"application/xml","bytes":105125,"sha256":"d7387f3259f3d6ed9ef2db9969867dd7c85728ec91f07c986b9a34ccbac4db69","final_host":"news.qq.com","final_path":"/sitemap/index.xml","ms":641,"url":"https://news.qq.com/sitemap/index.xml","declared":true,"xml":true},"files":{"llms_txt":{"status":200,"content_type":"text/html","bytes":4183,"sha256":"dd3ca8f2a7ee2f282645a07d06786a1fdb6f0caa1e090063805dc51f525c2464","final_host":"h5.news.qq.com","final_path":"/static/babyhome.htm","ms":576,"path":"/llms.txt","html":true,"json":null,"present":false},"agents_md":{"status":200,"content_type":"text/html","bytes":4183,"sha256":"dd3ca8f2a7ee2f282645a07d06786a1fdb6f0caa1e090063805dc51f525c2464","final_host":"h5.news.qq.com","final_path":"/static/babyhome.htm","ms":548,"path":"/agents.md","html":true,"json":null,"present":false},"agent_skills":{"status":200,"content_type":"text/html","bytes":4183,"sha256":"dd3ca8f2a7ee2f282645a07d06786a1fdb6f0caa1e090063805dc51f525c2464","final_host":"h5.news.qq.com","final_path":"/static/babyhome.htm","ms":547,"path":"/.well-known/agent-skills/index.json","html":true,"json":null,"present":false},"mcp_card":{"status":200,"content_type":"text/html","bytes":4183,"sha256":"dd3ca8f2a7ee2f282645a07d06786a1fdb6f0caa1e090063805dc51f525c2464","final_host":"h5.news.qq.com","final_path":"/static/babyhome.htm","ms":1071,"path":"/.well-known/mcp/server-card.json","html":true,"json":null,"present":false},"api_catalog":{"status":200,"content_type":"text/html","bytes":4183,"sha256":"dd3ca8f2a7ee2f282645a07d06786a1fdb6f0caa1e090063805dc51f525c2464","final_host":"h5.news.qq.com","final_path":"/static/babyhome.htm","ms":547,"path":"/.well-known/api-catalog","html":true,"json":null,"present":false},"a2a_card":{"status":200,"content_type":"text/html","bytes":4183,"sha256":"dd3ca8f2a7ee2f282645a07d06786a1fdb6f0caa1e090063805dc51f525c2464","final_host":"h5.news.qq.com","final_path":"/static/babyhome.htm","ms":547,"path":"/.well-known/agent-card.json","html":true,"json":null,"present":false}},"rings":{"v":1,"candidates":[],"mcp":[],"a2a":null,"webmcp":{"model_context":false,"bridge":false,"mcp_json":false,"link_rel":false,"any":false},"handshakes":0,"get_path":{"status":200,"content_type":"text/html","ms":21,"error":null},"get_host":{"status":530,"content_type":"text/plain","ms":163,"error":null}},"sf":[4,0,0,0,0,0,1,0,0],"derived":{"reachable":true,"home_class":"answers","robots_served":true,"robots_sitemap":true,"content_signal":false,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":[],"blocks_any_ai":false,"files":{"llms_txt":false,"agents_md":false,"agent_skills":false,"mcp_card":false,"api_catalog":false,"a2a_card":false},"any_capability":false,"ai_refused":[],"ai_thin":[],"aipref":0,"aipref_measured":true,"sf":[4,0,0,0,0,0,1,0,0],"rings":{"mcp_declared":false,"mcp_live":false,"mcp_auth":false,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":3,"fh":"www.qq.com","fp_other_host":[],"fp":"0373645632e76ad2","fp_bits":{"gptbot":0,"claudebot":0,"oaisearch":0,"perplexity":0}},"derive_v":5,"frame":"top1k","shard":1},{"v":1,"kind":"crawlcheck-census-record","run":43,"domain":"root-servers.net","rank":75,"measured_at":"2026-11-13T07:58:58.649Z","home":{"browser":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"root-servers.net","final_path":"/","ms":71,"identity":"Mobile browser","html":false},"gptbot":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"root-servers.net","final_path":"/","ms":0,"identity":"GPTBot","html":false},"claudebot":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"root-servers.net","final_path":"/","ms":1,"identity":"ClaudeBot","html":false},"oaisearch":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"root-servers.net","final_path":"/","ms":1,"identity":"OAI-SearchBot","html":false},"perplexity":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"root-servers.net","final_path":"/","ms":0,"identity":"PerplexityBot","html":false}},"robots":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"root-servers.net","final_path":"/robots.txt","ms":1,"html":false,"text":null,"agents":null},"sitemap":null,"files":null,"derived":{"reachable":false,"home_class":"no_site","robots_served":false,"robots_sitemap":false,"content_signal":false,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":[],"blocks_any_ai":false,"files":{"llms_txt":false,"agents_md":false,"agent_skills":false,"mcp_card":false,"api_catalog":false,"a2a_card":false},"any_capability":false,"ai_refused":[],"ai_thin":[],"rings":{"mcp_declared":false,"mcp_live":false,"mcp_auth":false,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":3,"fh":"root-servers.net","fp_other_host":[]},"derive_v":5,"frame":"top1k","shard":2},{"v":1,"kind":"crawlcheck-census-record","run":43,"domain":"nginx.org","rank":76,"measured_at":"2026-11-13T07:59:19.049Z","home":{"browser":{"status":200,"content_type":"text/html","bytes":6503,"sha256":"c89c77b64bc4de01e4ecb86b904770574e58744f287d9332c5637351b475b6d3","final_host":"nginx.org","final_path":"/","ms":431,"identity":"Mobile browser","html":true,"fp":"d5a07d05d1501da2"},"gptbot":{"status":200,"content_type":"text/html","bytes":6503,"sha256":"c89c77b64bc4de01e4ecb86b904770574e58744f287d9332c5637351b475b6d3","final_host":"nginx.org","final_path":"/","ms":422,"identity":"GPTBot","html":true,"fp":"d5a07d05d1501da2"},"claudebot":{"status":200,"content_type":"text/html","bytes":6503,"sha256":"c89c77b64bc4de01e4ecb86b904770574e58744f287d9332c5637351b475b6d3","final_host":"nginx.org","final_path":"/","ms":41,"identity":"ClaudeBot","html":true,"fp":"d5a07d05d1501da2"},"oaisearch":{"status":200,"content_type":"text/html","bytes":6503,"sha256":"c89c77b64bc4de01e4ecb86b904770574e58744f287d9332c5637351b475b6d3","final_host":"nginx.org","final_path":"/","ms":40,"identity":"OAI-SearchBot","html":true,"fp":"d5a07d05d1501da2"},"perplexity":{"status":200,"content_type":"text/html","bytes":6503,"sha256":"c89c77b64bc4de01e4ecb86b904770574e58744f287d9332c5637351b475b6d3","final_host":"nginx.org","final_path":"/","ms":56,"identity":"PerplexityBot","html":true,"fp":"d5a07d05d1501da2"}},"robots":{"status":200,"content_type":"text/plain","bytes":72,"sha256":"8f5b42f5d23ca623fa42f917a5c90b8677ef8d7c9b5ffc0324bbea8e7067f26e","final_host":"nginx.org","final_path":"/robots.txt","ms":292,"html":false,"text":"User-Agent: *\nDisallow: /libxslt/\nSitemap: http://nginx.org/sitemap.xml\n","agents":{"GPTBot":true,"OAI-SearchBot":true,"ChatGPT-User":true,"ClaudeBot":true,"Claude-SearchBot":true,"PerplexityBot":true,"Google-Extended":true,"Applebot-Extended":true,"CCBot":true,"Bytespider":true,"Meta-ExternalAgent":true,"Amazonbot":true}},"sitemap":{"status":200,"content_type":"text/xml","bytes":41548,"sha256":"f74d21966e65e01a5acc8aadf1d2811dabe401dcb5cd497c9e2a81f0f4037414","final_host":"nginx.org","final_path":"/sitemap.xml","ms":129,"url":"http://nginx.org/sitemap.xml","declared":true,"xml":true},"files":{"llms_txt":{"status":404,"content_type":"text/html","bytes":2078,"sha256":"3bc24a23c571aa75630ad6e6c6b44a981f381a68981e8cfba5eee97eb64bf587","final_host":"nginx.org","final_path":"/llms.txt","ms":38,"path":"/llms.txt","html":true,"json":null,"present":false},"agents_md":{"status":404,"content_type":"text/html","bytes":2078,"sha256":"3bc24a23c571aa75630ad6e6c6b44a981f381a68981e8cfba5eee97eb64bf587","final_host":"nginx.org","final_path":"/agents.md","ms":71,"path":"/agents.md","html":true,"json":null,"present":false},"agent_skills":{"status":404,"content_type":"text/html","bytes":2078,"sha256":"3bc24a23c571aa75630ad6e6c6b44a981f381a68981e8cfba5eee97eb64bf587","final_host":"nginx.org","final_path":"/.well-known/agent-skills/index.json","ms":180,"path":"/.well-known/agent-skills/index.json","html":true,"json":null,"present":false},"mcp_card":{"status":404,"content_type":"text/html","bytes":2078,"sha256":"3bc24a23c571aa75630ad6e6c6b44a981f381a68981e8cfba5eee97eb64bf587","final_host":"nginx.org","final_path":"/.well-known/mcp/server-card.json","ms":43,"path":"/.well-known/mcp/server-card.json","html":true,"json":null,"present":false},"api_catalog":{"status":404,"content_type":"text/html","bytes":2078,"sha256":"3bc24a23c571aa75630ad6e6c6b44a981f381a68981e8cfba5eee97eb64bf587","final_host":"nginx.org","final_path":"/.well-known/api-catalog","ms":198,"path":"/.well-known/api-catalog","html":true,"json":null,"present":false},"a2a_card":{"status":404,"content_type":"text/html","bytes":2078,"sha256":"3bc24a23c571aa75630ad6e6c6b44a981f381a68981e8cfba5eee97eb64bf587","final_host":"nginx.org","final_path":"/.well-known/agent-card.json","ms":235,"path":"/.well-known/agent-card.json","html":true,"json":null,"present":false}},"rings":{"v":1,"candidates":[],"mcp":[],"a2a":null,"webmcp":{"model_context":false,"bridge":false,"mcp_json":false,"link_rel":false,"any":false},"handshakes":0,"get_path":{"status":404,"content_type":"text/html","ms":41,"error":null},"get_host":{"status":530,"content_type":"text/plain","ms":125,"error":null}},"sf":[1,0,0,0,0,0,0,0,0],"derived":{"reachable":true,"home_class":"answers","robots_served":true,"robots_sitemap":true,"content_signal":false,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":[],"blocks_any_ai":false,"files":{"llms_txt":false,"agents_md":false,"agent_skills":false,"mcp_card":false,"api_catalog":false,"a2a_card":false},"any_capability":false,"ai_refused":[],"ai_thin":[],"aipref":0,"aipref_measured":true,"sf":[1,0,0,0,0,0,0,0,0],"rings":{"mcp_declared":false,"mcp_live":false,"mcp_auth":false,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":3,"fh":"nginx.org","fp_other_host":[],"fp":"d5a07d05d1501da2","fp_bits":{"gptbot":0,"claudebot":0,"oaisearch":0,"perplexity":0}},"derive_v":5,"frame":"top1k","shard":3},{"v":1,"kind":"crawlcheck-census-record","run":43,"domain":"akam.net","rank":77,"measured_at":"2026-11-13T08:05:49.397Z","home":{"browser":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"akam.net","final_path":"/","ms":34,"identity":"Mobile browser","html":false},"gptbot":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"akam.net","final_path":"/","ms":1,"identity":"GPTBot","html":false},"claudebot":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"akam.net","final_path":"/","ms":1,"identity":"ClaudeBot","html":false},"oaisearch":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"akam.net","final_path":"/","ms":1,"identity":"OAI-SearchBot","html":false},"perplexity":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"akam.net","final_path":"/","ms":0,"identity":"PerplexityBot","html":false}},"robots":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"akam.net","final_path":"/robots.txt","ms":4,"html":false,"text":null,"agents":null},"sitemap":null,"files":null,"derived":{"reachable":false,"home_class":"no_site","robots_served":false,"robots_sitemap":false,"content_signal":false,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":[],"blocks_any_ai":false,"files":{"llms_txt":false,"agents_md":false,"agent_skills":false,"mcp_card":false,"api_catalog":false,"a2a_card":false},"any_capability":false,"ai_refused":[],"ai_thin":[],"rings":{"mcp_declared":false,"mcp_live":false,"mcp_auth":false,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":3,"fh":"akam.net","fp_other_host":[]},"derive_v":5,"frame":"top1k","shard":4},{"v":1,"kind":"crawlcheck-census-record","run":43,"domain":"tiktokcdn.com","rank":78,"measured_at":"2026-11-13T08:07:00.657Z","home":{"browser":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"tiktokcdn.com","final_path":"/","ms":5,"identity":"Mobile browser","html":false},"gptbot":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"tiktokcdn.com","final_path":"/","ms":0,"identity":"GPTBot","html":false},"claudebot":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"tiktokcdn.com","final_path":"/","ms":1,"identity":"ClaudeBot","html":false},"oaisearch":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"tiktokcdn.com","final_path":"/","ms":0,"identity":"OAI-SearchBot","html":false},"perplexity":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"tiktokcdn.com","final_path":"/","ms":1,"identity":"PerplexityBot","html":false}},"robots":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"tiktokcdn.com","final_path":"/robots.txt","ms":0,"html":false,"text":null,"agents":null},"sitemap":null,"files":null,"derived":{"reachable":false,"home_class":"no_site","robots_served":false,"robots_sitemap":false,"content_signal":false,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":[],"blocks_any_ai":false,"files":{"llms_txt":false,"agents_md":false,"agent_skills":false,"mcp_card":false,"api_catalog":false,"a2a_card":false},"any_capability":false,"ai_refused":[],"ai_thin":[],"rings":{"mcp_declared":false,"mcp_live":false,"mcp_auth":false,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":3,"fh":"tiktokcdn.com","fp_other_host":[]},"derive_v":5,"frame":"top1k","shard":5},{"v":1,"kind":"crawlcheck-census-record","run":43,"domain":"edgekey.net","rank":79,"measured_at":"2026-11-13T08:07:15.177Z","home":{"browser":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"edgekey.net","final_path":"/","ms":8,"identity":"Mobile browser","html":false},"gptbot":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"edgekey.net","final_path":"/","ms":1,"identity":"GPTBot","html":false},"claudebot":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"edgekey.net","final_path":"/","ms":2,"identity":"ClaudeBot","html":false},"oaisearch":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"edgekey.net","final_path":"/","ms":1,"identity":"OAI-SearchBot","html":false},"perplexity":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"edgekey.net","final_path":"/","ms":2,"identity":"PerplexityBot","html":false}},"robots":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"edgekey.net","final_path":"/robots.txt","ms":1,"html":false,"text":null,"agents":null},"sitemap":null,"files":null,"derived":{"reachable":false,"home_class":"no_site","robots_served":false,"robots_sitemap":false,"content_signal":false,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":[],"blocks_any_ai":false,"files":{"llms_txt":false,"agents_md":false,"agent_skills":false,"mcp_card":false,"api_catalog":false,"a2a_card":false},"any_capability":false,"ai_refused":[],"ai_thin":[],"rings":{"mcp_declared":false,"mcp_live":false,"mcp_auth":false,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":3,"fh":"edgekey.net","fp_other_host":[]},"derive_v":5,"frame":"top1k","shard":6},{"v":1,"kind":"crawlcheck-census-record","run":43,"domain":"baidu.com","rank":80,"measured_at":"2026-11-13T08:08:20.375Z","home":{"browser":{"status":200,"content_type":"text/html","bytes":81142,"sha256":"09d9c37190263f96de6538e3dd03c8f7c91866bd69af434c23c17251c635f2a8","final_host":"www.baidu.com","final_path":"/","ms":2396,"identity":"Mobile browser","html":true},"gptbot":{"status":200,"content_type":"text/html","bytes":722668,"sha256":"96b9b89f0fc5b4ceb84e5e598f3b9d46e0294b678ea387e67976f5183f728e54","final_host":"www.baidu.com","final_path":"/","ms":2795,"identity":"GPTBot","html":true,"fp":"ea77dafc63cf13f7"},"claudebot":{"status":200,"content_type":"text/html","bytes":722897,"sha256":"bf07cae6bae3b8e49a25136285f046b890e80c706594b82a7a437b5d209ae9fb","final_host":"www.baidu.com","final_path":"/","ms":2842,"identity":"ClaudeBot","html":true,"fp":"ea77daec63cf13f7"},"oaisearch":{"status":200,"content_type":"text/html","bytes":722596,"sha256":"74d668afb25e83106ee5bc62ad29591702f4438a3dfb0a89f755fa6353e11437","final_host":"www.baidu.com","final_path":"/","ms":2808,"identity":"OAI-SearchBot","html":true,"fp":"ea77dafc63cf13f7"},"perplexity":{"status":null,"error":"timeout","ms":8000,"identity":"PerplexityBot","html":null}},"robots":{"status":200,"content_type":"text/plain","bytes":2814,"sha256":"c28ebeffb7baf9bd6407109e80dc95d87c75c0eaecc20b23884cb33111cc5df0","final_host":"www.baidu.com","final_path":"/robots.txt","ms":1955,"html":false,"text":"User-agent: Baiduspider\nDisallow: /baidu\nDisallow: /s?\nDisallow: /ulink?\nDisallow: /link?\nDisallow: /home/news/data/\nDisallow: /bh\n\nUser-agent: Googlebot\nDisallow: /baidu\nDisallow: /s?\nDisallow: /shifen/\nDisallow: /homepage/\nDisallow: /cpro\nDisallow: /ulink?\nDisallow: /link?\nDisallow: /home/news/data/\nDisallow: /bh\n\nUser-agent: MSNBot\nDisallow: /baidu\nDisallow: /s?\nDisallow: /shifen/\nDisallow: /homepage/\nDisallow: /cpro\nDisallow: /ulink?\nDisallow: /link?\nDisallow: /home/news/data/\nDisallow: /bh\n\nUser-agent: Baiduspider-image\nDisallow: /baidu\nDisallow: /s?\nDisallow: /shifen/\nDisallow: /homepage/\nDisallow: /cpro\nDisallow: /ulink?\nDisallow: /link?\nDisallow: /home/news/data/\nDisallow: /bh\n\nUser-agent: YoudaoBot\nDisallow: /baidu\nDisallow: /s?\nDisallow: /shifen/\nDisallow: /homepage/\nDisallow: /cpro\nDisallow: /ulink?\nDisallow: /link?\nDisallow: /home/news/data/\nDisallow: /bh\n\nUser-agent: Sogou web spider\nDisallow: /baidu\nDisallow: /s?\nDisallow: /shifen/\nDisallow: /homepage/\nDisallow: /cpro\nDisallow: /ulink?\nDisallow: /link?\nDisallow: /home/news/data/\nDisallow: /bh\n\nUser-agent: Sogou inst spider\nDisallow: /baidu\nDisallow: /s?\nDisallow: /shifen/\nDisallow: /homepage/\nDisallow: /cpro\nDisallow: /ulink?\nDisallow: /link?\nDisallow: /home/news/data/\nDisallow: /bh\n\nUser-agent: Sogou spider2\nDisallow: /baidu\nDisallow: /s?\nDisallow: /shifen/\nDisallow: /homepage/\nDisallow: /cpro\nDisallow: /ulink?\nDisallow: /link?\nDisallow: /home/news/data/\nDisallow: /bh\n\nUser-agent: Sogou blog\nDisallow: /baidu\nDisallow: /s?\nDisallow: /shifen/\nDisallow: /homepage/\nDisallow: /cpro\nDisallow: /ulink?\nDisallow: /link?\nDisallow: /home/news/data/\nDisallow: /bh\n\nUser-agent: Sogou News Spider\nDisallow: /baidu\nDisallow: /s?\nDisallow: /shifen/\nDisallow: /homepage/\nDisallow: /cpro\nDisallow: /ulink?\nDisallow: /link?\nDisallow: /home/news/data/\nDisallow: /bh\n\nUser-agent: Sogou Orion spider\nDisallow: /baidu\nDisallow: /s?\nDisallow: /shifen/\nDisallow: /homepage/\nDisallow: /cpro\nDisallow: /ulink?\nDisallow: /link?\nDisallow: /home/news/data/\nDisallow: /bh\n\nUser-agent: ChinasoSpider\nDisallow: /baidu\nDisallow: /s?\nDisallow: /shifen/\nDisallow: /homepage/\nDisallow: /cpro\nDisallow: /ulink?\nDisallow: /link?\nDisallow: /home/news/data/\nDisallow: /bh\n\nUser-agent: Sosospider\nDisallow: /baidu\nDisallow: /s?\nDisallow: /shifen/\nDisallow: /homepage/\nDisallow: /cpro\nDisallow: /ulink?\nDisallow: /link?\nDisallow: /home/news/data/\nDisallow: /bh\n\n\nUser-agent: yisouspider\nDisallow: /baidu\nDisallow: /s?\nDisallow: /shifen/\nDisallow: /homepage/\nDisallow: /cpro\nDisallow: /ulink?\nDisallow: /link?\nDisallow: /home/news/data/\nDisallow: /bh\n\nUser-agent: EasouSpider\nDisallow: /baidu\nDisallow: /s?\nDisallow: /shifen/\nDisallow: /homepage/\nDisallow: /cpro\nDisallow: /ulink?\nDisallow: /link?\nDisallow: /home/news/data/\nDisallow: /bh\n\nUser-agent: *\nDisallow: /\n","agents":{"GPTBot":false,"OAI-SearchBot":false,"ChatGPT-User":false,"ClaudeBot":false,"Claude-SearchBot":false,"PerplexityBot":false,"Google-Extended":false,"Applebot-Extended":false,"CCBot":false,"Bytespider":false,"Meta-ExternalAgent":false,"Amazonbot":false}},"sitemap":{"status":null,"error":"timeout","ms":8000,"url":"https://baidu.com/sitemap.xml","declared":false,"xml":false},"files":{"llms_txt":{"status":404,"content_type":"text/html","bytes":206,"sha256":"0da006306ca67b9d88367bc1a63929b073ed36fb046b9b61695f934bfa544b89","final_host":"www.baidu.com","final_path":"/llms.txt","ms":1287,"path":"/llms.txt","html":true,"json":null,"present":false},"agents_md":{"status":null,"error":"timeout","ms":8000,"path":"/agents.md","html":null,"json":null,"present":false},"agent_skills":{"status":null,"error":"timeout","ms":8000,"path":"/.well-known/agent-skills/index.json","html":null,"json":null,"present":false},"mcp_card":{"status":404,"content_type":"text/html","bytes":230,"sha256":"ff2106cf9a2975014654029d3a652abeccb0b8c216730f442ddabc48f763a431","final_host":"www.baidu.com","final_path":"/.well-known/mcp/server-card.json","ms":436,"path":"/.well-known/mcp/server-card.json","html":true,"json":null,"present":false},"api_catalog":{"status":404,"content_type":"text/html","bytes":221,"sha256":"923d0d4fc2d80a5fe8f23656d1f52a907c9fe33c7380164a03f29abd4b3979b6","final_host":"www.baidu.com","final_path":"/.well-known/api-catalog","ms":1265,"path":"/.well-known/api-catalog","html":true,"json":null,"present":false},"a2a_card":{"status":404,"content_type":"text/html","bytes":225,"sha256":"8ec3ff7fbb679822567977a30006dbab75591df75733881c63ffe988553cf549","final_host":"www.baidu.com","final_path":"/.well-known/agent-card.json","ms":400,"path":"/.well-known/agent-card.json","html":true,"json":null,"present":false}},"device_checked":{"desktop_bits":null,"status":0},"rings":{"v":1,"candidates":[],"mcp":[],"a2a":null,"webmcp":{"model_context":false,"bridge":false,"mcp_json":false,"link_rel":false,"any":false},"handshakes":0,"get_path":{"status":404,"content_type":"text/html","ms":468,"error":null},"get_host":{"status":404,"content_type":"text/html","ms":5501,"error":null}},"sf":[4,0,0,0,0,0,2,0,0],"derived":{"reachable":true,"home_class":"answers","robots_served":true,"robots_sitemap":false,"content_signal":false,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":["GPTBot","OAI-SearchBot","ChatGPT-User","ClaudeBot","Claude-SearchBot","PerplexityBot","Google-Extended","Applebot-Extended","CCBot","Bytespider","Meta-ExternalAgent","Amazonbot"],"blocks_any_ai":true,"files":{"llms_txt":false,"agents_md":false,"agent_skills":false,"mcp_card":false,"api_catalog":false,"a2a_card":false},"any_capability":false,"ai_refused":["perplexity"],"ai_thin":[],"aipref":0,"aipref_measured":true,"sf":[4,0,0,0,0,0,2,0,0],"rings":{"mcp_declared":false,"mcp_live":false,"mcp_auth":false,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":3,"fh":"www.baidu.com","fp_other_host":[]},"derive_v":5,"frame":"top1k","shard":7},{"v":1,"kind":"crawlcheck-census-record","run":43,"domain":"workers.dev","rank":81,"measured_at":"2026-11-13T07:57:14.313Z","home":{"browser":{"status":200,"content_type":"text/html","bytes":757359,"sha256":"7de13a425c1115a6b1ad67062f90c5e9203754004e0e20efac75bf4812507a44","final_host":"www.cloudflare.com","final_path":"/products/workers","ms":163,"identity":"Mobile browser","html":true,"fp":"53623efc065a9de3"},"gptbot":{"status":200,"content_type":"text/html","bytes":757359,"sha256":"7de13a425c1115a6b1ad67062f90c5e9203754004e0e20efac75bf4812507a44","final_host":"www.cloudflare.com","final_path":"/products/workers","ms":165,"identity":"GPTBot","html":true,"fp":"53623efc065a9de3"},"claudebot":{"status":200,"content_type":"text/html","bytes":757359,"sha256":"7de13a425c1115a6b1ad67062f90c5e9203754004e0e20efac75bf4812507a44","final_host":"www.cloudflare.com","final_path":"/products/workers","ms":167,"identity":"ClaudeBot","html":true,"fp":"53623efc065a9de3"},"oaisearch":{"status":200,"content_type":"text/html","bytes":757359,"sha256":"7de13a425c1115a6b1ad67062f90c5e9203754004e0e20efac75bf4812507a44","final_host":"www.cloudflare.com","final_path":"/products/workers","ms":165,"identity":"OAI-SearchBot","html":true,"fp":"53623efc065a9de3"},"perplexity":{"status":200,"content_type":"text/html","bytes":757359,"sha256":"7de13a425c1115a6b1ad67062f90c5e9203754004e0e20efac75bf4812507a44","final_host":"www.cloudflare.com","final_path":"/products/workers","ms":169,"identity":"PerplexityBot","html":true,"fp":"53623efc065a9de3"}},"robots":{"status":200,"content_type":"text/html","bytes":756924,"sha256":"80c5d24bf2e4c3cbddcd91a1b15699b1ff19407af9165abd58c36660c2f02c88","final_host":"www.cloudflare.com","final_path":"/products/workers","ms":158,"html":true,"text":null,"agents":null},"sitemap":{"status":200,"content_type":"text/html","bytes":756924,"sha256":"80c5d24bf2e4c3cbddcd91a1b15699b1ff19407af9165abd58c36660c2f02c88","final_host":"www.cloudflare.com","final_path":"/products/workers","ms":133,"url":"https://workers.dev/sitemap.xml","declared":false,"xml":false},"files":{"llms_txt":{"status":200,"content_type":"text/markdown","bytes":3314,"sha256":"096e84c5fb5f1b086aca292560dab76d533bee7e5c5daa55cabe2d720f36054e","final_host":"www.cloudflare.com","final_path":"/products/workers","ms":117,"path":"/llms.txt","html":false,"json":null,"present":true},"agents_md":{"status":200,"content_type":"text/markdown","bytes":3314,"sha256":"096e84c5fb5f1b086aca292560dab76d533bee7e5c5daa55cabe2d720f36054e","final_host":"www.cloudflare.com","final_path":"/products/workers","ms":109,"path":"/agents.md","html":false,"json":null,"present":true},"agent_skills":{"status":200,"content_type":"text/html","bytes":756924,"sha256":"80c5d24bf2e4c3cbddcd91a1b15699b1ff19407af9165abd58c36660c2f02c88","final_host":"www.cloudflare.com","final_path":"/products/workers","ms":138,"path":"/.well-known/agent-skills/index.json","html":true,"json":null,"present":false},"mcp_card":{"status":200,"content_type":"text/html","bytes":756924,"sha256":"80c5d24bf2e4c3cbddcd91a1b15699b1ff19407af9165abd58c36660c2f02c88","final_host":"www.cloudflare.com","final_path":"/products/workers","ms":137,"path":"/.well-known/mcp/server-card.json","html":true,"json":null,"present":false},"api_catalog":{"status":200,"content_type":"text/html","bytes":756924,"sha256":"80c5d24bf2e4c3cbddcd91a1b15699b1ff19407af9165abd58c36660c2f02c88","final_host":"www.cloudflare.com","final_path":"/products/workers","ms":118,"path":"/.well-known/api-catalog","html":true,"json":null,"present":false},"a2a_card":{"status":200,"content_type":"text/html","bytes":756924,"sha256":"80c5d24bf2e4c3cbddcd91a1b15699b1ff19407af9165abd58c36660c2f02c88","final_host":"www.cloudflare.com","final_path":"/products/workers","ms":129,"path":"/.well-known/agent-card.json","html":true,"json":null,"present":false}},"rings":{"v":1,"candidates":[],"mcp":[],"a2a":null,"webmcp":{"model_context":false,"bridge":false,"mcp_json":false,"link_rel":false,"any":false},"handshakes":0,"get_path":{"status":200,"content_type":"text/html","ms":190,"error":null},"get_host":{"status":530,"content_type":"text/plain","ms":14,"error":null}},"sf":[4,0,0,0,0,0,1,0,1],"derived":{"reachable":true,"home_class":"answers","robots_served":false,"robots_sitemap":false,"content_signal":false,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":[],"blocks_any_ai":false,"files":{"llms_txt":true,"agents_md":true,"agent_skills":false,"mcp_card":false,"api_catalog":false,"a2a_card":false},"any_capability":false,"ai_refused":[],"ai_thin":[],"sf":[4,0,0,0,0,0,1,0,1],"rings":{"mcp_declared":false,"mcp_live":false,"mcp_auth":false,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":3,"fh":"www.cloudflare.com","fp_other_host":[],"fp":"53623efc065a9de3","fp_bits":{"gptbot":0,"claudebot":0,"oaisearch":0,"perplexity":0}},"derive_v":5,"frame":"top1k","shard":0},{"v":1,"kind":"crawlcheck-census-record","run":43,"domain":"tiktokv.com","rank":82,"measured_at":"2026-11-13T07:58:36.062Z","home":{"browser":{"status":404,"content_type":"text/html","bytes":546,"sha256":"5dd23ed6464f587908166860364362088446bf508f2793c4df0ad2119675c33e","final_host":"tiktokv.com","final_path":"/","ms":137,"identity":"Mobile browser","html":true},"gptbot":{"status":404,"content_type":"text/html","bytes":144,"sha256":"b282ac8512c5e029959282e6fd450dc119f5db4a072869eabf45e31440596248","final_host":"tiktokv.com","final_path":"/","ms":149,"identity":"GPTBot","html":true},"claudebot":{"status":404,"content_type":"text/html","bytes":144,"sha256":"b282ac8512c5e029959282e6fd450dc119f5db4a072869eabf45e31440596248","final_host":"tiktokv.com","final_path":"/","ms":152,"identity":"ClaudeBot","html":true},"oaisearch":{"status":404,"content_type":"text/html","bytes":144,"sha256":"b282ac8512c5e029959282e6fd450dc119f5db4a072869eabf45e31440596248","final_host":"tiktokv.com","final_path":"/","ms":177,"identity":"OAI-SearchBot","html":true},"perplexity":{"status":404,"content_type":"text/html","bytes":144,"sha256":"b282ac8512c5e029959282e6fd450dc119f5db4a072869eabf45e31440596248","final_host":"tiktokv.com","final_path":"/","ms":212,"identity":"PerplexityBot","html":true}},"robots":{"status":404,"content_type":"text/html","bytes":144,"sha256":"b282ac8512c5e029959282e6fd450dc119f5db4a072869eabf45e31440596248","final_host":"tiktokv.com","final_path":"/robots.txt","ms":190,"html":true,"text":null,"agents":null},"sitemap":null,"files":null,"derived":{"reachable":false,"home_class":"other","robots_served":false,"robots_sitemap":false,"content_signal":false,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":[],"blocks_any_ai":false,"files":{"llms_txt":false,"agents_md":false,"agent_skills":false,"mcp_card":false,"api_catalog":false,"a2a_card":false},"any_capability":false,"ai_refused":[],"ai_thin":[],"rings":{"mcp_declared":false,"mcp_live":false,"mcp_auth":false,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":3,"fh":"tiktokv.com","fp_other_host":[]},"derive_v":5,"frame":"top1k","shard":1},{"v":1,"kind":"crawlcheck-census-record","run":43,"domain":"chatgpt.com","rank":83,"measured_at":"2026-11-13T07:59:31.658Z","home":{"browser":{"status":200,"content_type":"text/html","bytes":460898,"sha256":"33a4a377ea749371397a40d5af8913f5c5e41a813592344c038e56130642601f","final_host":"chatgpt.com","final_path":"/","ms":402,"identity":"Mobile browser","html":true,"fp":"67454e6b59979aad"},"gptbot":{"status":200,"content_type":"text/html","bytes":451138,"sha256":"fadb15f707267666e6eef414447051f1449dfa651dcf2d45da0732d974374f02","final_host":"chatgpt.com","final_path":"/","ms":365,"identity":"GPTBot","html":true,"fp":"67454e6b59979aad"},"claudebot":{"status":200,"content_type":"text/html","bytes":470310,"sha256":"f5860c9c7763a9846dba66667a2b09a23e749d975659c199f3cb4f3b45ff3128","final_host":"chatgpt.com","final_path":"/","ms":365,"identity":"ClaudeBot","html":true,"fp":"e7456f6b599f9ead"},"oaisearch":{"status":200,"content_type":"text/html","bytes":450594,"sha256":"d010dfda9496a8a9953cb51adfb629cf149f7ebc7a7f0bcce37b7f7314f6c09b","final_host":"chatgpt.com","final_path":"/","ms":368,"identity":"OAI-SearchBot","html":true,"fp":"67454e6b59979aad"},"perplexity":{"status":200,"content_type":"text/html","bytes":457063,"sha256":"2389ac7022d6b670a117bfd02c374efd6e356fd280b26c024eb60a37ecebe63b","final_host":"chatgpt.com","final_path":"/","ms":749,"identity":"PerplexityBot","html":true,"fp":"67456e7b59979aad"}},"robots":{"status":200,"content_type":"text/plain","bytes":4573,"sha256":"7368cdfdeb325f204dc704c0696f0f9e4137d80440f596c1b1f71f872160ba04","final_host":"chatgpt.com","final_path":"/robots.txt","ms":12,"html":false,"text":"# https://www.robotstxt.org/robotstxt.html\n\nUser-agent: CCBot\nDisallow: /\n\nUser-agent: img2dataset\nDisallow: /\n\nUser-agent: Google-Extended\nDisallow: /\n\n# Prevent shared pages and GPTs from being used as Google Ads destinations\nUser-agent: AdsBot-Google\nUser-agent: AdsBot-Google-Mobile\nDisallow: /s/\nDisallow: /g/\n\nUser-agent: anthropic-ai\nDisallow: /\n\nUser-agent: Claude-Web\nDisallow: /\n\nUser-agent: Omgilibot\nDisallow: /\n\nUser-agent: Omgili\nDisallow: /\n\nUser-agent: FacebookBot\nDisallow: /\n\nUser-agent: Bytespider\nDisallow: /\n\nUser-agent: magpie-crawler\nDisallow: /\n\nUser-agent: PerplexityBot\nDisallow: /\n\nUser-agent: Perplexity‑User\nDisallow: /\n\n# General rules for all other bots\nUser-agent: *\n# Allow Googlebot to render the indexable homepage; production CSS and JavaScript live here.\nAllow: /unauth-mweb/assets/\n\n# Place allows first to avoid bots skipping after Disallow: /\n# Allow exactly the homepage\nAllow: /$\n# Allow the homepage with any query parameters\nAllow: /?*\nAllow: /api/share/og/\nAllow: /g/\nAllow: /s/\nAllow: /gg/v/\nAllow: /m/\nAllow: /share/\nAllow: /maps$\nAllow: /maps/\nAllow: /*/maps$\nAllow: /*/maps/\nAllow: /canvas/shared/\nAllow: /*/images\nAllow: /images\nAllow: /*/library\nAllow: /library\nAllow: /favicon.ico\nAllow: /assets/favicon\nAllow: /cdn/assets/favicon\nAllow: /cdn/assets/\nAllow: /auth/\nAllow: /gpts$\nAllow: /codex\nAllow: /*/codex\nAllow: /search$\nAllow: /backend-anon/\nAllow: /public-api/\nAllow: /sitemap.xml\nAllow: /marketing-sitemap.xml\nAllow: /writing-tools-sitemap.xml\nAllow: /100chats\nAllow: /api/public_content/\nAllow: /backend-api/public_content/\nAllow: /?ref=dotcom\n# Static Landing Pages\nAllow: /overview\nAllow: /*/overview\nAllow: /features\nAllow: /*/features\nAllow: /redeem$\nAllow: /redeem/\nAllow: /redeem?\nAllow: /health$\nAllow: /health/$\nAllow: /*/health$\nAllow: /*/health/$\nAllow: /voice$\nAllow: /voice/$\nAllow: /*/voice$\nAllow: /*/voice/$\nAllow: /apps\nAllow: /*/apps\nAllow: /finances\nAllow: /*/finances\nAllow: /shopping\nAllow: /*/shopping\nAllow: /use-cases\nAllow: /*/use-cases\nAllow: /learn\nAllow: /*/learn\nAllow: /business\nAllow: /*/business\nAllow: /*/business-hub\nAllow: /pricing\nAllow: /*/pricing\nAllow: /download\nAllow: /*/download\nAllow: /students\nAllow: /students/2026\nAllow: /*/college-students\nAllow: /college-students\nAllow: /contact-sales\nAllow: /*/contact-sales\nAllow: /100chats-project\nAllow: /*/100chats-project\nAllow: /merchants\nAllow: /*/merchants\nAllow: /parent-resources\nAllow: /*/parent-resources\nAllow: /plans\nAllow: /*/plans\nAllow: /plugins\nAllow: /translate\nAllow: /*/translate\nAllow: /writing\nAllow: /*/writing\nAllow: /ask/\nAllow: /*/ask\nAllow: /futures\nAllow: /*/futures\nAllow: /football\nAllow: /*/football\nAllow: /remote\nAllow: /*/remote\nAllow: /work\nAllow: /*/work\n# Allow crawlers to read the campaign page's noindex directive.\nAllow: /get-dot$\nAllow: /get-dot/$\nAllow: /get-dot?\nAllow: /get-dot/?\nAllow: /*/get-dot$\nAllow: /*/get-dot/$\nAllow: /*/get-dot?\nAllow: /*/get-dot/?\nAllow: /dots\nAllow: /*/dots\nAllow: /sites\nAllow: /*/sites\nAllow: /space\nAllow: /*/space\nAllow: /import-to-chatgpt\nAllow: /*/import-to-chatgpt\n# Exact locale specific homepages\nAllow: /am/$\nAllow: /ar/$\nAllow: /bg-BG/$\nAllow: /bn-BD/$\nAllow: /bs-BA/$\nAllow: /ca-ES/$\nAllow: /cs-CZ/$\nAllow: /da-DK/$\nAllow: /de-DE/$\nAllow: /el-GR/$\nAllow: /es-ES/$\nAllow: /es-419/$\nAllow: /et-EE/$\nAllow: /fa-IR/$\nAllow: /fi-FI/$\nAllow: /fr-FR/$\nAllow: /fr-CA/$\nAllow: /gu-IN/$\nAllow: /he-IL/$\nAllow: /hi-IN/$\nAllow: /hr-HR/$\nAllow: /hu-HU/$\nAllow: /hy-AM/$\nAllow: /id-ID/$\nAllow: /is-IS/$\nAllow: /it-IT/$\nAllow: /ja-JP/$\nAllow: /ka-GE/$\nAllow: /kk/$\nAllow: /kn-IN/$\nAllow: /ko-KR/$\nAllow: /lt/$\nAllow: /lv-LV/$\nAllow: /mk-MK/$\nAllow: /ml/$\nAllow: /mn/$\nAllow: /mr-IN/$\nAllow: /ms-MY/$\nAllow: /my-MM/$\nAllow: /nb-NO/$\nAllow: /nl-NL/$\nAllow: /pa/$\nAllow: /pl-PL/$\nAllow: /pt-BR/$\nAllow: /pt-PT/$\nAllow: /ro-RO/$\nAllow: /ru-RU/$\nAllow: /sk-SK/$\nAllow: /sl-SI/$\nAllow: /so-SO/$\nAllow: /sq-AL/$\nAllow: /sr-RS/$\nAllow: /sv-SE/$\nAllow: /sw-TZ/$\nAllow: /ta-IN/$\nAllow: /te-IN/$\nAllow: /th-TH/$\nAllow: /tl/$\nAllow: /tr-TR/$\nAllow: /uk-UA/$\nAllow: /ur/$\nAllow: /vi-VN/$\nAllow: /zh-CN/$\nAllow: /zh-TW/$\nAllow: /zh-HK/$\n\n# Now block everything else\nDisallow: /\n\n# Specific disallows (redundant for some bots, but still useful for those that respect precedence)\nDisallow: /auth/logout\nDisallow: /auth/login?*\nDisallow: /backend-anon/sentinel/*\nDisallow: /backend-anon/conversation$\nDisallow: /account-link/*\n\nSitemap: https://chatgpt.com/sitemap.xml\nSitemap: https://chatgpt.com/marketing-sitemap.xml\nSitemap: https://chatgpt.com/writing-tools-sitemap.xml\n","agents":{"GPTBot":true,"OAI-SearchBot":true,"ChatGPT-User":true,"ClaudeBot":true,"Claude-SearchBot":true,"PerplexityBot":false,"Google-Extended":false,"Applebot-Extended":true,"CCBot":false,"Bytespider":false,"Meta-ExternalAgent":true,"Amazonbot":true}},"sitemap":{"status":200,"content_type":"application/xml","bytes":4949,"sha256":"675d73e5d9b94382ea5cd98eb58d84b64560bd0b8350e49370881b172af0395e","final_host":"chatgpt.com","final_path":"/sitemap.xml","ms":18,"url":"https://chatgpt.com/sitemap.xml","declared":true,"xml":true},"files":{"llms_txt":{"status":404,"content_type":null,"bytes":0,"sha256":"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855","final_host":"chatgpt.com","final_path":"/llms.txt","ms":33,"path":"/llms.txt","html":false,"json":null,"present":false},"agents_md":{"status":404,"content_type":null,"bytes":0,"sha256":"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855","final_host":"chatgpt.com","final_path":"/agents.md","ms":52,"path":"/agents.md","html":false,"json":null,"present":false},"agent_skills":{"status":404,"content_type":null,"bytes":0,"sha256":"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855","final_host":"chatgpt.com","final_path":"/.well-known/agent-skills/index.json","ms":114,"path":"/.well-known/agent-skills/index.json","html":false,"json":null,"present":false},"mcp_card":{"status":404,"content_type":null,"bytes":0,"sha256":"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855","final_host":"chatgpt.com","final_path":"/.well-known/mcp/server-card.json","ms":31,"path":"/.well-known/mcp/server-card.json","html":false,"json":null,"present":false},"api_catalog":{"status":200,"content_type":"text/html","bytes":716699,"sha256":"536f0d5c891a2ea656fc4b1b88567a9efc10485d19fd21381b72e43de20d13fa","final_host":"chatgpt.com","final_path":"/.well-known/api-catalog","ms":160,"path":"/.well-known/api-catalog","html":true,"json":null,"present":false},"a2a_card":{"status":404,"content_type":null,"bytes":0,"sha256":"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855","final_host":"chatgpt.com","final_path":"/.well-known/agent-card.json","ms":22,"path":"/.well-known/agent-card.json","html":false,"json":null,"present":false}},"rings":{"v":1,"candidates":[],"mcp":[],"a2a":null,"webmcp":{"model_context":false,"bridge":false,"mcp_json":false,"link_rel":false,"any":false},"handshakes":0,"get_path":{"status":200,"content_type":"text/html","ms":155,"error":null},"get_host":{"status":530,"content_type":"text/plain","ms":15,"error":null}},"sf":[4,0,0,0,0,0,0,0,1],"derived":{"reachable":true,"home_class":"answers","robots_served":true,"robots_sitemap":true,"content_signal":false,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":["PerplexityBot","Google-Extended","CCBot","Bytespider"],"blocks_any_ai":true,"files":{"llms_txt":false,"agents_md":false,"agent_skills":false,"mcp_card":false,"api_catalog":false,"a2a_card":false},"any_capability":false,"ai_refused":[],"ai_thin":[],"aipref":0,"aipref_measured":true,"sf":[4,0,0,0,0,0,0,0,1],"rings":{"mcp_declared":false,"mcp_live":false,"mcp_auth":false,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":3,"fh":"chatgpt.com","fp_other_host":[],"fp":"67454e6b59979aad","fp_bits":{"gptbot":0,"claudebot":5,"oaisearch":0,"perplexity":2}},"derive_v":5,"frame":"top1k","shard":2},{"v":1,"kind":"crawlcheck-census-record","run":43,"domain":"yandex.net","rank":84,"measured_at":"2026-11-13T07:59:45.450Z","home":{"browser":{"status":200,"content_type":"text/html","bytes":375509,"sha256":"d4274450d635569254b255f9bfc810ad07e47fba97df3eff84d8bbe60b91acad","final_host":"ya.ru","final_path":"/","ms":1860,"identity":"Mobile browser","html":true},"gptbot":{"status":200,"content_type":"text/html","bytes":417940,"sha256":"d6391482ee60557252bf292aaf751994d0295e5c416b728f9e959c8bebe65bc9","final_host":"ya.ru","final_path":"/","ms":1854,"identity":"GPTBot","html":true,"fp":"9c400a0e74450022"},"claudebot":{"status":200,"content_type":"text/html","bytes":418049,"sha256":"a7d9a86fe2892f5393ca980be587503af08c38a8f064a7f7a98b39eed22d86eb","final_host":"ya.ru","final_path":"/","ms":1851,"identity":"ClaudeBot","html":true,"fp":"9c400a0e74450022"},"oaisearch":{"status":200,"content_type":"text/html","bytes":418078,"sha256":"6143898a74fbbdac6f634b936e681c7bcc3f514e67e8cc633641cdc7bd159e76","final_host":"ya.ru","final_path":"/","ms":1854,"identity":"OAI-SearchBot","html":true,"fp":"9c400a0e74450022"},"perplexity":{"status":200,"content_type":"text/html","bytes":418120,"sha256":"8373e5e1467e9d70da1e6e69e020cd8fc7badb8a1517738812b878dc316529f2","final_host":"ya.ru","final_path":"/","ms":1859,"identity":"PerplexityBot","html":true,"fp":"9c400a0e74450022"}},"robots":{"status":404,"content_type":null,"bytes":9,"sha256":"0019dfc4b32d63c1392aa264aed2253c1e0c2fb09216f8e2cc269bbfb8bb49b5","final_host":"yandex.net","final_path":"/robots.txt","ms":978,"html":false,"text":null,"agents":null},"sitemap":{"status":404,"content_type":"text/html","bytes":11808,"sha256":"a6489e4d360e44df9cc51d7cced67e97067d5cec062f9496f12dcc19df9cd6f7","final_host":"ya.ru","final_path":"/sitemap.xml","ms":472,"url":"https://yandex.net/sitemap.xml","declared":false,"xml":false},"files":{"llms_txt":{"status":404,"content_type":"text/plain","bytes":9,"sha256":"e3ebaa16dd9d9b9fc107c42183fb6cf9d22927e1af03dbbdfa0ccc38e4e4ac31","final_host":"ya.ru","final_path":"/llms.txt","ms":458,"path":"/llms.txt","html":false,"json":null,"present":false},"agents_md":{"status":404,"content_type":"text/plain","bytes":9,"sha256":"e3ebaa16dd9d9b9fc107c42183fb6cf9d22927e1af03dbbdfa0ccc38e4e4ac31","final_host":"ya.ru","final_path":"/agents.md","ms":367,"path":"/agents.md","html":false,"json":null,"present":false},"agent_skills":{"status":404,"content_type":null,"bytes":9,"sha256":"0019dfc4b32d63c1392aa264aed2253c1e0c2fb09216f8e2cc269bbfb8bb49b5","final_host":"yandex.net","final_path":"/.well-known/agent-skills/index.json","ms":159,"path":"/.well-known/agent-skills/index.json","html":false,"json":null,"present":false},"mcp_card":{"status":404,"content_type":null,"bytes":9,"sha256":"0019dfc4b32d63c1392aa264aed2253c1e0c2fb09216f8e2cc269bbfb8bb49b5","final_host":"yandex.net","final_path":"/.well-known/mcp/server-card.json","ms":173,"path":"/.well-known/mcp/server-card.json","html":false,"json":null,"present":false},"api_catalog":{"status":404,"content_type":null,"bytes":9,"sha256":"0019dfc4b32d63c1392aa264aed2253c1e0c2fb09216f8e2cc269bbfb8bb49b5","final_host":"yandex.net","final_path":"/.well-known/api-catalog","ms":158,"path":"/.well-known/api-catalog","html":false,"json":null,"present":false},"a2a_card":{"status":404,"content_type":null,"bytes":9,"sha256":"0019dfc4b32d63c1392aa264aed2253c1e0c2fb09216f8e2cc269bbfb8bb49b5","final_host":"yandex.net","final_path":"/.well-known/agent-card.json","ms":490,"path":"/.well-known/agent-card.json","html":false,"json":null,"present":false}},"device_checked":{"desktop_bits":null,"status":200},"rings":{"v":1,"candidates":[],"mcp":[],"a2a":null,"webmcp":{"model_context":false,"bridge":false,"mcp_json":false,"link_rel":false,"any":false},"handshakes":0,"get_path":{"status":404,"content_type":"application/json","ms":360,"error":null},"get_host":{"status":null,"content_type":null,"ms":8000,"error":"timeout"}},"sf":[4,0,0,0,0,0,1,0,0],"derived":{"reachable":true,"home_class":"answers","robots_served":false,"robots_sitemap":false,"content_signal":false,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":[],"blocks_any_ai":false,"files":{"llms_txt":false,"agents_md":false,"agent_skills":false,"mcp_card":false,"api_catalog":false,"a2a_card":false},"any_capability":false,"ai_refused":[],"ai_thin":[],"sf":[4,0,0,0,0,0,1,0,0],"rings":{"mcp_declared":false,"mcp_live":false,"mcp_auth":false,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":3,"fh":"ya.ru","fp_other_host":[]},"derive_v":5,"frame":"top1k","shard":3},{"v":1,"kind":"crawlcheck-census-record","run":43,"domain":"cloudflare-dns.com","rank":85,"measured_at":"2026-11-13T08:06:14.315Z","home":{"browser":{"status":200,"content_type":"text/html","bytes":229418,"sha256":"cb173d7e74602e093bb23f2f179313958328583f302d5623b9259c3ac1fb7e38","final_host":"one.one.one.one","final_path":"/dns","ms":52,"identity":"Mobile browser","html":true,"fp":"155b3d20036ed672"},"gptbot":{"status":200,"content_type":"text/html","bytes":229418,"sha256":"cb173d7e74602e093bb23f2f179313958328583f302d5623b9259c3ac1fb7e38","final_host":"one.one.one.one","final_path":"/dns","ms":52,"identity":"GPTBot","html":true,"fp":"155b3d20036ed672"},"claudebot":{"status":200,"content_type":"text/html","bytes":229418,"sha256":"cb173d7e74602e093bb23f2f179313958328583f302d5623b9259c3ac1fb7e38","final_host":"one.one.one.one","final_path":"/dns","ms":53,"identity":"ClaudeBot","html":true,"fp":"155b3d20036ed672"},"oaisearch":{"status":200,"content_type":"text/html","bytes":229418,"sha256":"cb173d7e74602e093bb23f2f179313958328583f302d5623b9259c3ac1fb7e38","final_host":"one.one.one.one","final_path":"/dns","ms":53,"identity":"OAI-SearchBot","html":true,"fp":"155b3d20036ed672"},"perplexity":{"status":200,"content_type":"text/html","bytes":229418,"sha256":"cb173d7e74602e093bb23f2f179313958328583f302d5623b9259c3ac1fb7e38","final_host":"one.one.one.one","final_path":"/dns","ms":53,"identity":"PerplexityBot","html":true,"fp":"155b3d20036ed672"}},"robots":{"status":200,"content_type":"text/plain","bytes":1248,"sha256":"8fa3036c68bfcbd32365f6225d24333264093b7cd38d306e106b4dbdc934fd5b","final_host":"one.one.one.one","final_path":"/robots.txt","ms":52,"html":false,"text":"# As a condition of accessing this website, you agree to abide by the following\n# content signals:\n\n# (a)  If a content-signal = yes, you may collect content for the corresponding\n#      use.\n# (b)  If a content-signal = no, you may not collect content for the\n#      corresponding use.\n# (c)  If the website operator does not include a content signal for a\n#      corresponding use, the website operator neither grants nor restricts\n#      permission via content signal with respect to the corresponding use.\n\n# The content signals and their meanings are:\n\n# search:   building a search index and providing search results (e.g., returning\n#           hyperlinks and short excerpts from your website's contents). Search does not\n#           include providing AI-generated search summaries.\n# ai-input: inputting content into one or more AI models (e.g., retrieval\n#           augmented generation, grounding, or other real-time taking of content for\n#           generative AI search answers).\n# ai-train: training or fine-tuning AI models.\n\n# ANY RESTRICTIONS EXPRESSED VIA CONTENT SIGNALS ARE EXPRESS RESERVATIONS OF\n# RIGHTS UNDER ARTICLE 4 OF THE EUROPEAN UNION DIRECTIVE 2019/790 ON COPYRIGHT\n# AND RELATED RIGHTS IN THE DIGITAL SINGLE MARKET.\n","agents":{"GPTBot":true,"OAI-SearchBot":true,"ChatGPT-User":true,"ClaudeBot":true,"Claude-SearchBot":true,"PerplexityBot":true,"Google-Extended":true,"Applebot-Extended":true,"CCBot":true,"Bytespider":true,"Meta-ExternalAgent":true,"Amazonbot":true}},"sitemap":{"status":404,"content_type":"text/html","bytes":1285,"sha256":"8017fa1f7764a80571cc0a99c7dbdfaa33dad3ec31f19239d56d38b0a6f421fe","final_host":"one.one.one.one","final_path":"/sitemap.xml","ms":52,"url":"https://cloudflare-dns.com/sitemap.xml","declared":false,"xml":false},"files":{"llms_txt":{"status":404,"content_type":"text/html","bytes":1285,"sha256":"8017fa1f7764a80571cc0a99c7dbdfaa33dad3ec31f19239d56d38b0a6f421fe","final_host":"one.one.one.one","final_path":"/llms.txt","ms":43,"path":"/llms.txt","html":true,"json":null,"present":false},"agents_md":{"status":404,"content_type":"text/html","bytes":1285,"sha256":"8017fa1f7764a80571cc0a99c7dbdfaa33dad3ec31f19239d56d38b0a6f421fe","final_host":"one.one.one.one","final_path":"/agents.md","ms":36,"path":"/agents.md","html":true,"json":null,"present":false},"agent_skills":{"status":404,"content_type":"text/html","bytes":1285,"sha256":"8017fa1f7764a80571cc0a99c7dbdfaa33dad3ec31f19239d56d38b0a6f421fe","final_host":"one.one.one.one","final_path":"/.well-known/agent-skills/index.json","ms":39,"path":"/.well-known/agent-skills/index.json","html":true,"json":null,"present":false},"mcp_card":{"status":404,"content_type":"text/html","bytes":1285,"sha256":"8017fa1f7764a80571cc0a99c7dbdfaa33dad3ec31f19239d56d38b0a6f421fe","final_host":"one.one.one.one","final_path":"/.well-known/mcp/server-card.json","ms":58,"path":"/.well-known/mcp/server-card.json","html":true,"json":null,"present":false},"api_catalog":{"status":404,"content_type":"text/html","bytes":1285,"sha256":"8017fa1f7764a80571cc0a99c7dbdfaa33dad3ec31f19239d56d38b0a6f421fe","final_host":"one.one.one.one","final_path":"/.well-known/api-catalog","ms":54,"path":"/.well-known/api-catalog","html":true,"json":null,"present":false},"a2a_card":{"status":404,"content_type":"text/html","bytes":1285,"sha256":"8017fa1f7764a80571cc0a99c7dbdfaa33dad3ec31f19239d56d38b0a6f421fe","final_host":"one.one.one.one","final_path":"/.well-known/agent-card.json","ms":47,"path":"/.well-known/agent-card.json","html":true,"json":null,"present":false}},"rings":{"v":1,"candidates":[],"mcp":[],"a2a":null,"webmcp":{"model_context":false,"bridge":false,"mcp_json":false,"link_rel":false,"any":false},"handshakes":0,"get_path":{"status":404,"content_type":"text/html","ms":45,"error":null},"get_host":{"status":404,"content_type":"text/html","ms":41,"error":null}},"sf":[1,0,0,0,0,0,2,0,1],"derived":{"reachable":true,"home_class":"answers","robots_served":true,"robots_sitemap":false,"content_signal":false,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":[],"blocks_any_ai":false,"files":{"llms_txt":false,"agents_md":false,"agent_skills":false,"mcp_card":false,"api_catalog":false,"a2a_card":false},"any_capability":false,"ai_refused":[],"ai_thin":[],"aipref":0,"aipref_measured":true,"sf":[1,0,0,0,0,0,2,0,1],"rings":{"mcp_declared":false,"mcp_live":false,"mcp_auth":false,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":3,"fh":"one.one.one.one","fp_other_host":[],"fp":"155b3d20036ed672","fp_bits":{"gptbot":0,"claudebot":0,"oaisearch":0,"perplexity":0}},"derive_v":5,"frame":"top1k","shard":4},{"v":1,"kind":"crawlcheck-census-record","run":43,"domain":"opera.com","rank":86,"measured_at":"2026-11-13T08:07:18.369Z","home":{"browser":{"status":200,"content_type":"text/html","bytes":196129,"sha256":"5502c88b6aa45b1e67774201483532ee051c0e1628af09a41cfdffcb9ca9b22d","final_host":"www.opera.com","final_path":"/","ms":820,"identity":"Mobile browser","html":true,"fp":"aa2201a8171172bc"},"gptbot":{"status":200,"content_type":"text/html","bytes":192951,"sha256":"effed25c932ad6b33f7cd6237507db3f7dca88bd42ca5076a00aafbcee878d2e","final_host":"www.opera.com","final_path":"/","ms":817,"identity":"GPTBot","html":true,"fp":"aa3201a8153172bc"},"claudebot":{"status":200,"content_type":"text/html","bytes":192951,"sha256":"effed25c932ad6b33f7cd6237507db3f7dca88bd42ca5076a00aafbcee878d2e","final_host":"www.opera.com","final_path":"/","ms":816,"identity":"ClaudeBot","html":true,"fp":"aa3201a8153172bc"},"oaisearch":{"status":200,"content_type":"text/html","bytes":192951,"sha256":"effed25c932ad6b33f7cd6237507db3f7dca88bd42ca5076a00aafbcee878d2e","final_host":"www.opera.com","final_path":"/","ms":817,"identity":"OAI-SearchBot","html":true,"fp":"aa3201a8153172bc"},"perplexity":{"status":200,"content_type":"text/html","bytes":192927,"sha256":"a1937c1a04cebae8f44dde0686d7a3470bb7a31f3a04c6fa44cd211bc20f49e4","final_host":"www.opera.com","final_path":"/","ms":819,"identity":"PerplexityBot","html":true,"fp":"aa3201a8153172bc"}},"robots":{"status":200,"content_type":"text/plain","bytes":994,"sha256":"0c6b878d90d9783f0a0972e38888f8dfef8393e317be80a266a82f3da548f20f","final_host":"www.opera.com","final_path":"/robots.txt","ms":841,"html":false,"text":"User-agent: *\nAllow: /help/tutorials/\nAllow: /browser/tutorials/\nAllow: /support/tutorials/\nAllow: /lp/br-themes\nAllow: /pt-br/lp/br-themes\nAllow: /lp/video-streaming\nAllow: /*/lp/video-streaming\nDisallow: /o/\nDisallow: /*/o/\nDisallow: /abtest/\nDisallow: /*/abtest/\nDisallow: /api/\nDisallow: /client/\nDisallow: /downloadassets/\nDisallow: /*/downloadassets/\nDisallow: /download/get*\nDisallow: /get$\nDisallow: /get/\nDisallow: /promoassets/\nDisallow: /*/promoassets/\nDisallow: /campaign/\nDisallow: /*/campaign/\nDisallow: /suite/\nDisallow: /*/suite/\nDisallow: /info/\nDisallow: /*/info/\nDisallow: /computer/thanks\nDisallow: /*/computer/thanks\nDisallow: /computer/thanks-partner\nDisallow: /*/computer/thanks-partner\nDisallow: /ofa/\nDisallow: /*/ofa/\nDisallow: /oma/\nDisallow: /*/oma/\nDisallow: */tlp/*\nDisallow: */lp/*\nDisallow: /tutorials\nDisallow: /*/tutorials\nDisallow: /opera/features\nDisallow: /*/opera/features\nDisallow: /opera-coupons-uninstall\nSitemap: https://www.opera.com/sitemap-index.xml","agents":{"GPTBot":true,"OAI-SearchBot":true,"ChatGPT-User":true,"ClaudeBot":true,"Claude-SearchBot":true,"PerplexityBot":true,"Google-Extended":true,"Applebot-Extended":true,"CCBot":true,"Bytespider":true,"Meta-ExternalAgent":true,"Amazonbot":true}},"sitemap":{"status":200,"content_type":"application/xml","bytes":314,"sha256":"8eb2cdb83e21bab426f66f7b493ba60caa5b818a2c1a4f6e8b6fbbc1992dc718","final_host":"www.opera.com","final_path":"/sitemap-index.xml","ms":20,"url":"https://www.opera.com/sitemap-index.xml","declared":true,"xml":true},"files":{"llms_txt":{"status":200,"content_type":"text/plain","bytes":11613,"sha256":"abefbde9fb5c1ef40e063c0d21aeff38c71e1f9899b3b7cc8d924a89deffb013","final_host":"www.opera.com","final_path":"/llms.txt","ms":102,"path":"/llms.txt","html":false,"json":null,"present":true},"agents_md":{"status":404,"content_type":"text/html","bytes":117105,"sha256":"66102af4819550cbccec289a340798573adc1f1cc6e6e317065c2dc3254506ea","final_host":"www.opera.com","final_path":"/agents.md","ms":129,"path":"/agents.md","html":true,"json":null,"present":false},"agent_skills":{"status":404,"content_type":"text/html","bytes":117157,"sha256":"afd4c0ec0bb742249ef15a9550a0c7d42d309b9a3d168176f0247af7d7433b77","final_host":"www.opera.com","final_path":"/.well-known/agent-skills/index.json","ms":109,"path":"/.well-known/agent-skills/index.json","html":true,"json":null,"present":false},"mcp_card":{"status":404,"content_type":"text/html","bytes":117201,"sha256":"f6d2ece360869743cd0e933769f7f076382f62b358d162dd87a8d0032a1ff69f","final_host":"www.opera.com","final_path":"/.well-known/mcp/server-card.json","ms":99,"path":"/.well-known/mcp/server-card.json","html":true,"json":null,"present":false},"api_catalog":{"status":404,"content_type":"text/html","bytes":117133,"sha256":"f129b8d6edb44a38ec544ad00886573524b7858183db77ec8349322bd6ba230c","final_host":"www.opera.com","final_path":"/.well-known/api-catalog","ms":101,"path":"/.well-known/api-catalog","html":true,"json":null,"present":false},"a2a_card":{"status":404,"content_type":"text/html","bytes":117141,"sha256":"0d340bc48bb92ec3e3849127ff350c8d01956249c1c1537415935de1094cff2b","final_host":"www.opera.com","final_path":"/.well-known/agent-card.json","ms":101,"path":"/.well-known/agent-card.json","html":true,"json":null,"present":false}},"rings":{"v":1,"candidates":[],"mcp":[],"a2a":null,"webmcp":{"model_context":false,"bridge":false,"mcp_json":false,"link_rel":false,"any":false},"handshakes":0,"get_path":{"status":404,"content_type":"text/html","ms":124,"error":null},"get_host":{"status":530,"content_type":"text/plain","ms":90,"error":null}},"sf":[1,0,0,0,0,0,1,0,0],"derived":{"reachable":true,"home_class":"answers","robots_served":true,"robots_sitemap":true,"content_signal":false,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":[],"blocks_any_ai":false,"files":{"llms_txt":true,"agents_md":false,"agent_skills":false,"mcp_card":false,"api_catalog":false,"a2a_card":false},"any_capability":false,"ai_refused":[],"ai_thin":[],"aipref":0,"aipref_measured":true,"sf":[1,0,0,0,0,0,1,0,0],"rings":{"mcp_declared":false,"mcp_live":false,"mcp_auth":false,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":3,"fh":"www.opera.com","fp_other_host":[],"fp":"aa2201a8171172bc","fp_bits":{"gptbot":3,"claudebot":3,"oaisearch":3,"perplexity":3}},"derive_v":5,"frame":"top1k","shard":5},{"v":1,"kind":"crawlcheck-census-record","run":43,"domain":"vimeo.com","rank":87,"measured_at":"2026-11-13T08:07:47.278Z","home":{"browser":{"status":200,"content_type":"text/html","bytes":1758980,"sha256":"2243cc50d2adcf427150534709082613f9d0328e2ff7417d14f7ad2fb02c092d","final_host":"vimeo.com","final_path":"/","ms":101,"identity":"Mobile browser","html":true,"fp":"e699fe5c0d05a34a"},"gptbot":{"status":200,"content_type":"text/html","bytes":1758980,"sha256":"2243cc50d2adcf427150534709082613f9d0328e2ff7417d14f7ad2fb02c092d","final_host":"vimeo.com","final_path":"/","ms":54,"identity":"GPTBot","html":true,"fp":"e699fe5c0d05a34a"},"claudebot":{"status":200,"content_type":"text/html","bytes":1758980,"sha256":"2243cc50d2adcf427150534709082613f9d0328e2ff7417d14f7ad2fb02c092d","final_host":"vimeo.com","final_path":"/","ms":77,"identity":"ClaudeBot","html":true,"fp":"e699fe5c0d05a34a"},"oaisearch":{"status":200,"content_type":"text/html","bytes":1758980,"sha256":"2243cc50d2adcf427150534709082613f9d0328e2ff7417d14f7ad2fb02c092d","final_host":"vimeo.com","final_path":"/","ms":54,"identity":"OAI-SearchBot","html":true,"fp":"e699fe5c0d05a34a"},"perplexity":{"status":200,"content_type":"text/html","bytes":1758980,"sha256":"2243cc50d2adcf427150534709082613f9d0328e2ff7417d14f7ad2fb02c092d","final_host":"vimeo.com","final_path":"/","ms":88,"identity":"PerplexityBot","html":true,"fp":"e699fe5c0d05a34a"}},"robots":{"status":200,"content_type":"text/plain","bytes":30242,"sha256":"b076c41a641a33acc075b9275c9edb0cb100308adcf93a9ee4ca326a005b81a9","final_host":"vimeo.com","final_path":"/robots.txt","ms":41,"html":false,"text":"# Notice: The use of robots, scrapers, or other automated means to access Vimeo \n# without the express written permission of Vimeo is strictly prohibited.\n# See https://vimeo.com/terms#acceptable_use_policy\n# Vimeo permits limited automated access to certain public-facing pages by approved services,\n# including search engines and select AI assistants, in accordance with this robots.txt file.\n# Access to user-generated content, including videos, user profiles, comments, and metadata, \n# is not permitted for scraping, harvesting, or use in training machine learning models.\n# For more on Vimeo's position on AI and content usage, see https://vimeo.com/blog/post/vimeos-position-on-ai\n\nUser-agent: *\nDisallow: /*/format:thumbnail\nDisallow: /download/\nDisallow: /*/download?\nDisallow: /couchmode/\nDisallow: /groups/*/forum/*\nDisallow: /musicstore/preview\nDisallow: /musicstore/download\nDisallow: /forgot_password\nDisallow: /help/forgot_login\nDisallow: /help/verify_email\nDisallow: /help/zendesk_sso\nDisallow: /help/sso\nDisallow: /user/verify\nDisallow: /search/\nDisallow: /search?\nDisallow: /search\\?\nDisallow: /_\nDisallow: /*mkc=*&\nDisallow: /*&mkc=*\nDisallow: /*gl=*&\nDisallow: /*&gl=*\nDisallow: /_global_notifications\nDisallow: /*?*referrer_vuid=\nDisallow: /*?*vimeo_search\nDisallow: /store/\nDisallow: /*?*_gl=\nDisallow: /*/collections/channels\nDisallow: /manage/videos/\nDisallow: /flarepoint/\nDisallow: /*/likes \nDisallow: /reviews/\nDisallow: /*/sort:*/*:\nDisallow: /ablincoln/\nDisallow: /*tag:\nDisallow: /*/videos/\nDisallow: /legal/~gitbook/\nDisallow: /manage/showcase\nAllow: /api/*/likes \nDisallow: /api/v3/me$\nDisallow: /api/v3/me/\nDisallow: /api/v3/me?\nDisallow: /api/v3/modworld\nDisallow: /api/v3/*/comments\nAllow: /_rv/viewer\nAllow: /_next/viewer\nAllow: /_rv/gtm\nAllow: /_rv/jwt\nAllow: /_next/jwt\nAllow: /_rv/title\nAllow: /_next/title\n# Clip Page Parameters\nDisallow: /*?_=\nDisallow: /*?_hsenc=\nDisallow: /*?_hsmi=\nDisallow: /*?*&_=\nDisallow: /*?*&_hsenc=\nDisallow: /*?*&_hsmi=\nDisallow: /*?*&1=\nDisallow: /*?*&activityReferer=\nDisallow: /*?*&adcPathname=\nDisallow: /*?*&ajax=\nDisallow: /*?*&allowfullscreen=\nDisallow: /*?*&autohide=\nDisallow: /*?*&autopause=\nDisallow: /*?*&background=\nDisallow: /*?*&badge=\nDisallow: /*?*&brid=\nDisallow: /*?*&byline=\nDisallow: /*?*&callback=\nDisallow: /*?*&cb=\nDisallow: /*?*&cc_load_policy=\nDisallow: /*?*&ceid=\nDisallow: /*?*&cjdata=\nDisallow: /*?*&cjevent=\nDisallow: /*?*&clickid=\nDisallow: /*?*&clipId=\nDisallow: /*?*&color=\nDisallow: /*?*&consentEnabled=\nDisallow: /*?*&content=\nDisallow: /*?*&context=\nDisallow: /*?*&controls=\nDisallow: /*?*&d=\nDisallow: /*?*&data=\nDisallow: /*?*&dnt=\nDisallow: /*?*&ei=\nDisallow: /*?*&ej=\nDisallow: /*?*&embed_email_provider=\nDisallow: /*?*&emci=\nDisallow: /*?*&emdi=\nDisallow: /*?*&enablejsapi=\nDisallow: /*?*&f1=\nDisallow: /*?*&filter_location=\nDisallow: /*?*&filter_skills=\nDisallow: /*?*&fit=\nDisallow: /*?*&flashvars=\nDisallow: /*?*&foe=\nDisallow: /*?*&format=\nDisallow: /*?*&from=\nDisallow: /*?*&fromApp=\nDisallow: /*?*&fullscreen=\nDisallow: /*?*&h=\nDisallow: /*?*&hashKey=\nDisallow: /*?*&hd=\nDisallow: /*?*&height=\nDisallow: /*?*&hss_meta=\nDisallow: /*?*&id=\nDisallow: /*?*&iframe=\nDisallow: /*?*&itok=\nDisallow: /*?*&keepEmbedCode=\nDisallow: /*?*&keyboard=\nDisallow: /*?1=\nDisallow: /*?activityReferer=\nDisallow: /*?adcPathname=\nDisallow: /*?ajax=\nDisallow: /*?allowfullscreen=\nDisallow: /*?autohide=\nDisallow: /*?autopause=\nDisallow: /*?background=\nDisallow: /*?badge=\nDisallow: /*?brid=\nDisallow: /*?byline=\nDisallow: /*?callback=\nDisallow: /*?cb=\nDisallow: /*?cc_load_policy=\nDisallow: /*?ceid=\nDisallow: /*?cjdata=\nDisallow: /*?cjevent=\nDisallow: /*?clickid=\nDisallow: /*?clipId=\nDisallow: /*?color=\nDisallow: /*?consentEnabled=\nDisallow: /*?content=\nDisallow: /*?context=\nDisallow: /*?controls=\nDisallow: /*?d=\nDisallow: /*?data=\nDisallow: /*?dnt=\nDisallow: /*?ei=\nDisallow: /*?ej=\nDisallow: /*?embed_email_provider=\nDisallow: /*?emci=\nDisallow: /*?emdi=\nDisallow: /*?enablejsapi=\nDisallow: /*?f1=\nDisallow: /*?filter_location=\nDisallow: /*?filter_skills=\nDisallow: /*?fit=\nDisallow: /*?flashvars=\nDisallow: /*?foe=\nDisallow: /*?format=\nDisallow: /*?from=\nDisallow: /*?fromApp=\nDisallow: /*?fullscreen=\nDisallow: /*?h=\nDisallow: /*?hashKey=\nDisallow: /*?hd=\nDisallow: /*?height=\nDisallow: /*?hss_meta=\nDisallow: /*?id=\nDisallow: /*?iframe=\nDisallow: /*?itok=\nDisallow: /*?keepEmbedCode=\nDisallow: /*?keyboard=\nDisallow: /*?*&%3Bfe=\nDisallow: /*?*&%3Bfl=\nDisallow: /*?*&kref=\nDisallow: /*?*&kuid=\nDisallow: /*?*&lang=\nDisallow: /*?*&lightbox[modal]=\nDisallow: /*?*&linkId=\nDisallow: /*?*&lite=\nDisallow: /*?*&login=\nDisallow: /*?*&loop=\nDisallow: /*?*&maxheight=\nDisallow: /*?*&maxw=\nDisallow: /*?*&maxwidth=\nDisallow: /*?*&mc_cid=\nDisallow: /*?*&method=\nDisallow: /*?*&mibextid=\nDisallow: /*?*&modestbranding=\nDisallow: /*?*&ms=\nDisallow: /*?*&msockid=\nDisallow: /*?*&mute=\nDisallow: /*?*&odnBg=\nDisallow: /*?*&odnHeight=\nDisallow: /*?*&odnWidth=\nDisallow: /*?*&output=\nDisallow: /*?*&outro=\nDisallow: /*?*&per_page=\nDisallow: /*?*&pgroup=\nDisallow: /*?*&platform=\nDisallow: /*?*&play=\nDisallow: /*?*&player_id=\nDisallow: /*?*&playerapiid=\nDisallow: /*?*&playlist=\nDisallow: /*?*&playsinline=\nDisallow: /*?*&portrait=\nDisallow: /*?*&progress_bar=\nDisallow: /*?*&quality_selector=\nDisallow: /*?*&quality=\nDisallow: /*?*&ra=\nDisallow: /*?*&random=\nDisallow: /*?*&redirect=\nDisallow: /*?*&rel=\nDisallow: /*?*&resize=\nDisallow: /*?*&responsive=\nDisallow: /*?*&rmxv=\nDisallow: /*?*&rt=\nDisallow: /*?*&rv=\nDisallow: /*?*&s=\nDisallow: /*?*&sa=\nDisallow: /*?*&scrlybrkr=\nDisallow: /*?*&sd=\nDisallow: /*?*&server=\nDisallow: /*?*&sfmc_id=\nDisallow: /*?*&show_byline=\nDisallow: /*?*&show_portrait=\nDisallow: /*?*&show_title=\nDisallow: /*?*&showinfo=\nDisallow: /*?*&sig=\nDisallow: /*?*&signup=\nDisallow: /*?*&sizes=\nDisallow: /*?*&sl_key=\nDisallow: /*?*&sort=\nDisallow: /*?*&speed=\nDisallow: /*?*&sr=\nDisallow: /*?*&ssl=\nDisallow: /*?*&ssp_iabi=\nDisallow: /*?*&start=\nDisallow: /*?*&streamid=\nDisallow: /*?*&sv=\nDisallow: /*?*&syclid=\nDisallow: /*?*&tblci=\nDisallow: /*?*&test=\nDisallow: /*?*&texttrack=\nDisallow: /*?*&title=\nDisallow: /*?*&tq=\nDisallow: /*?*&transparent=\nDisallow: /*?*&trk=\nDisallow: /*?*&type=\nDisallow: /*?*&url=\nDisallow: /*?*&user_id=\nDisallow: /*?*&usg=\nDisallow: /*?*&utm_date=\nDisallow: /*?*&utm_id=\nDisallow: /*?*&v=\nDisallow: /*?*&version=\nDisallow: /*?*&volumeOff=\nDisallow: /*?*&vq=\nDisallow: /*?*&w=\nDisallow: /*?*&ysclid=\nDisallow: /*?%3Bfe=\nDisallow: /*?%3Bfl=\nDisallow: /*?kref=\nDisallow: /*?kuid=\nDisallow: /*?lang=\nDisallow: /*?lightbox[modal]=\nDisallow: /*?linkId=\nDisallow: /*?lite=\nDisallow: /*?login=\nDisallow: /*?loop=\nDisallow: /*?maxheight=\nDisallow: /*?maxw=\nDisallow: /*?maxwidth=\nDisallow: /*?mc_cid=\nDisallow: /*?method=\nDisallow: /*?mibextid=\nDisallow: /*?modestbranding=\nDisallow: /*?ms=\nDisallow: /*?msockid=\nDisallow: /*?mute=\nDisallow: /*?odnBg=\nDisallow: /*?odnHeight=\nDisallow: /*?odnWidth=\nDisallow: /*?output=\nDisallow: /*?outro=\nDisallow: /*?per_page=\nDisallow: /*?pgroup=\nDisallow: /*?platform=\nDisallow: /*?play=\nDisallow: /*?player_id=\nDisallow: /*?playerapiid=\nDisallow: /*?playlist=\nDisallow: /*?playsinline=\nDisallow: /*?portrait=\nDisallow: /*?progress_bar=\nDisallow: /*?quality_selector=\nDisallow: /*?quality=\nDisallow: /*?ra=\nDisallow: /*?random=\nDisallow: /*?redirect=\nDisallow: /*?rel=\nDisallow: /*?resize=\nDisallow: /*?responsive=\nDisallow: /*?rmxv=\nDisallow: /*?rt=\nDisallow: /*?rv=\nDisallow: /*?s=\nDisallow: /*?sa=\nDisallow: /*?scrlybrkr=\nDisallow: /*?sd=\nDisallow: /*?server=\nDisallow: /*?sfmc_id=\nDisallow: /*?show_byline=\nDisallow: /*?show_portrait=\nDisallow: /*?show_title=\nDisallow: /*?showinfo=\nDisallow: /*?sig=\nDisallow: /*?signup=\nDisallow: /*?sizes=\nDisallow: /*?sl_key=\nDisallow: /*?sort=\nDisallow: /*?speed=\nDisallow: /*?sr=\nDisallow: /*?ssl=\nDisallow: /*?ssp_iabi=\nDisallow: /*?start=\nDisallow: /*?streamid=\nDisallow: /*?sv=\nDisallow: /*?syclid=\nDisallow: /*?tblci=\nDisallow: /*?test=\nDisallow: /*?texttrack=\nDisallow: /*?title=\nDisallow: /*?tq=\nDisallow: /*?transparent=\nDisallow: /*?trk=\nDisallow: /*?type=\nDisallow: /*?url=\nDisallow: /*?user_id=\nDisallow: /*?usg=\nDisallow: /*?utm_date=\nDisallow: /*?utm_id=\nDisallow: /*?v=\nDisallow: /*?version=\nDisallow: /*?volumeOff=\nDisallow: /*?vq=\nDisallow: /*?w=\nDisallow: /*?ysclid=\nAllow: /*/*?_=\nAllow: /*/*?_hsenc=\nAllow: /*/*?_hsmi=\nAllow: /*/*?*&_=\nAllow: /*/*?*&_hsenc=\nAllow: /*/*?*&_hsmi=\nAllow: /*/*?*&1=\nAllow: /*/*?*&activityReferer=\nAllow: /*/*?*&adcPathname=\nAllow: /*/*?*&ajax=\nAllow: /*/*?*&allowfullscreen=\nAllow: /*/*?*&autohide=\nAllow: /*/*?*&autopause=\nAllow: /*/*?*&background=\nAllow: /*/*?*&badge=\nAllow: /*/*?*&brid=\nAllow: /*/*?*&byline=\nAllow: /*/*?*&callback=\nAllow: /*/*?*&cb=\nAllow: /*/*?*&cc_load_policy=\nAllow: /*/*?*&ceid=\nAllow: /*/*?*&cjdata=\nAllow: /*/*?*&cjevent=\nAllow: /*/*?*&clickid=\nAllow: /*/*?*&clipId=\nAllow: /*/*?*&color=\nAllow: /*/*?*&consentEnabled=\nAllow: /*/*?*&content=\nAllow: /*/*?*&context=\nAllow: /*/*?*&controls=\nAllow: /*/*?*&d=\nAllow: /*/*?*&data=\nAllow: /*/*?*&dnt=\nAllow: /*/*?*&ei=\nAllow: /*/*?*&ej=\nAllow: /*/*?*&embed_email_provider=\nAllow: /*/*?*&emci=\nAllow: /*/*?*&emdi=\nAllow: /*/*?*&enablejsapi=\nAllow: /*/*?*&f1=\nAllow: /*/*?*&filter_location=\nAllow: /*/*?*&filter_skills=\nAllow: /*/*?*&fit=\nAllow: /*/*?*&flashvars=\nAllow: /*/*?*&foe=\nAllow: /*/*?*&format=\nAllow: /*/*?*&from=\nAllow: /*/*?*&fromApp=\nAllow: /*/*?*&fullscreen=\nAllow: /*/*?*&h=\nAllow: /*/*?*&hashKey=\nAllow: /*/*?*&hd=\nAllow: /*/*?*&height=\nAllow: /*/*?*&hss_meta=\nAllow: /*/*?*&id=\nAllow: /*/*?*&iframe=\nAllow: /*/*?*&itok=\nAllow: /*/*?*&keepEmbedCode=\nAllow: /*/*?*&keyboard=\nAllow: /*/*?1=\nAllow: /*/*?activityReferer=\nAllow: /*/*?adcPathname=\nAllow: /*/*?ajax=\nAllow: /*/*?allowfullscreen=\nAllow: /*/*?autohide=\nAllow: /*/*?autopause=\nAllow: /*/*?background=\nAllow: /*/*?badge=\nAllow: /*/*?brid=\nAllow: /*/*?byline=\nAllow: /*/*?callback=\nAllow: /*/*?cb=\nAllow: /*/*?cc_load_policy=\nAllow: /*/*?ceid=\nAllow: /*/*?cjdata=\nAllow: /*/*?cjevent=\nAllow: /*/*?clickid=\nAllow: /*/*?clipId=\nAllow: /*/*?color=\nAllow: /*/*?consentEnabled=\nAllow: /*/*?content=\nAllow: /*/*?context=\nAllow: /*/*?controls=\nAllow: /*/*?d=\nAllow: /*/*?data=\nAllow: /*/*?dnt=\nAllow: /*/*?ei=\nAllow: /*/*?ej=\nAllow: /*/*?embed_email_provider=\nAllow: /*/*?emci=\nAllow: /*/*?emdi=\nAllow: /*/*?enablejsapi=\nAllow: /*/*?f1=\nAllow: /*/*?filter_location=\nAllow: /*/*?filter_skills=\nAllow: /*/*?fit=\nAllow: /*/*?flashvars=\nAllow: /*/*?foe=\nAllow: /*/*?format=\nAllow: /*/*?from=\nAllow: /*/*?fromApp=\nAllow: /*/*?fullscreen=\nAllow: /*/*?h=\nAllow: /*/*?hashKey=\nAllow: /*/*?hd=\nAllow: /*/*?height=\nAllow: /*/*?hss_meta=\nAllow: /*/*?id=\nAllow: /*/*?iframe=\nAllow: /*/*?itok=\nAllow: /*/*?keepEmbedCode=\nAllow: /*/*?keyboard=\nAllow: /*/*?*&%3Bfe=\nAllow: /*/*?*&%3Bfl=\nAllow: /*/*?*&kref=\nAllow: /*/*?*&kuid=\nAllow: /*/*?*&lang=\nAllow: /*/*?*&lightbox[modal]=\nAllow: /*/*?*&linkId=\nAllow: /*/*?*&lite=\nAllow: /*/*?*&login=\nAllow: /*/*?*&loop=\nAllow: /*/*?*&maxheight=\nAllow: /*/*?*&maxw=\nAllow: /*/*?*&maxwidth=\nAllow: /*/*?*&mc_cid=\nAllow: /*/*?*&method=\nAllow: /*/*?*&mibextid=\nAllow: /*/*?*&modestbranding=\nAllow: /*/*?*&ms=\nAllow: /*/*?*&msockid=\nAllow: /*/*?*&mute=\nAllow: /*/*?*&odnBg=\nAllow: /*/*?*&odnHeight=\nAllow: /*/*?*&odnWidth=\nAllow: /*/*?*&output=\nAllow: /*/*?*&outro=\nAllow: /*/*?*&per_page=\nAllow: /*/*?*&pgroup=\nAllow: /*/*?*&platform=\nAllow: /*/*?*&play=\nAllow: /*/*?*&player_id=\nAllow: /*/*?*&playerapiid=\nAllow: /*/*?*&playlist=\nAllow: /*/*?*&playsinline=\nAllow: /*/*?*&portrait=\nAllow: /*/*?*&progress_bar=\nAllow: /*/*?*&quality_selector=\nAllow: /*/*?*&quality=\nAllow: /*/*?*&ra=\nAllow: /*/*?*&random=\nAllow: /*/*?*&redirect=\nAllow: /*/*?*&rel=\nAllow: /*/*?*&resize=\nAllow: /*/*?*&responsive=\nAllow: /*/*?*&rmxv=\nAllow: /*/*?*&rt=\nAllow: /*/*?*&rv=\nAllow: /*/*?*&s=\nAllow: /*/*?*&sa=\nAllow: /*/*?*&scrlybrkr=\nAllow: /*/*?*&sd=\nAllow: /*/*?*&server=\nAllow: /*/*?*&sfmc_id=\nAllow: /*/*?*&show_byline=\nAllow: /*/*?*&show_portrait=\nAllow: /*/*?*&show_title=\nAllow: /*/*?*&showinfo=\nAllow: /*/*?*&sig=\nAllow: /*/*?*&signup=\nAllow: /*/*?*&sizes=\nAllow: /*/*?*&sl_key=\nAllow: /*/*?*&sort=\nAllow: /*/*?*&speed=\nAllow: /*/*?*&sr=\nAllow: /*/*?*&ssl=\nAllow: /*/*?*&ssp_iabi=\nAllow: /*/*?*&start=\nAllow: /*/*?*&streamid=\nAllow: /*/*?*&sv=\nAllow: /*/*?*&syclid=\nAllow: /*/*?*&tblci=\nAllow: /*/*?*&test=\nAllow: /*/*?*&texttrack=\nAllow: /*/*?*&title=\nAllow: /*/*?*&tq=\nAllow: /*/*?*&transparent=\nAllow: /*/*?*&trk=\nAllow: /*/*?*&type=\nAllow: /*/*?*&url=\nAllow: /*/*?*&user_id=\nAllow: /*/*?*&usg=\nAllow: /*/*?*&utm_date=\nAllow: /*/*?*&utm_id=\nAllow: /*/*?*&v=\nAllow: /*/*?*&version=\nAllow: /*/*?*&volumeOff=\nAllow: /*/*?*&vq=\nAllow: /*/*?*&w=\nAllow: /*/*?*&ysclid=\nAllow: /*/*?%3Bfe=\nAllow: /*/*?%3Bfl=\nAllow: /*/*?kref=\nAllow: /*/*?kuid=\nAllow: /*/*?lang=\nAllow: /*/*?lightbox[modal]=\nAllow: /*/*?linkId=\nAllow: /*/*?lite=\nAllow: /*/*?login=\nAllow: /*/*?loop=\nAllow: /*/*?maxheight=\nAllow: /*/*?maxw=\nAllow: /*/*?maxwidth=\nAllow: /*/*?mc_cid=\nAllow: /*/*?method=\nAllow: /*/*?mibextid=\nAllow: /*/*?modestbranding=\nAllow: /*/*?ms=\nAllow: /*/*?msockid=\nAllow: /*/*?mute=\nAllow: /*/*?odnBg=\nAllow: /*/*?odnHeight=\nAllow: /*/*?odnWidth=\nAllow: /*/*?output=\nAllow: /*/*?outro=\nAllow: /*/*?per_page=\nAllow: /*/*?pgroup=\nAllow: /*/*?platform=\nAllow: /*/*?play=\nAllow: /*/*?player_id=\nAllow: /*/*?playerapiid=\nAllow: /*/*?playlist=\nAllow: /*/*?playsinline=\nAllow: /*/*?portrait=\nAllow: /*/*?progress_bar=\nAllow: /*/*?quality_selector=\nAllow: /*/*?quality=\nAllow: /*/*?ra=\nAllow: /*/*?random=\nAllow: /*/*?redirect=\nAllow: /*/*?rel=\nAllow: /*/*?resize=\nAllow: /*/*?responsive=\nAllow: /*/*?rmxv=\nAllow: /*/*?rt=\nAllow: /*/*?rv=\nAllow: /*/*?s=\nAllow: /*/*?sa=\nAllow: /*/*?scrlybrkr=\nAllow: /*/*?sd=\nAllow: /*/*?server=\nAllow: /*/*?sfmc_id=\nAllow: /*/*?show_byline=\nAllow: /*/*?show_portrait=\nAllow: /*/*?show_title=\nAllow: /*/*?showinfo=\nAllow: /*/*?sig=\nAllow: /*/*?signup=\nAllow: /*/*?sizes=\nAllow: /*/*?sl_key=\nAllow: /*/*?sort=\nAllow: /*/*?speed=\nAllow: /*/*?sr=\nAllow: /*/*?ssl=\nAllow: /*/*?ssp_iabi=\nAllow: /*/*?start=\nAllow: /*/*?streamid=\nAllow: /*/*?sv=\nAllow: /*/*?syclid=\nAllow: /*/*?tblci=\nAllow: /*/*?test=\nAllow: /*/*?texttrack=\nAllow: /*/*?title=\nAllow: /*/*?tq=\nAllow: /*/*?transparent=\nAllow: /*/*?trk=\nAllow: /*/*?type=\nAllow: /*/*?url=\nAllow: /*/*?user_id=\nAllow: /*/*?usg=\nAllow: /*/*?utm_date=\nAllow: /*/*?utm_id=\nAllow: /*/*?v=\nAllow: /*/*?version=\nAllow: /*/*?volumeOff=\nAllow: /*/*?vq=\nAllow: /*/*?w=\nAllow: /*/*?ysclid=\nCrawl-delay: 2\n\nUser-agent: ia_archiver\nDisallow: /api/v3\nDisallow: /*/format:thumbnail\nDisallow: /download/\nDisallow: /*/download?\nDisallow: /couchmode/\nDisallow: /groups/*/forum/*\nDisallow: /musicstore/preview\nDisallow: /musicstore/download\nDisallow: /forgot_password\nDisallow: /help/forgot_login\nDisallow: /help/verify_email\nDisallow: /help/zendesk_sso\nDisallow: /help/sso\nDisallow: /user/verify\nDisallow: /search/\nDisallow: /search?\nDisallow: /search\\?\nDisallow: /_\nDisallow: /*mkc=*&\nDisallow: /*&mkc=*\nDisallow: /*gl=*&\nDisallow: /*&gl=*\nDisallow: /_global_notifications\nDisallow: /*?*referrer_vuid=\nDisallow: /*?*vimeo_search\nDisallow: /store/\nDisallow: /*?*_gl=\nDisallow: /*/collections/channels\nDisallow: /manage/videos/\nDisallow: /flarepoint/\nDisallow: /*/likes \nDisallow: /reviews/\nDisallow: /*/sort:*/*:\nDisallow: /ablincoln/\nDisallow: /*tag:\nDisallow: /*/videos/\nDisallow: /legal/~gitbook/\nDisallow: /manage/showcase\nDisallow: /api/v3/me$\nDisallow: /api/v3/me/\nDisallow: /api/v3/me?\nDisallow: /api/v3/modworld\nDisallow: /api/v3/*/comments\n# Clip Page Parameters\nDisallow: /*?_=\nDisallow: /*?_hsenc=\nDisallow: /*?_hsmi=\nDisallow: /*?*&_=\nDisallow: /*?*&_hsenc=\nDisallow: /*?*&_hsmi=\nDisallow: /*?*&1=\nDisallow: /*?*&activityReferer=\nDisallow: /*?*&adcPathname=\nDisallow: /*?*&ajax=\nDisallow: /*?*&allowfullscreen=\nDisallow: /*?*&autohide=\nDisallow: /*?*&autopause=\nDisallow: /*?*&background=\nDisallow: /*?*&badge=\nDisallow: /*?*&brid=\nDisallow: /*?*&byline=\nDisallow: /*?*&callback=\nDisallow: /*?*&cb=\nDisallow: /*?*&cc_load_policy=\nDisallow: /*?*&ceid=\nDisallow: /*?*&cjdata=\nDisallow: /*?*&cjevent=\nDisallow: /*?*&clickid=\nDisallow: /*?*&clipId=\nDisallow: /*?*&color=\nDisallow: /*?*&consentEnabled=\nDisallow: /*?*&content=\nDisallow: /*?*&context=\nDisallow: /*?*&controls=\nDisallow: /*?*&d=\nDisallow: /*?*&data=\nDisallow: /*?*&dnt=\nDisallow: /*?*&ei=\nDisallow: /*?*&ej=\nDisallow: /*?*&embed_email_provider=\nDisallow: /*?*&emci=\nDisallow: /*?*&emdi=\nDisallow: /*?*&enablejsapi=\nDisallow: /*?*&f1=\nDisallow: /*?*&filter_location=\nDisallow: /*?*&filter_skills=\nDisallow: /*?*&fit=\nDisallow: /*?*&flashvars=\nDisallow: /*?*&foe=\nDisallow: /*?*&format=\nDisallow: /*?*&from=\nDisallow: /*?*&fromApp=\nDisallow: /*?*&fullscreen=\nDisallow: /*?*&h=\nDisallow: /*?*&hashKey=\nDisallow: /*?*&hd=\nDisallow: /*?*&height=\nDisallow: /*?*&hss_meta=\nDisallow: /*?*&id=\nDisallow: /*?*&iframe=\nDisallow: /*?*&itok=\nDisallow: /*?*&keepEmbedCode=\nDisallow: /*?*&keyboard=\nDisallow: /*?1=\nDisallow: /*?activityReferer=\nDisallow: /*?adcPathname=\nDisallow: /*?ajax=\nDisallow: /*?allowfullscreen=\nDisallow: /*?autohide=\nDisallow: /*?autopause=\nDisallow: /*?background=\nDisallow: /*?badge=\nDisallow: /*?brid=\nDisallow: /*?byline=\nDisallow: /*?callback=\nDisallow: /*?cb=\nDisallow: /*?cc_load_policy=\nDisallow: /*?ceid=\nDisallow: /*?cjdata=\nDisallow: /*?cjevent=\nDisallow: /*?clickid=\nDisallow: /*?clipId=\nDisallow: /*?color=\nDisallow: /*?consentEnabled=\nDisallow: /*?content=\nDisallow: /*?context=\nDisallow: /*?controls=\nDisallow: /*?d=\nDisallow: /*?data=\nDisallow: /*?dnt=\nDisallow: /*?ei=\nDisallow: /*?ej=\nDisallow: /*?embed_email_provider=\nDisallow: /*?emci=\nDisallow: /*?emdi=\nDisallow: /*?enablejsapi=\nDisallow: /*?f1=\nDisallow: /*?filter_location=\nDisallow: /*?filter_skills=\nDisallow: /*?fit=\nDisallow: /*?flashvars=\nDisallow: /*?foe=\nDisallow: /*?format=\nDisallow: /*?from=\nDisallow: /*?fromApp=\nDisallow: /*?fullscreen=\nDisallow: /*?h=\nDisallow: /*?hashKey=\nDisallow: /*?hd=\nDisallow: /*?height=\nDisallow: /*?hss_meta=\nDisallow: /*?id=\nDisallow: /*?iframe=\nDisallow: /*?itok=\nDisallow: /*?keepEmbedCode=\nDisallow: /*?keyboard=\nDisallow: /*?*&%3Bfe=\nDisallow: /*?*&%3Bfl=\nDisallow: /*?*&kref=\nDisallow: /*?*&kuid=\nDisallow: /*?*&lang=\nDisallow: /*?*&lightbox[modal]=\nDisallow: /*?*&linkId=\nDisallow: /*?*&lite=\nDisallow: /*?*&login=\nDisallow: /*?*&loop=\nDisallow: /*?*&maxheight=\nDisallow: /*?*&maxw=\nDisallow: /*?*&maxwidth=\nDisallow: /*?*&mc_cid=\nDisallow: /*?*&method=\nDisallow: /*?*&mibextid=\nDisallow: /*?*&modestbranding=\nDisallow: /*?*&ms=\nDisallow: /*?*&msockid=\nDisallow: /*?*&mute=\nDisallow: /*?*&odnBg=\nDisallow: /*?*&odnHeight=\nDisallow: /*?*&odnWidth=\nDisallow: /*?*&output=\nDisallow: /*?*&outro=\nDisallow: /*?*&per_page=\nDisallow: /*?*&pgroup=\nDisallow: /*?*&platform=\nDisallow: /*?*&play=\nDisallow: /*?*&player_id=\nDisallow: /*?*&playerapiid=\nDisallow: /*?*&playlist=\nDisallow: /*?*&playsinline=\nDisallow: /*?*&portrait=\nDisallow: /*?*&progress_bar=\nDisallow: /*?*&quality_selector=\nDisallow: /*?*&quality=\nDisallow: /*?*&ra=\nDisallow: /*?*&random=\nDisallow: /*?*&redirect=\nDisallow: /*?*&rel=\nDisallow: /*?*&resize=\nDisallow: /*?*&responsive=\nDisallow: /*?*&rmxv=\nDisallow: /*?*&rt=\nDisallow: /*?*&rv=\nDisallow: /*?*&s=\nDisallow: /*?*&sa=\nDisallow: /*?*&scrlybrkr=\nDisallow: /*?*&sd=\nDisallow: /*?*&server=\nDisallow: /*?*&sfmc_id=\nDisallow: /*?*&show_byline=\nDisallow: /*?*&show_portrait=\nDisallow: /*?*&show_title=\nDisallow: /*?*&showinfo=\nDisallow: /*?*&sig=\nDisallow: /*?*&signup=\nDisallow: /*?*&sizes=\nDisallow: /*?*&sl_key=\nDisallow: /*?*&sort=\nDisallow: /*?*&speed=\nDisallow: /*?*&sr=\nDisallow: /*?*&ssl=\nDisallow: /*?*&ssp_iabi=\nDisallow: /*?*&start=\nDisallow: /*?*&streamid=\nDisallow: /*?*&sv=\nDisallow: /*?*&syclid=\nDisallow: /*?*&tblci=\nDisallow: /*?*&test=\nDisallow: /*?*&texttrack=\nDisallow: /*?*&title=\nDisallow: /*?*&tq=\nDisallow: /*?*&transparent=\nDisallow: /*?*&trk=\nDisallow: /*?*&type=\nDisallow: /*?*&url=\nDisallow: /*?*&user_id=\nDisallow: /*?*&usg=\nDisallow: /*?*&utm_date=\nDisallow: /*?*&utm_id=\nDisallow: /*?*&v=\nDisallow: /*?*&version=\nDisallow: /*?*&volumeOff=\nDisallow: /*?*&vq=\nDisallow: /*?*&w=\nDisallow: /*?*&ysclid=\nDisallow: /*?%3Bfe=\nDisallow: /*?%3Bfl=\nDisallow: /*?kref=\nDisallow: /*?kuid=\nDisallow: /*?lang=\nDisallow: /*?lightbox[modal]=\nDisallow: /*?linkId=\nDisallow: /*?lite=\nDisallow: /*?login=\nDisallow: /*?loop=\nDisallow: /*?maxheight=\nDisallow: /*?maxw=\nDisallow: /*?maxwidth=\nDisallow: /*?mc_cid=\nDisallow: /*?method=\nDisallow: /*?mibextid=\nDisallow: /*?modestbranding=\nDisallow: /*?ms=\nDisallow: /*?msockid=\nDisallow: /*?mute=\nDisallow: /*?odnBg=\nDisallow: /*?odnHeight=\nDisallow: /*?odnWidth=\nDisallow: /*?output=\nDisallow: /*?outro=\nDisallow: /*?per_page=\nDisallow: /*?pgroup=\nDisallow: /*?platform=\nDisallow: /*?play=\nDisallow: /*?player_id=\nDisallow: /*?playerapiid=\nDisallow: /*?playlist=\nDisallow: /*?playsinline=\nDisallow: /*?portrait=\nDisallow: /*?progress_bar=\nDisallow: /*?quality_selector=\nDisallow: /*?quality=\nDisallow: /*?ra=\nDisallow: /*?random=\nDisallow: /*?redirect=\nDisallow: /*?rel=\nDisallow: /*?resize=\nDisallow: /*?responsive=\nDisallow: /*?rmxv=\nDisallow: /*?rt=\nDisallow: /*?rv=\nDisallow: /*?s=\nDisallow: /*?sa=\nDisallow: /*?scrlybrkr=\nDisallow: /*?sd=\nDisallow: /*?server=\nDisallow: /*?sfmc_id=\nDisallow: /*?show_byline=\nDisallow: /*?show_portrait=\nDisallow: /*?show_title=\nDisallow: /*?showinfo=\nDisallow: /*?sig=\nDisallow: /*?signup=\nDisallow: /*?sizes=\nDisallow: /*?sl_key=\nDisallow: /*?sort=\nDisallow: /*?speed=\nDisallow: /*?sr=\nDisallow: /*?ssl=\nDisallow: /*?ssp_iabi=\nDisallow: /*?start=\nDisallow: /*?streamid=\nDisallow: /*?sv=\nDisallow: /*?syclid=\nDisallow: /*?tblci=\nDisallow: /*?test=\nDisallow: /*?texttrack=\nDisallow: /*?title=\nDisallow: /*?tq=\nDisallow: /*?transparent=\nDisallow: /*?trk=\nDisallow: /*?type=\nDisallow: /*?url=\nDisallow: /*?user_id=\nDisallow: /*?usg=\nDisallow: /*?utm_date=\nDisallow: /*?utm_id=\nDisallow: /*?v=\nDisallow: /*?version=\nDisallow: /*?volumeOff=\nDisallow: /*?vq=\nDisallow: /*?w=\nDisallow: /*?ysclid=\nCrawl-delay: 2\n\nUser-agent: SeznamBot\nRequest-rate: 1/10s\nDisallow: */format:thumbnail\nDisallow: /download/\nDisallow: /*/download?\nDisallow: /couchmode/\nDisallow: /_global_notifications\nDisallow: /musicstore/preview\nDisallow: /musicstore/download\nDisallow: /forgot_password\nDisallow: /help/forgot_login\nDisallow: /help/verify_email\nDisallow: /help/zendesk_sso\nDisallow: /help/sso\nDisallow: /api/\nDisallow: /flarepoint/\nDisallow: /user/verify\nDisallow: /search/\nDisallow: /search?\nDisallow: /search\\?\nDisallow: /_*\n\nUser-agent: msnbot\nCrawl-delay: 5\n\nUser-agent: Bingbot\nCrawl-delay: 5\n\nUser-agent: AdIdxBot\nCrawl-delay: 5\n\nUser-agent: BingPreview\nCrawl-delay: 5\n\nUser-agent: Yandex\nCrawl-delay: 2\n\n# Block Open AI - Allowing specific marketing/informational paths\nUser-agent: GPTBot\nDisallow: /\nAllow: /features/\nAllow: /solutions/\nAllow: /product/\nAllow: /video-uploader\nAllow: /upgrade-plan\nAllow: /enterprise/\nAllow: /integrations/\nAllow: /blog/\nAllow: /create/\nAllow: /ott/\nAllow: /customers/\nAllow: /events/\nAllow: /campaigns/\nAllow: /about/\nAllow: /vimeo-academy/\nAllow: /$\nAllow: /es/features/\nAllow: /es/solutions/\nAllow: /es/product/\nAllow: /es/video-uploader\nAllow: /es/enterprise/\nAllow: /es/integrations/\nAllow: /es/blog/\nAllow: /es/create/\nAllow: /es/ott/\nAllow: /es/customers/\nAllow: /es/events/\nAllow: /es/campaigns/\nAllow: /es/about/\nAllow: /es/vimeo-academy/\nAllow: /es/$\nAllow: /fr/features/\nAllow: /fr/solutions/\nAllow: /fr/product/\nAllow: /fr/video-uploader\nAllow: /fr/enterprise/\nAllow: /fr/integrations/\nAllow: /fr/blog/\nAllow: /fr/create/\nAllow: /fr/ott/\nAllow: /fr/customers/\nAllow: /fr/events/\nAllow: /fr/campaigns/\nAllow: /fr/about/\nAllow: /fr/vimeo-academy/\nAllow: /fr/$\nAllow: /de/features/\nAllow: /de/solutions/\nAllow: /de/product/\nAllow: /de/video-uploader\nAllow: /de/enterprise/\nAllow: /de/integrations/\nAllow: /de/blog/\nAllow: /de/create/\nAllow: /de/ott/\nAllow: /de/customers/\nAllow: /de/events/\nAllow: /de/campaigns/\nAllow: /de/about/\nAllow: /de/vimeo-academy/\nAllow: /de/$\nAllow: /pt-br/features/\nAllow: /pt-br/solutions/\nAllow: /pt-br/product/\nAllow: /pt-BR/video-uploader\nAllow: /pt-br/enterprise/\nAllow: /pt-br/integrations/\nAllow: /pt-br/blog/\nAllow: /pt-br/create/\nAllow: /pt-br/ott/\nAllow: /pt-br/customers/\nAllow: /pt-br/events/\nAllow: /pt-br/campaigns/\nAllow: /pt-br/about/\nAllow: /pt-br/vimeo-academy/\nAllow: /pt-br/$\nAllow: /ko/features/\nAllow: /ko/solutions/\nAllow: /ko/product/\nAllow: /ko/video-uploader\nAllow: /ko/enterprise/\nAllow: /ko/integrations/\nAllow: /ko/blog/\nAllow: /ko/create/\nAllow: /ko/ott/\nAllow: /ko/customers/\nAllow: /ko/events/\nAllow: /ko/campaigns/\nAllow: /ko/about/\nAllow: /ko/vimeo-academy/\nAllow: /ko/$\nAllow: /jp/features/\nAllow: /jp/solutions/\nAllow: /jp/product/\nAllow: /jp/video-uploader\nAllow: /jp/enterprise/\nAllow: /jp/integrations/\nAllow: /jp/blog/\nAllow: /jp/create/\nAllow: /jp/ott/\nAllow: /jp/customers/\nAllow: /jp/events/\nAllow: /jp/campaigns/\nAllow: /jp/about/\nAllow: /jp/vimeo-academy/\nAllow: /jp/$\n\n# Block Google (Gemini) - Allowing specific marketing/informational paths\nUser-agent: Google-Extended\nDisallow: /\nAllow: /features/\nAllow: /solutions/\nAllow: /product/\nAllow: /video-uploader\nAllow: /upgrade-plan\nAllow: /enterprise/\nAllow: /integrations/\nAllow: /blog/\nAllow: /create/\nAllow: /ott/\nAllow: /customers/\nAllow: /events/\nAllow: /campaigns/\nAllow: /about/\nAllow: /vimeo-academy/\nAllow: /$\nAllow: /es/features/\nAllow: /es/solutions/\nAllow: /es/product/\nAllow: /es/video-uploader\nAllow: /es/enterprise/\nAllow: /es/integrations/\nAllow: /es/blog/\nAllow: /es/create/\nAllow: /es/ott/\nAllow: /es/customers/\nAllow: /es/events/\nAllow: /es/campaigns/\nAllow: /es/about/\nAllow: /es/vimeo-academy/\nAllow: /es/$\nAllow: /fr/features/\nAllow: /fr/solutions/\nAllow: /fr/product/\nAllow: /fr/video-uploader\nAllow: /fr/enterprise/\nAllow: /fr/integrations/\nAllow: /fr/blog/\nAllow: /fr/create/\nAllow: /fr/ott/\nAllow: /fr/customers/\nAllow: /fr/events/\nAllow: /fr/campaigns/\nAllow: /fr/about/\nAllow: /fr/vimeo-academy/\nAllow: /fr/$\nAllow: /de/features/\nAllow: /de/solutions/\nAllow: /de/product/\nAllow: /de/video-uploader\nAllow: /de/enterprise/\nAllow: /de/integrations/\nAllow: /de/blog/\nAllow: /de/create/\nAllow: /de/ott/\nAllow: /de/customers/\nAllow: /de/events/\nAllow: /de/campaigns/\nAllow: /de/about/\nAllow: /de/vimeo-academy/\nAllow: /de/$\nAllow: /pt-br/features/\nAllow: /pt-br/solutions/\nAllow: /pt-br/product/\nAllow: /pt-BR/video-uploader\nAllow: /pt-br/enterprise/\nAllow: /pt-br/integrations/\nAllow: /pt-br/blog/\nAllow: /pt-br/create/\nAllow: /pt-br/ott/\nAllow: /pt-br/customers/\nAllow: /pt-br/events/\nAllow: /pt-br/campaigns/\nAllow: /pt-br/about/\nAllow: /pt-br/vimeo-academy/\nAllow: /pt-br/$\nAllow: /ko/features/\nAllow: /ko/solutions/\nAllow: /ko/product/\nAllow: /ko/video-uploader\nAllow: /ko/enterprise/\nAllow: /ko/integrations/\nAllow: /ko/blog/\nAllow: /ko/create/\nAllow: /ko/ott/\nAllow: /ko/customers/\nAllow: /ko/events/\nAllow: /ko/campaigns/\nAllow: /ko/about/\nAllow: /ko/vimeo-academy/\nAllow: /ko/$\nAllow: /jp/features/\nAllow: /jp/solutions/\nAllow: /jp/product/\nAllow: /jp/video-uploader\nAllow: /jp/enterprise/\nAllow: /jp/integrations/\nAllow: /jp/blog/\nAllow: /jp/create/\nAllow: /jp/ott/\nAllow: /jp/customers/\nAllow: /jp/events/\nAllow: /jp/campaigns/\nAllow: /jp/about/\nAllow: /jp/vimeo-academy/\nAllow: /jp/$\n\n# Block Claude - Allowing specific marketing/informational paths\nUser-agent: anthropic-ai\nUser-agent: Claude-Web\nUser-agent: Claudebot\nDisallow: /\nAllow: /features/\nAllow: /solutions/\nAllow: /product/\nAllow: /video-uploader\nAllow: /upgrade-plan\nAllow: /enterprise/\nAllow: /integrations/\nAllow: /blog/\nAllow: /create/\nAllow: /ott/\nAllow: /customers/\nAllow: /events/\nAllow: /campaigns/\nAllow: /about/\nAllow: /vimeo-academy/\nAllow: /$\nAllow: /es/features/\nAllow: /es/solutions/\nAllow: /es/product/\nAllow: /es/video-uploader\nAllow: /es/enterprise/\nAllow: /es/integrations/\nAllow: /es/blog/\nAllow: /es/create/\nAllow: /es/ott/\nAllow: /es/customers/\nAllow: /es/events/\nAllow: /es/campaigns/\nAllow: /es/about/\nAllow: /es/vimeo-academy/\nAllow: /es/$\nAllow: /fr/features/\nAllow: /fr/solutions/\nAllow: /fr/product/\nAllow: /fr/video-uploader\nAllow: /fr/enterprise/\nAllow: /fr/integrations/\nAllow: /fr/blog/\nAllow: /fr/create/\nAllow: /fr/ott/\nAllow: /fr/customers/\nAllow: /fr/events/\nAllow: /fr/campaigns/\nAllow: /fr/about/\nAllow: /fr/vimeo-academy/\nAllow: /fr/$\nAllow: /de/features/\nAllow: /de/solutions/\nAllow: /de/product/\nAllow: /de/video-uploader\nAllow: /de/enterprise/\nAllow: /de/integrations/\nAllow: /de/blog/\nAllow: /de/create/\nAllow: /de/ott/\nAllow: /de/customers/\nAllow: /de/events/\nAllow: /de/campaigns/\nAllow: /de/about/\nAllow: /de/vimeo-academy/\nAllow: /de/$\nAllow: /pt-br/features/\nAllow: /pt-br/solutions/\nAllow: /pt-br/product/\nAllow: /pt-BR/video-uploader\nAllow: /pt-br/enterprise/\nAllow: /pt-br/integrations/\nAllow: /pt-br/blog/\nAllow: /pt-br/create/\nAllow: /pt-br/ott/\nAllow: /pt-br/customers/\nAllow: /pt-br/events/\nAllow: /pt-br/campaigns/\nAllow: /pt-br/about/\nAllow: /pt-br/vimeo-academy/\nAllow: /pt-br/$\nAllow: /ko/features/\nAllow: /ko/solutions/\nAllow: /ko/product/\nAllow: /ko/video-uploader\nAllow: /ko/enterprise/\nAllow: /ko/integrations/\nAllow: /ko/blog/\nAllow: /ko/create/\nAllow: /ko/ott/\nAllow: /ko/customers/\nAllow: /ko/events/\nAllow: /ko/campaigns/\nAllow: /ko/about/\nAllow: /ko/vimeo-academy/\nAllow: /ko/$\nAllow: /jp/features/\nAllow: /jp/solutions/\nAllow: /jp/product/\nAllow: /jp/video-uploader\nAllow: /jp/enterprise/\nAllow: /jp/integrations/\nAllow: /jp/blog/\nAllow: /jp/create/\nAllow: /jp/ott/\nAllow: /jp/customers/\nAllow: /jp/events/\nAllow: /jp/campaigns/\nAllow: /jp/about/\nAllow: /jp/vimeo-academy/\nAllow: /jp/$\n\n# Block CommonCrawl\nUser-agent: CCBot\nUser-agent: CC-Crawler\nUser-agent: Common Crawl\nUser-agent: CommonCrawl\nDisallow: /\n\n# Block Diffbot\nUser-agent: Diffbot\nDisallow: /\n\n# Block Meta (Facebook)\nUser-agent: FacebookBot\nDisallow: /\n\n# Block ByteDance\nUser-agent: Bytespider\nDisallow: /\n\n# Block Webz.io\nUser-agent: Omgilibot\nDisallow: /\nUser-agent: Omgili\nDisallow: /\n\n# Block ImagesiftBot\nUser-agent: ImagesiftBot\nDisallow: /\n\n# Block Meltwater\nUser-agent: Meltwater\nDisallow: /\n\n# Block Seekr\nUser-agent: Seekr\nDisallow: /\n\n# Block Grok\nUser-agent: Grok\nDisallow: /\n\n# Sitemap declaration\nSitemap: https://vimeo.com/marketing.xml.gz\nSitemap: https://vimeo.com/sitemap/videos.xml.gz\n\n","agents":{"GPTBot":true,"OAI-SearchBot":true,"ChatGPT-User":true,"ClaudeBot":true,"Claude-SearchBot":true,"PerplexityBot":true,"Google-Extended":true,"Applebot-Extended":true,"CCBot":false,"Bytespider":false,"Meta-ExternalAgent":true,"Amazonbot":true}},"sitemap":{"status":200,"content_type":"application/xml","bytes":2759,"sha256":"cc3acaba5ab90666a4248794babebd24e2d5edfd3f04a4db29b8088f7014dc3b","final_host":"vimeo.com","final_path":"/marketing.xml.gz","ms":20,"url":"https://vimeo.com/marketing.xml.gz","declared":true,"xml":true},"files":{"llms_txt":{"status":404,"content_type":"text/html","bytes":163574,"sha256":"9e1c34c126e0ebd50e658e6cf0c2a5192909e778148e3b95dc534bdce80375e1","final_host":"vimeo.com","final_path":"/llms.txt","ms":189,"path":"/llms.txt","html":true,"json":null,"present":false},"agents_md":{"status":404,"content_type":"text/html","bytes":111658,"sha256":"886600c19b428b9234d353c9a6026d320e90e9dc3ac554b98b02d0892a35b88b","final_host":"vimeo.com","final_path":"/agents.md","ms":104,"path":"/agents.md","html":true,"json":null,"present":false},"agent_skills":{"status":404,"content_type":"text/html","bytes":163574,"sha256":"9e1c34c126e0ebd50e658e6cf0c2a5192909e778148e3b95dc534bdce80375e1","final_host":"vimeo.com","final_path":"/.well-known/agent-skills/index.json","ms":377,"path":"/.well-known/agent-skills/index.json","html":true,"json":null,"present":false},"mcp_card":{"status":404,"content_type":"text/html","bytes":163574,"sha256":"9e1c34c126e0ebd50e658e6cf0c2a5192909e778148e3b95dc534bdce80375e1","final_host":"vimeo.com","final_path":"/.well-known/mcp/server-card.json","ms":333,"path":"/.well-known/mcp/server-card.json","html":true,"json":null,"present":false},"api_catalog":{"status":404,"content_type":"text/html","bytes":163574,"sha256":"9e1c34c126e0ebd50e658e6cf0c2a5192909e778148e3b95dc534bdce80375e1","final_host":"vimeo.com","final_path":"/.well-known/api-catalog","ms":163,"path":"/.well-known/api-catalog","html":true,"json":null,"present":false},"a2a_card":{"status":404,"content_type":"text/html","bytes":163574,"sha256":"9e1c34c126e0ebd50e658e6cf0c2a5192909e778148e3b95dc534bdce80375e1","final_host":"vimeo.com","final_path":"/.well-known/agent-card.json","ms":126,"path":"/.well-known/agent-card.json","html":true,"json":null,"present":false}},"rings":{"v":1,"candidates":[{"url":"https://mcp.vimeo.com/mcp","source":"conventional_host"}],"mcp":[{"url":"https://mcp.vimeo.com/mcp","source":"conventional_host","outcome":"auth_required","status":401,"ms":73,"initialized":false,"protocol_version":null,"server":null,"tools":null,"tools_sha256":null,"auth_header":true}],"a2a":null,"webmcp":{"model_context":false,"bridge":false,"mcp_json":false,"link_rel":false,"any":false},"handshakes":1,"get_path":{"status":200,"content_type":"text/html","ms":328,"error":null},"get_host":{"status":401,"content_type":"text/plain","ms":148,"error":null}},"sf":[-1,0,0,0,0,0,2,0,1],"derived":{"reachable":true,"home_class":"answers","robots_served":true,"robots_sitemap":true,"content_signal":false,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":["CCBot","Bytespider"],"blocks_any_ai":true,"files":{"llms_txt":false,"agents_md":false,"agent_skills":false,"mcp_card":false,"api_catalog":false,"a2a_card":false},"any_capability":false,"ai_refused":[],"ai_thin":[],"aipref":0,"aipref_measured":true,"sf":[-1,0,0,0,0,0,2,0,1],"rings":{"mcp_declared":true,"mcp_live":false,"mcp_auth":true,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":3,"fh":"vimeo.com","fp_other_host":[],"fp":"e699fe5c0d05a34a","fp_bits":{"gptbot":0,"claudebot":0,"oaisearch":0,"perplexity":0}},"derive_v":5,"frame":"top1k","shard":6},{"v":1,"kind":"crawlcheck-census-record","run":43,"domain":"gvt2.com","rank":88,"measured_at":"2026-11-13T08:09:13.777Z","home":{"browser":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"gvt2.com","final_path":"/","ms":97,"identity":"Mobile browser","html":false},"gptbot":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"gvt2.com","final_path":"/","ms":97,"identity":"GPTBot","html":false},"claudebot":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"gvt2.com","final_path":"/","ms":97,"identity":"ClaudeBot","html":false},"oaisearch":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"gvt2.com","final_path":"/","ms":97,"identity":"OAI-SearchBot","html":false},"perplexity":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"gvt2.com","final_path":"/","ms":96,"identity":"PerplexityBot","html":false}},"robots":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"gvt2.com","final_path":"/robots.txt","ms":97,"html":false,"text":null,"agents":null},"sitemap":null,"files":null,"derived":{"reachable":false,"home_class":"no_site","robots_served":false,"robots_sitemap":false,"content_signal":false,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":[],"blocks_any_ai":false,"files":{"llms_txt":false,"agents_md":false,"agent_skills":false,"mcp_card":false,"api_catalog":false,"a2a_card":false},"any_capability":false,"ai_refused":[],"ai_thin":[],"rings":{"mcp_declared":false,"mcp_live":false,"mcp_auth":false,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":3,"fh":"gvt2.com","fp_other_host":[]},"derive_v":5,"frame":"top1k","shard":7},{"v":1,"kind":"crawlcheck-census-record","run":43,"domain":"cdninstagram.com","rank":89,"measured_at":"2026-11-13T07:57:14.313Z","home":{"browser":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"cdninstagram.com","final_path":"/","ms":10,"identity":"Mobile browser","html":false},"gptbot":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"cdninstagram.com","final_path":"/","ms":5,"identity":"GPTBot","html":false},"claudebot":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"cdninstagram.com","final_path":"/","ms":3,"identity":"ClaudeBot","html":false},"oaisearch":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"cdninstagram.com","final_path":"/","ms":3,"identity":"OAI-SearchBot","html":false},"perplexity":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"cdninstagram.com","final_path":"/","ms":2,"identity":"PerplexityBot","html":false}},"robots":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"cdninstagram.com","final_path":"/robots.txt","ms":1,"html":false,"text":null,"agents":null},"sitemap":null,"files":null,"derived":{"reachable":false,"home_class":"no_site","robots_served":false,"robots_sitemap":false,"content_signal":false,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":[],"blocks_any_ai":false,"files":{"llms_txt":false,"agents_md":false,"agent_skills":false,"mcp_card":false,"api_catalog":false,"a2a_card":false},"any_capability":false,"ai_refused":[],"ai_thin":[],"rings":{"mcp_declared":false,"mcp_live":false,"mcp_auth":false,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":3,"fh":"cdninstagram.com","fp_other_host":[]},"derive_v":5,"frame":"top1k","shard":0},{"v":1,"kind":"crawlcheck-census-record","run":43,"domain":"openai.com","rank":90,"measured_at":"2026-11-13T07:58:36.062Z","home":{"browser":{"status":403,"content_type":"text/html","bytes":9861,"sha256":"00a67f557a47ea402b973df8d63e9de5aa346cc0ba63c2207727260f70904975","final_host":"openai.com","final_path":"/","ms":8,"identity":"Mobile browser","html":true},"gptbot":{"status":403,"content_type":"text/html","bytes":9861,"sha256":"ea746c20f728abbb154998df3a9b32f38e95ac54b2af5b217c19fe8ccc3d725b","final_host":"openai.com","final_path":"/","ms":2,"identity":"GPTBot","html":true},"claudebot":{"status":403,"content_type":"text/html","bytes":9775,"sha256":"a0380de2af57a0d43b4e34656d009b270e2f937684652bdd699a2c7d59237002","final_host":"openai.com","final_path":"/","ms":1,"identity":"ClaudeBot","html":true},"oaisearch":{"status":403,"content_type":"text/html","bytes":9882,"sha256":"0ce6f4edb1471969a7396339bfde638aa3acafd0003061a6ae51391aeb25f7fd","final_host":"openai.com","final_path":"/","ms":1,"identity":"OAI-SearchBot","html":true},"perplexity":{"status":403,"content_type":"text/html","bytes":9968,"sha256":"c379fdd1ebfd8b5b2dadeab3c25360cc2b883a012327b5a9f37fbc54bc9e52bf","final_host":"openai.com","final_path":"/","ms":1,"identity":"PerplexityBot","html":true}},"robots":{"status":200,"content_type":"text/plain","bytes":98,"sha256":"c53100563b9d289efeebb6f826a20c566578e1337e3096e298eb9da5ecc2e498","final_host":"openai.com","final_path":"/robots.txt","ms":4,"html":false,"text":"User-agent: *\nAllow: /\nDisallow: /microsoft-for-startups/\n\nSitemap: https://openai.com/sitemap.xml","agents":{"GPTBot":true,"OAI-SearchBot":true,"ChatGPT-User":true,"ClaudeBot":true,"Claude-SearchBot":true,"PerplexityBot":true,"Google-Extended":true,"Applebot-Extended":true,"CCBot":true,"Bytespider":true,"Meta-ExternalAgent":true,"Amazonbot":true}},"sitemap":null,"files":null,"derived":{"reachable":false,"home_class":"refused_us","robots_served":false,"robots_sitemap":false,"content_signal":false,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":[],"blocks_any_ai":false,"files":{"llms_txt":false,"agents_md":false,"agent_skills":false,"mcp_card":false,"api_catalog":false,"a2a_card":false},"any_capability":false,"ai_refused":[],"ai_thin":[],"rings":{"mcp_declared":false,"mcp_live":false,"mcp_auth":false,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":3,"fh":"openai.com","fp_other_host":[]},"derive_v":5,"frame":"top1k","shard":1},{"v":1,"kind":"crawlcheck-census-record","run":43,"domain":"f5.com","rank":91,"measured_at":"2026-11-13T07:59:31.658Z","home":{"browser":{"status":200,"content_type":"text/html","bytes":596746,"sha256":"84aa88986c25f684a07069fb6ad3d86ce2384a69397a66e76f34cd9a6ec8fe30","final_host":"www.f5.com","final_path":"/","ms":744,"identity":"Mobile browser","html":true,"fp":"6e38f356021bc19b"},"gptbot":{"status":200,"content_type":"text/html","bytes":596746,"sha256":"84aa88986c25f684a07069fb6ad3d86ce2384a69397a66e76f34cd9a6ec8fe30","final_host":"www.f5.com","final_path":"/","ms":389,"identity":"GPTBot","html":true,"fp":"6e38f356021bc19b"},"claudebot":{"status":200,"content_type":"text/html","bytes":596746,"sha256":"84aa88986c25f684a07069fb6ad3d86ce2384a69397a66e76f34cd9a6ec8fe30","final_host":"www.f5.com","final_path":"/","ms":389,"identity":"ClaudeBot","html":true,"fp":"6e38f356021bc19b"},"oaisearch":{"status":200,"content_type":"text/html","bytes":596703,"sha256":"3bc09e7d09c0dc9ada38993fef8575da1f7098e9475f748db35b01179f8004aa","final_host":"www.f5.com","final_path":"/","ms":940,"identity":"OAI-SearchBot","html":true,"fp":"6e38f356021bc19b"},"perplexity":{"status":200,"content_type":"text/html","bytes":596746,"sha256":"736af678e1931da8b4b3fcb6251705ce58ab290b9d760ce2545dd83ec21d7dcf","final_host":"www.f5.com","final_path":"/","ms":428,"identity":"PerplexityBot","html":true,"fp":"6e38f356021bc19b"}},"robots":{"status":200,"content_type":"text/plain","bytes":156,"sha256":"eca371645a1484bc5d7dfc8e150f188b7e70d7754e6c00c1770de364f38e1401","final_host":"www.f5.com","final_path":"/robots.txt","ms":596,"html":false,"text":"# *\nUser-agent: *\nAllow: /\n\n# Host\nHost: https://www.f5.com\n\n# Sitemaps\nSitemap: https://www.f5.com/sitemap.xml\nSitemap: https://www.f5.com/labs/sitemap.xml","agents":{"GPTBot":true,"OAI-SearchBot":true,"ChatGPT-User":true,"ClaudeBot":true,"Claude-SearchBot":true,"PerplexityBot":true,"Google-Extended":true,"Applebot-Extended":true,"CCBot":true,"Bytespider":true,"Meta-ExternalAgent":true,"Amazonbot":true}},"sitemap":{"status":200,"content_type":"application/xml","bytes":1287,"sha256":"2705ee74dea18c947ac642a4a07d1e985e48869398a4e6bf48a56879ef38f266","final_host":"www.f5.com","final_path":"/sitemap.xml","ms":100,"url":"https://www.f5.com/sitemap.xml","declared":true,"xml":true},"files":{"llms_txt":{"status":404,"content_type":"text/html","bytes":166704,"sha256":"2ff2b2a9fbf260944b37527ea458ba52fef51a94cc0ffa1338c68ffe4c40900c","final_host":"www.f5.com","final_path":"/llms.txt","ms":1117,"path":"/llms.txt","html":true,"json":null,"present":false},"agents_md":{"status":404,"content_type":"text/html","bytes":166707,"sha256":"7f0a138a68fb29a6eeba8dc8bef3b177d0bdbdaf5f2589e384e0cfc00468f9d5","final_host":"www.f5.com","final_path":"/agents.md","ms":926,"path":"/agents.md","html":true,"json":null,"present":false},"agent_skills":{"status":404,"content_type":"text/html","bytes":11333,"sha256":"0d7fbf17a8a42de1036be6b5dec4e7fe82111a3cb6b920fb8bba1a4a15255258","final_host":"www.f5.com","final_path":"/.well-known/agent-skills/index.json","ms":169,"path":"/.well-known/agent-skills/index.json","html":true,"json":null,"present":false},"mcp_card":{"status":404,"content_type":"text/html","bytes":11324,"sha256":"155f794d3c5f2b6004291a09e2fc31f4652df737540894941f4d3bf8dc721b8c","final_host":"www.f5.com","final_path":"/.well-known/mcp/server-card.json","ms":167,"path":"/.well-known/mcp/server-card.json","html":true,"json":null,"present":false},"api_catalog":{"status":404,"content_type":"text/html","bytes":11293,"sha256":"12debf650dbe3dd9b46159f0a05fffbe0758a81f65609dbdca6e2901caeb4476","final_host":"www.f5.com","final_path":"/.well-known/api-catalog","ms":200,"path":"/.well-known/api-catalog","html":true,"json":null,"present":false},"a2a_card":{"status":404,"content_type":"text/html","bytes":11305,"sha256":"4e809310babcc40c5bd89ad33fb762fa5feb3670f44b39851fa8140e23283f95","final_host":"www.f5.com","final_path":"/.well-known/agent-card.json","ms":268,"path":"/.well-known/agent-card.json","html":true,"json":null,"present":false}},"rings":{"v":1,"candidates":[],"mcp":[],"a2a":null,"webmcp":{"model_context":false,"bridge":false,"mcp_json":false,"link_rel":false,"any":false},"handshakes":0,"get_path":{"status":404,"content_type":"text/html","ms":952,"error":null},"get_host":{"status":530,"content_type":"text/plain","ms":115,"error":null}},"sf":[1,0,0,0,0,0,0,0,0],"derived":{"reachable":true,"home_class":"answers","robots_served":true,"robots_sitemap":true,"content_signal":false,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":[],"blocks_any_ai":false,"files":{"llms_txt":false,"agents_md":false,"agent_skills":false,"mcp_card":false,"api_catalog":false,"a2a_card":false},"any_capability":false,"ai_refused":[],"ai_thin":[],"aipref":0,"aipref_measured":true,"sf":[1,0,0,0,0,0,0,0,0],"rings":{"mcp_declared":false,"mcp_live":false,"mcp_auth":false,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":3,"fh":"www.f5.com","fp_other_host":[],"fp":"6e38f356021bc19b","fp_bits":{"gptbot":0,"claudebot":0,"oaisearch":0,"perplexity":0}},"derive_v":5,"frame":"top1k","shard":2},{"v":1,"kind":"crawlcheck-census-record","run":43,"domain":"gwfb.net","rank":92,"measured_at":"2026-11-13T07:59:45.450Z","home":{"browser":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"gwfb.net","final_path":"/","ms":763,"identity":"Mobile browser","html":false},"gptbot":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"gwfb.net","final_path":"/","ms":1,"identity":"GPTBot","html":false},"claudebot":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"gwfb.net","final_path":"/","ms":1,"identity":"ClaudeBot","html":false},"oaisearch":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"gwfb.net","final_path":"/","ms":1,"identity":"OAI-SearchBot","html":false},"perplexity":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"gwfb.net","final_path":"/","ms":0,"identity":"PerplexityBot","html":false}},"robots":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"gwfb.net","final_path":"/robots.txt","ms":1,"html":false,"text":null,"agents":null},"sitemap":null,"files":null,"derived":{"reachable":false,"home_class":"no_site","robots_served":false,"robots_sitemap":false,"content_signal":false,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":[],"blocks_any_ai":false,"files":{"llms_txt":false,"agents_md":false,"agent_skills":false,"mcp_card":false,"api_catalog":false,"a2a_card":false},"any_capability":false,"ai_refused":[],"ai_thin":[],"rings":{"mcp_declared":false,"mcp_live":false,"mcp_auth":false,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":3,"fh":"gwfb.net","fp_other_host":[]},"derive_v":5,"frame":"top1k","shard":3},{"v":1,"kind":"crawlcheck-census-record","run":43,"domain":"sentry.io","rank":93,"measured_at":"2026-11-13T08:06:14.315Z","home":{"browser":{"status":200,"content_type":"text/html","bytes":625105,"sha256":"01bc1944d19a57a29ba1fa454b177c91222139a3e78cc1dcdf04707dd03e7790","final_host":"sentry.io","final_path":"/welcome","ms":383,"identity":"Mobile browser","html":true,"fp":"4c20af501f70cf77"},"gptbot":{"status":200,"content_type":"text/markdown","bytes":19540,"sha256":"b52e76af70e3830da7de994c72d70255f681f73b74065f240a804736f3a3a80d","final_host":"sentry.io","final_path":"/welcome","ms":271,"identity":"GPTBot","html":false},"claudebot":{"status":200,"content_type":"text/markdown","bytes":19540,"sha256":"b52e76af70e3830da7de994c72d70255f681f73b74065f240a804736f3a3a80d","final_host":"sentry.io","final_path":"/welcome","ms":288,"identity":"ClaudeBot","html":false},"oaisearch":{"status":200,"content_type":"text/markdown","bytes":19540,"sha256":"b52e76af70e3830da7de994c72d70255f681f73b74065f240a804736f3a3a80d","final_host":"sentry.io","final_path":"/welcome","ms":287,"identity":"OAI-SearchBot","html":false},"perplexity":{"status":200,"content_type":"text/markdown","bytes":19540,"sha256":"b52e76af70e3830da7de994c72d70255f681f73b74065f240a804736f3a3a80d","final_host":"sentry.io","final_path":"/welcome","ms":281,"identity":"PerplexityBot","html":false}},"robots":{"status":200,"content_type":"text/plain","bytes":140,"sha256":"c0f7d803d97db6c0bf75804b97eed4a533f7249c27994e253ad4fa403652da96","final_host":"sentry.io","final_path":"/robots.txt","ms":151,"html":false,"text":"User-agent: *\nContent-Signal: search=yes, ai-input=yes, ai-train=yes\nDisallow: /api/\nAllow: /\n\nSitemap: https://sentry.io/sitemap-index.xml\n","agents":{"GPTBot":true,"OAI-SearchBot":true,"ChatGPT-User":true,"ClaudeBot":true,"Claude-SearchBot":true,"PerplexityBot":true,"Google-Extended":true,"Applebot-Extended":true,"CCBot":true,"Bytespider":true,"Meta-ExternalAgent":true,"Amazonbot":true}},"sitemap":{"status":200,"content_type":"application/xml","bytes":372,"sha256":"8bfba18ac85d5dbb7f46917c1e54f491c1b2a0df8559c46e9d92307ef1d41a89","final_host":"sentry.io","final_path":"/sitemap-index.xml","ms":153,"url":"https://sentry.io/sitemap-index.xml","declared":true,"xml":true},"files":{"llms_txt":{"status":200,"content_type":"text/plain","bytes":93571,"sha256":"ad086d7028cf74d7e820ef1e5ff44135e7a75f6942e2d6043f30f412a07514ba","final_host":"sentry.io","final_path":"/llms.txt","ms":236,"path":"/llms.txt","html":false,"json":null,"present":true},"agents_md":{"status":200,"content_type":"text/markdown","bytes":976,"sha256":"d74022d0efda2763dde039ddcf1155c5d0a4783227ba9d88a47eed66435a7683","final_host":"sentry.io","final_path":"/agents.md","ms":198,"path":"/agents.md","html":false,"json":null,"present":true},"agent_skills":{"status":200,"content_type":"application/json","bytes":13,"sha256":"0b513961955003886d251c9091e61f5dc648adbb93b431b1a951b08e0f6201ac","final_host":"sentry.io","final_path":"/.well-known/agent-skills/index.json","ms":103,"path":"/.well-known/agent-skills/index.json","html":false,"json":true,"present":true},"mcp_card":{"status":200,"content_type":"application/json","bytes":234,"sha256":"5fb322b52608b7465c0ca90eb50ef8cea2e63a0d657a32f35cc2765023551ae2","final_host":"sentry.io","final_path":"/.well-known/mcp/server-card.json","ms":102,"path":"/.well-known/mcp/server-card.json","html":false,"json":true,"present":true},"api_catalog":{"status":200,"content_type":"application/linkset+json","bytes":179,"sha256":"e906b19b2175c78512435aa44daef69d900227c4f8e2e225186a626f5d98733f","final_host":"sentry.io","final_path":"/.well-known/api-catalog","ms":103,"path":"/.well-known/api-catalog","html":false,"json":true,"present":true},"a2a_card":{"status":200,"content_type":"text/html","bytes":27399,"sha256":"c8c23e77922f8c148673493b12a5eae63f6a78607fedf2a5468ef7393327d096","final_host":"sentry.io","final_path":"/auth/login/.well-known","ms":355,"path":"/.well-known/agent-card.json","html":true,"json":null,"present":false}},"rings":{"v":1,"candidates":[{"url":"https://mcp.sentry.dev/mcp","source":"mcp_card"},{"url":"https://sentry.io","source":"mcp_card"},{"url":"https://mcp.sentry.dev","source":"agents_md"},{"url":"https://mcp.sentry.dev/","source":"homepage_link"}],"mcp":[{"url":"https://mcp.sentry.dev/mcp","source":"mcp_card","outcome":"auth_required","status":401,"ms":36,"initialized":false,"protocol_version":null,"server":null,"tools":null,"tools_sha256":null,"auth_header":true},{"url":"https://sentry.io","source":"mcp_card","outcome":"auth_required","status":403,"ms":94,"initialized":false,"protocol_version":null,"server":null,"tools":null,"tools_sha256":null,"auth_header":false},{"url":"https://mcp.sentry.dev","source":"agents_md","outcome":"http_error","status":405,"ms":31,"initialized":false,"protocol_version":null,"server":null,"tools":null,"tools_sha256":null,"auth_header":false}],"a2a":null,"webmcp":{"model_context":false,"bridge":false,"mcp_json":false,"link_rel":false,"any":false},"handshakes":3,"get_path":{"status":200,"content_type":"text/html","ms":285,"error":null},"get_host":{"status":200,"content_type":"text/html","ms":668,"error":null}},"sf":[-1,0,0,0,0,0,1,0,0],"derived":{"reachable":true,"home_class":"answers","robots_served":true,"robots_sitemap":true,"content_signal":true,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":[],"blocks_any_ai":false,"files":{"llms_txt":true,"agents_md":true,"agent_skills":true,"mcp_card":true,"api_catalog":true,"a2a_card":false},"any_capability":true,"ai_refused":[],"ai_thin":["gptbot","claudebot","oaisearch","perplexity"],"aipref":0,"aipref_measured":true,"sf":[-1,0,0,0,0,0,1,0,0],"rings":{"mcp_declared":true,"mcp_live":false,"mcp_auth":true,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":3,"fh":"sentry.io","fp_other_host":[],"fp":"4c20af501f70cf77","fp_bits":{}},"derive_v":5,"frame":"top1k","shard":4},{"v":1,"kind":"crawlcheck-census-record","run":43,"domain":"a2z.com","rank":94,"measured_at":"2026-11-13T08:07:18.369Z","home":{"browser":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"a2z.com","final_path":"/","ms":38,"identity":"Mobile browser","html":false},"gptbot":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"a2z.com","final_path":"/","ms":37,"identity":"GPTBot","html":false},"claudebot":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"a2z.com","final_path":"/","ms":37,"identity":"ClaudeBot","html":false},"oaisearch":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"a2z.com","final_path":"/","ms":35,"identity":"OAI-SearchBot","html":false},"perplexity":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"a2z.com","final_path":"/","ms":34,"identity":"PerplexityBot","html":false}},"robots":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"a2z.com","final_path":"/robots.txt","ms":13,"html":false,"text":null,"agents":null},"sitemap":null,"files":null,"derived":{"reachable":false,"home_class":"no_site","robots_served":false,"robots_sitemap":false,"content_signal":false,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":[],"blocks_any_ai":false,"files":{"llms_txt":false,"agents_md":false,"agent_skills":false,"mcp_card":false,"api_catalog":false,"a2a_card":false},"any_capability":false,"ai_refused":[],"ai_thin":[],"rings":{"mcp_declared":false,"mcp_live":false,"mcp_auth":false,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":3,"fh":"a2z.com","fp_other_host":[]},"derive_v":5,"frame":"top1k","shard":5},{"v":1,"kind":"crawlcheck-census-record","run":43,"domain":"nginx.com","rank":95,"measured_at":"2026-11-13T08:07:47.278Z","home":{"browser":{"status":200,"content_type":"text/html","bytes":434951,"sha256":"58194062b9acb2f809b88a7cd92756e5e109396f3149a0ab10cf8c4239c57058","final_host":"www.f5.com","final_path":"/products/nginx","ms":1406,"identity":"Mobile browser","html":true,"fp":"31793d5e224b6812"},"gptbot":{"status":200,"content_type":"text/html","bytes":434994,"sha256":"961b6ca716a8fcda07341f9c0fbd92c49b96ec15e7037f59cc7558222ed8a8ae","final_host":"www.f5.com","final_path":"/products/nginx","ms":848,"identity":"GPTBot","html":true,"fp":"31793d5e224b6812"},"claudebot":{"status":200,"content_type":"text/html","bytes":434994,"sha256":"961b6ca716a8fcda07341f9c0fbd92c49b96ec15e7037f59cc7558222ed8a8ae","final_host":"www.f5.com","final_path":"/products/nginx","ms":667,"identity":"ClaudeBot","html":true,"fp":"31793d5e224b6812"},"oaisearch":{"status":200,"content_type":"text/html","bytes":434994,"sha256":"961b6ca716a8fcda07341f9c0fbd92c49b96ec15e7037f59cc7558222ed8a8ae","final_host":"www.f5.com","final_path":"/products/nginx","ms":647,"identity":"OAI-SearchBot","html":true,"fp":"31793d5e224b6812"},"perplexity":{"status":200,"content_type":"text/html","bytes":434994,"sha256":"961b6ca716a8fcda07341f9c0fbd92c49b96ec15e7037f59cc7558222ed8a8ae","final_host":"www.f5.com","final_path":"/products/nginx","ms":631,"identity":"PerplexityBot","html":true,"fp":"31793d5e224b6812"}},"robots":{"status":200,"content_type":"text/html","bytes":434994,"sha256":"961b6ca716a8fcda07341f9c0fbd92c49b96ec15e7037f59cc7558222ed8a8ae","final_host":"www.f5.com","final_path":"/products/nginx","ms":628,"html":true,"text":null,"agents":null},"sitemap":{"status":200,"content_type":"text/html","bytes":434951,"sha256":"6646d5097acccfc854f2b3bd9f0962d444b5dcc602dccaadac42810ffca8ef36","final_host":"www.f5.com","final_path":"/products/nginx","ms":395,"url":"https://nginx.com/sitemap.xml","declared":false,"xml":false},"files":{"llms_txt":{"status":200,"content_type":"text/html","bytes":434951,"sha256":"6646d5097acccfc854f2b3bd9f0962d444b5dcc602dccaadac42810ffca8ef36","final_host":"www.f5.com","final_path":"/products/nginx","ms":321,"path":"/llms.txt","html":true,"json":null,"present":false},"agents_md":{"status":200,"content_type":"text/html","bytes":434951,"sha256":"6646d5097acccfc854f2b3bd9f0962d444b5dcc602dccaadac42810ffca8ef36","final_host":"www.f5.com","final_path":"/products/nginx","ms":272,"path":"/agents.md","html":true,"json":null,"present":false},"agent_skills":{"status":200,"content_type":"text/html","bytes":434951,"sha256":"6646d5097acccfc854f2b3bd9f0962d444b5dcc602dccaadac42810ffca8ef36","final_host":"www.f5.com","final_path":"/products/nginx","ms":310,"path":"/.well-known/agent-skills/index.json","html":true,"json":null,"present":false},"mcp_card":{"status":200,"content_type":"text/html","bytes":434951,"sha256":"6646d5097acccfc854f2b3bd9f0962d444b5dcc602dccaadac42810ffca8ef36","final_host":"www.f5.com","final_path":"/products/nginx","ms":308,"path":"/.well-known/mcp/server-card.json","html":true,"json":null,"present":false},"api_catalog":{"status":200,"content_type":"text/html","bytes":434951,"sha256":"6646d5097acccfc854f2b3bd9f0962d444b5dcc602dccaadac42810ffca8ef36","final_host":"www.f5.com","final_path":"/products/nginx","ms":224,"path":"/.well-known/api-catalog","html":true,"json":null,"present":false},"a2a_card":{"status":200,"content_type":"text/html","bytes":434951,"sha256":"6646d5097acccfc854f2b3bd9f0962d444b5dcc602dccaadac42810ffca8ef36","final_host":"www.f5.com","final_path":"/products/nginx","ms":213,"path":"/.well-known/agent-card.json","html":true,"json":null,"present":false}},"rings":{"v":1,"candidates":[],"mcp":[],"a2a":null,"webmcp":{"model_context":false,"bridge":false,"mcp_json":false,"link_rel":false,"any":false},"handshakes":0,"get_path":{"status":200,"content_type":"text/html","ms":299,"error":null},"get_host":{"status":530,"content_type":"text/plain","ms":200,"error":null}},"sf":[1,0,0,0,0,0,0,0,0],"derived":{"reachable":true,"home_class":"answers","robots_served":false,"robots_sitemap":false,"content_signal":false,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":[],"blocks_any_ai":false,"files":{"llms_txt":false,"agents_md":false,"agent_skills":false,"mcp_card":false,"api_catalog":false,"a2a_card":false},"any_capability":false,"ai_refused":[],"ai_thin":[],"sf":[1,0,0,0,0,0,0,0,0],"rings":{"mcp_declared":false,"mcp_live":false,"mcp_auth":false,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":3,"fh":"www.f5.com","fp_other_host":[],"fp":"31793d5e224b6812","fp_bits":{"gptbot":0,"claudebot":0,"oaisearch":0,"perplexity":0}},"derive_v":5,"frame":"top1k","shard":6},{"v":1,"kind":"crawlcheck-census-record","run":43,"domain":"ytimg.com","rank":96,"measured_at":"2026-11-13T08:09:13.777Z","home":{"browser":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"ytimg.com","final_path":"/","ms":84,"identity":"Mobile browser","html":false},"gptbot":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"ytimg.com","final_path":"/","ms":83,"identity":"GPTBot","html":false},"claudebot":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"ytimg.com","final_path":"/","ms":83,"identity":"ClaudeBot","html":false},"oaisearch":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"ytimg.com","final_path":"/","ms":83,"identity":"OAI-SearchBot","html":false},"perplexity":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"ytimg.com","final_path":"/","ms":83,"identity":"PerplexityBot","html":false}},"robots":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"ytimg.com","final_path":"/robots.txt","ms":83,"html":false,"text":null,"agents":null},"sitemap":null,"files":null,"derived":{"reachable":false,"home_class":"no_site","robots_served":false,"robots_sitemap":false,"content_signal":false,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":[],"blocks_any_ai":false,"files":{"llms_txt":false,"agents_md":false,"agent_skills":false,"mcp_card":false,"api_catalog":false,"a2a_card":false},"any_capability":false,"ai_refused":[],"ai_thin":[],"rings":{"mcp_declared":false,"mcp_live":false,"mcp_auth":false,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":3,"fh":"ytimg.com","fp_other_host":[]},"derive_v":5,"frame":"top1k","shard":7},{"v":1,"kind":"crawlcheck-census-record","run":43,"domain":"mozilla.org","rank":97,"measured_at":"2026-11-13T07:57:14.313Z","home":{"browser":{"status":200,"content_type":"text/html","bytes":47880,"sha256":"27baafc3e78d109eb7dc35740fec98d8398593b1c05504c7e6b90e54a7622a90","final_host":"www.mozilla.org","final_path":"/","ms":472,"identity":"Mobile browser","html":true,"fp":"4b53d53d5d2e04c8"},"gptbot":{"status":200,"content_type":"text/html","bytes":47880,"sha256":"27baafc3e78d109eb7dc35740fec98d8398593b1c05504c7e6b90e54a7622a90","final_host":"www.mozilla.org","final_path":"/","ms":498,"identity":"GPTBot","html":true,"fp":"4b53d53d5d2e04c8"},"claudebot":{"status":200,"content_type":"text/html","bytes":47880,"sha256":"27baafc3e78d109eb7dc35740fec98d8398593b1c05504c7e6b90e54a7622a90","final_host":"www.mozilla.org","final_path":"/","ms":473,"identity":"ClaudeBot","html":true,"fp":"4b53d53d5d2e04c8"},"oaisearch":{"status":200,"content_type":"text/html","bytes":47880,"sha256":"27baafc3e78d109eb7dc35740fec98d8398593b1c05504c7e6b90e54a7622a90","final_host":"www.mozilla.org","final_path":"/","ms":497,"identity":"OAI-SearchBot","html":true,"fp":"4b53d53d5d2e04c8"},"perplexity":{"status":200,"content_type":"text/html","bytes":47880,"sha256":"27baafc3e78d109eb7dc35740fec98d8398593b1c05504c7e6b90e54a7622a90","final_host":"www.mozilla.org","final_path":"/","ms":472,"identity":"PerplexityBot","html":true,"fp":"4b53d53d5d2e04c8"}},"robots":{"status":200,"content_type":"text/plain","bytes":66,"sha256":"cf0fd2842659297b0d6fce407bed86bf424ae6666051b881dbd72e871d687fe3","final_host":"www.mozilla.org","final_path":"/robots.txt","ms":446,"html":false,"text":"user-agent: *\ndisallow: /*/newsletter/existing/\ndisallow: /*/etc/\n","agents":{"GPTBot":true,"OAI-SearchBot":true,"ChatGPT-User":true,"ClaudeBot":true,"Claude-SearchBot":true,"PerplexityBot":true,"Google-Extended":true,"Applebot-Extended":true,"CCBot":true,"Bytespider":true,"Meta-ExternalAgent":true,"Amazonbot":true}},"sitemap":{"status":404,"content_type":"text/html","bytes":9034,"sha256":"34530cfeb2f17787ac14129faf86a81161755506d5c9f0fdde95951685b14873","final_host":"www.mozilla.org","final_path":"/en-us/sitemap.xml","ms":328,"url":"https://mozilla.org/sitemap.xml","declared":false,"xml":false},"files":{"llms_txt":{"status":404,"content_type":"text/html","bytes":9031,"sha256":"b664c2da3df8de134c59006f6f210c5d9c68181bdb319d140e024b60f9b7e62c","final_host":"www.mozilla.org","final_path":"/en-us/llms.txt","ms":747,"path":"/llms.txt","html":true,"json":null,"present":false},"agents_md":{"status":404,"content_type":"text/html","bytes":9032,"sha256":"8713d9966f48e4fb34fdcc7824a2af3bdffe30b67176516fc018382523f226c5","final_host":"www.mozilla.org","final_path":"/en-us/agents.md","ms":369,"path":"/agents.md","html":true,"json":null,"present":false},"agent_skills":{"status":404,"content_type":"text/html","bytes":9058,"sha256":"c622575313f5a7ac6be048cd0902398ea125252e953761891519f29271d86932","final_host":"www.mozilla.org","final_path":"/.well-known/agent-skills/index.json","ms":278,"path":"/.well-known/agent-skills/index.json","html":true,"json":null,"present":false},"mcp_card":{"status":404,"content_type":"text/html","bytes":9055,"sha256":"3c0fe4f824b499593dae68e10f61e71263cc4d4245ffdcb41c9175c2f043ea2b","final_host":"www.mozilla.org","final_path":"/.well-known/mcp/server-card.json","ms":291,"path":"/.well-known/mcp/server-card.json","html":true,"json":null,"present":false},"api_catalog":{"status":404,"content_type":"text/html","bytes":9046,"sha256":"0a7f58fbb523626fe51b95ffdcfc4f9600d7e06b445015a97df0545a81c7eb8d","final_host":"www.mozilla.org","final_path":"/.well-known/api-catalog","ms":302,"path":"/.well-known/api-catalog","html":true,"json":null,"present":false},"a2a_card":{"status":404,"content_type":"text/html","bytes":9050,"sha256":"cd0277893a71f081ca60bb73396ab16456c37c59d4a686bc8d43b0dea3dca326","final_host":"www.mozilla.org","final_path":"/.well-known/agent-card.json","ms":269,"path":"/.well-known/agent-card.json","html":true,"json":null,"present":false}},"rings":{"v":1,"candidates":[],"mcp":[],"a2a":null,"webmcp":{"model_context":false,"bridge":false,"mcp_json":false,"link_rel":false,"any":false},"handshakes":0,"get_path":{"status":404,"content_type":"text/html","ms":478,"error":null},"get_host":{"status":530,"content_type":"text/plain","ms":238,"error":null}},"sf":[-1,0,0,0,0,0,0,0,0],"derived":{"reachable":true,"home_class":"answers","robots_served":true,"robots_sitemap":false,"content_signal":false,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":[],"blocks_any_ai":false,"files":{"llms_txt":false,"agents_md":false,"agent_skills":false,"mcp_card":false,"api_catalog":false,"a2a_card":false},"any_capability":false,"ai_refused":[],"ai_thin":[],"aipref":0,"aipref_measured":true,"sf":[-1,0,0,0,0,0,0,0,0],"rings":{"mcp_declared":false,"mcp_live":false,"mcp_auth":false,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":3,"fh":"www.mozilla.org","fp_other_host":[],"fp":"4b53d53d5d2e04c8","fp_bits":{"gptbot":0,"claudebot":0,"oaisearch":0,"perplexity":0}},"derive_v":5,"frame":"top1k","shard":0},{"v":1,"kind":"crawlcheck-census-record","run":43,"domain":"highwebmedia.com","rank":98,"measured_at":"2026-11-13T07:58:36.062Z","home":{"browser":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"highwebmedia.com","final_path":"/","ms":6,"identity":"Mobile browser","html":false},"gptbot":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"highwebmedia.com","final_path":"/","ms":4,"identity":"GPTBot","html":false},"claudebot":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"highwebmedia.com","final_path":"/","ms":3,"identity":"ClaudeBot","html":false},"oaisearch":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"highwebmedia.com","final_path":"/","ms":0,"identity":"OAI-SearchBot","html":false},"perplexity":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"highwebmedia.com","final_path":"/","ms":0,"identity":"PerplexityBot","html":false}},"robots":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"highwebmedia.com","final_path":"/robots.txt","ms":0,"html":false,"text":null,"agents":null},"sitemap":null,"files":null,"derived":{"reachable":false,"home_class":"no_site","robots_served":false,"robots_sitemap":false,"content_signal":false,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":[],"blocks_any_ai":false,"files":{"llms_txt":false,"agents_md":false,"agent_skills":false,"mcp_card":false,"api_catalog":false,"a2a_card":false},"any_capability":false,"ai_refused":[],"ai_thin":[],"rings":{"mcp_declared":false,"mcp_live":false,"mcp_auth":false,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":3,"fh":"highwebmedia.com","fp_other_host":[]},"derive_v":5,"frame":"top1k","shard":1},{"v":1,"kind":"crawlcheck-census-record","run":43,"domain":"yandex.ru","rank":99,"measured_at":"2026-11-13T07:59:31.658Z","home":{"browser":{"status":200,"content_type":"text/html","bytes":3061,"sha256":"7496305ccd1025e5d7c38303bd1976571743615cd09500d481fbf6807ce7114b","final_host":"sso.passport.yandex.ru","final_path":"/push","ms":2824,"identity":"Mobile browser","html":true},"gptbot":{"status":200,"content_type":"text/html","bytes":2900,"sha256":"bcb935253647c909957b1e563bf1b3156f8646a4198cad79879a36a4ea8970fe","final_host":"sso.passport.yandex.ru","final_path":"/push","ms":2795,"identity":"GPTBot","html":true},"claudebot":{"status":200,"content_type":"text/html","bytes":2901,"sha256":"5ab44a0a1638789528c0bcdc109cda644b47ebb4cdc6b1031cbee0ad66f371ef","final_host":"sso.passport.yandex.ru","final_path":"/push","ms":2814,"identity":"ClaudeBot","html":true},"oaisearch":{"status":200,"content_type":"text/html","bytes":2901,"sha256":"3f4e292eedab93690967d74946bef854e0756cef6add5a40566c952d224eb5b4","final_host":"sso.passport.yandex.ru","final_path":"/push","ms":2727,"identity":"OAI-SearchBot","html":true},"perplexity":{"status":200,"content_type":"text/html","bytes":2898,"sha256":"4da22481bdd974e21931b95f9d00426e1f1425c4e9f0bf99937d8fe9dbe8d788","final_host":"sso.passport.yandex.ru","final_path":"/push","ms":2290,"identity":"PerplexityBot","html":true}},"robots":{"status":200,"content_type":"text/plain","bytes":40657,"sha256":"c51a208183d07eb761c7dea99d104021b60c0840f91d43194f016aa5a44a3f46","final_host":"yandex.ru","final_path":"/robots.txt","ms":822,"html":false,"text":"# yandex.ru\nUser-agent: *\nDisallow: /?\nDisallow: /403.html\nDisallow: /404.html\nDisallow: /500.html\nDisallow: /adddata\nDisallow: /adresa-segmentator\nDisallow: /advanced_engl.html\nDisallow: /advertising\nDisallow: /alice/term/\nDisallow: /alice/*/term/\nDisallow: /alice/old-browser\nDisallow: /alice/store/orders/\nDisallow: /alice*?\nDisallow: /alice/*&\nDisallow: /alice/chat/\nDisallow: /alice?share=\nDisallow: /all-supported-params\nDisallow: /articles\nDisallow: /aura/\nAllow: /aura/promo/\nDisallow: /blog/*?text=\nDisallow: /blog/*?tag=\nDisallow: /blog/*/*/*\nDisallow: /blog/punto\nDisallow: /MAPS/\nDisallow: /MaPS/\nDisallow: /MapS/\nDisallow: /Maps/\nDisallow: /mApS/\nDisallow: /harita/business/\nDisallow: /maps/business/\nDisallow: /maps-api/docs/js-api/reference-guide/instruction-gxs.html\nDisallow: /maps-api/docs/js-api/reference-guide/instruction-rs.html\nDisallow: /catalog/?text=\nDisallow: /chat\nDisallow: /cgi-bin/\nDisallow: /cgi/\nDisallow: /clck\nDisallow: /collections/\nDisallow: /company/*.rss\nDisallow: /company/search\nDisallow: /comments/*\nDisallow: /courier\nDisallow: /conflagexp\nDisallow: /cy\nDisallow: /cycounter\nDisallow: /dzen\nDisallow: /experiments.xml\nDisallow: /formfeedback\nDisallow: /fresh\nDisallow: /goto_issue/\nDisallow: /goto_rubric/\nDisallow: /gundem\nDisallow: /i/\nDisallow: /images-data\nDisallow: /images.html\nDisallow: /images/*\nDisallow: /images-apphost/*\nAllow: /images/$\nAllow: /images/smart/$\nAllow: /images/touch/$\nAllow: /images/touch/?\nAllow: /images/search?text=*&source=morda*\nAllow: /images/search?rpt=imageview\nAllow: /images/touch/search?text=*&source=morda*\nAllow: /images/touch/search?rpt=imageview\nAllow: /images/pad/search?text=*&source=morda*\nAllow: /images/pad/search?rpt=imageview\nDisallow: /index_m\nDisallow: /iznanka/\nDisallow: /int_search_summary\nDisallow: /delivery_tracking\nDisallow: /jobs/skill-diagnostic/private/*\nDisallow: /keyboard_qwerty.html\nDisallow: /logotypes\nDisallow: /business/widget\nDisallow: /local/\nDisallow: /lab/neurodetector\nDisallow: /lab/habr-detector\nDisallow: /more_samples\nDisallow: /msearch\nDisallow: /map-constructor/loader*\nDisallow: /maps/print/*\nAllow: /maps/*?lang=ru$\nAllow: /maps/*?lang=uk$\nAllow: /maps/*?lang=kk$\nAllow: /maps/*?lang=uz$\nDisallow: /nmaps/*?\nDisallow: /mapeditor/*?\nDisallow: /metro/*?from\nDisallow: /masters\nDisallow: /services/executor/\nDisallow: /n?\nDisallow: /n$\nDisallow: /n/\nDisallow: /neuralsearch/api\nDisallow: /norobot\nDisallow: /opensearch.xml\nDisallow: /padsearch\nDisallow: /person\nDisallow: /podpiska/login.pl\nDisallow: /pogoda/search\nDisallow: /pogoda/404\nDisallow: /pogoda/0\nDisallow: /polling\nDisallow: /poll/\nDisallow: /portal/set/\nDisallow: /portal/tune/\nDisallow: /prefetch\nDisallow: /promo/*goodbye*\nDisallow: /promo/diskelement\nDisallow: /promo/launcher/mgoodbye/*\nDisallow: /promo/*welcome*\nDisallow: /promo/yobject/changelog*\nDisallow: /project/eda/partnership_services/$\nDisallow: /q/profile/*/answers\nDisallow: /q/profile/*/questions\nDisallow: /q/profile/*/verifies\nDisallow: /q/profile/*/articles\nDisallow: /q/profile/*/subscribers\nDisallow: /q/profile/*/org_answers\nDisallow: /q/user/*/answers\nDisallow: /q/user/*/questions\nDisallow: /q/user/*/verifies\nDisallow: /q/user/*/articles\nDisallow: /q/user/*/subscribers\nDisallow: /q/user/*/org_answers\nDisallow: /soft/bm/goodbye\nDisallow: /soft/chrome/ext-install/*\nDisallow: /soft/chrome/searchline-install/*\nDisallow: /soft/*goodbye*\nDisallow: /soft/punto/mac/uninstall/*\nDisallow: /soft/punto/win/uninstall/*\nDisallow: /soft/*welcome*\nDisallow: /soft/win/?*\nDisallow: /quotes\nDisallow: /realty*?*pinned_offer_id=\nDisallow: /realty*?*pinned_nb_id=\nDisallow: /realty/search\nDisallow: /realty*?*text=\nDisallow: /realty/result\nDisallow: /realty/map_pins\nDisallow: /realty/map_dust\nDisallow: /realty/callme\nDisallow: /redir\nDisallow: /region_map\nDisallow: /regions_list.xml\nDisallow: /regions.html?\nDisallow: /s/\nDisallow: /save\nDisallow: /safety/?\nDisallow: /safety?\nDisallow: /search\nAllow: /search/entity/touch/\nDisallow: /setup\nDisallow: /showcaptcha\nDisallow: /sitesearch\nDisallow: /sl/*.html\nDisallow: /soft/extensions/goodbye\nDisallow: /sportagent\nDisallow: /storeclick\nDisallow: /storerequest\nDisallow: /subscribe/confirm.pl\nDisallow: /subscribe/view.pl\nAllow: /support/\nDisallow: /support/direct-images\nDisallow: /support/direct-tooltips\nDisallow: /support/distr\nDisallow: /support/dsp\nDisallow: /support/fe4be44a295cc679e19bf0b8f133083d\nDisallow: /support/maps-beta\nDisallow: /support/market-images\nDisallow: /support/security\nDisallow: /support/webmaster-images\nDisallow: /support/*zout_\nDisallow: /support/search-results/\nDisallow: /support/praktikum/flow.html\nDisallow: /support/praktikum-english/\nDisallow: /telsearch\nDisallow: /themes\nDisallow: /toggle-experiment\nDisallow: /touchsearch\nDisallow: /tune*retpath=\nDisallow: /turbo\nAllow: /ugcpub/\nDisallow: /ugcpub/cabinet\nDisallow: /v$\nDisallow: /viewconfig$\nDisallow: /images/v$\nDisallow: /images/viewconfig$\nAllow: /tutor\nDisallow: /tutor/admin\nDisallow: /tutor/search/problems\nDisallow: /versions\nDisallow: /video/*\nAllow: /video/$\nAllow: /video/?\nAllow: /video/sitemap\nAllow: /video/movies/entity\nAllow: /video/touch/movies/entity\nAllow: /video/touch/$\nAllow: /video/touch/?\nAllow: /video/preview\nAllow: /video/touch/preview\nAllow: /video/search?text=video$\nAllow: /video/touch/search?text=video$\nAllow: /video/search?text=%D0%B2%D0%B8%D0%B4%D0%B5%D0%BE$\nAllow: /video/touch/search?text=%D0%B2%D0%B8%D0%B4%D0%B5%D0%BE$\nAllow: /video/video_sitemap_*.xml\nAllow: /video/movies_sitemap_*.xml\nDisallow: /xmlsearch\nDisallow: /yandsearch\nDisallow: /yca/cy\nDisallow: /soft/distribution\nDisallow: /soft/?*\nDisallow: /promo/launcher/?*\nDisallow: /opera/?*\nDisallow: /firefox/?*\nDisallow: /soft/bm/?*\nDisallow: /soft/browsers/?*\nDisallow: /soft/punto/?*\nDisallow: /ie/?*\nDisallow: /soft/*goodbye*\nDisallow: /adult\nDisallow: /addappterless\nDisallow: /archive/api/image.jpg*\nAllow: /archive$\nAllow: /archive/project/sovsport$\nDisallow: /all?\nDisallow: /games/*?*reqId*\nDisallow: /games/*?*pageId*\nDisallow: /games/*?experimental=\nDisallow: /games/*?*app-id=\nDisallow: /games/*?*game-id=\nDisallow: /games/*?*div-rtx-reqid=\nDisallow: /games/*?*header=\nDisallow: /games/*?*from=\nDisallow: /games/*?*clid=\nDisallow: /games/*?*deviceid=\nDisallow: /games/*?*flags=\nDisallow: /games/*?*inapp_id=\nDisallow: /games/*?*k50id=\nDisallow: /games/*?*lang=\nDisallow: /games/*?*payload=\nDisallow: /games/*?*pfid=\nDisallow: /games/*?*promo_id=\nDisallow: /games/*?*promo_intent=\nDisallow: /games/*?*referrer=\nDisallow: /games/*?*utm_\nDisallow: /games/*?*ypr=\nDisallow: /familysearch\nDisallow: /schoolsearch\nDisallow: /yandpage\nDisallow: *maps/covid19*\nDisallow: /promo/market/mobile/app*?from=\nDisallow: /support2/*zout_\nDisallow: /activity/\nDisallow: /sport\nDisallow: /support2/sport\n\nAllow: /medicine\nDisallow: /medicine/search\nDisallow: /medicine?text=\nDisallow: /finance\nAllow: /finance/reviews\nAllow: /finance/banks/*/reviews\nDisallow: /finance/reviews*?*\nDisallow: /finance/banks/*/reviews*?\n\nSitemap: https://yandex.ru/support/sitemap.xml\nSitemap: https://yandex.ru/blog/sitemap.xml\nSitemap: https://frontend.s3.yandex.net/health/sitemap.xml\nSitemap: https://yandex.ru/uslugi/sitemap/sitemap.xml.gz\nSitemap: https://yandex.ru/turbo/public/sitemap.xml\nSitemap: https://yandex.ru/tutor/sitemap/?sitemap=sitemap.xml\nSitemap: https://yandex.ru/turbo/talk-sitemap-index.xml\nSitemap: https://yandex.ru/games/sitemaps/sitemap.index.xml\nSitemap: https://yandex.ru/q/sitemap/sitemap-index.xml\nSitemap: https://yandex.ru/images/sitemap.xml\nSitemap: https://yandex.ru/medicine/sitemaps/sitemap.xml\nSitemap: https://yandex.ru/finance/sitemaps/sitemap.xml\nSitemap: https://yandex.ru/yandexapp/ru/callerid/sitemap.xml\nSitemap: https://yandex.ru/video/sitemap_index.xml\n\nUser-agent: Bingbot\nDisallow: /?\nDisallow: /403.html\nDisallow: /404.html\nDisallow: /500.html\nDisallow: /adddata\nDisallow: /adresa-segmentator\nDisallow: /advanced_engl.html\nDisallow: /advertising\nDisallow: /alice/term/\nDisallow: /alice/*/term/\nDisallow: /alice/old-browser\nDisallow: /alice/store/orders/\nDisallow: /alice*?\nDisallow: /alice/*&\nDisallow: /alice/chat/\nDisallow: /alice?share=\nDisallow: /all-supported-params\nDisallow: /articles\nDisallow: /aura/\nAllow: /aura/promo/\nDisallow: /blog/*?text=\nDisallow: /blog/*?tag=\nDisallow: /blog/*/*/*\nDisallow: /blog/punto\nDisallow: /MAPS/\nDisallow: /MaPS/\nDisallow: /MapS/\nDisallow: /Maps/\nDisallow: /mApS/\nDisallow: /maps-api/docs/js-api/reference-guide/instruction-gxs.html\nDisallow: /maps-api/docs/js-api/reference-guide/instruction-rs.html\nDisallow: /catalog/?text=\nDisallow: /chat\nDisallow: /cgi-bin/\nDisallow: /cgi/\nDisallow: /clck\nDisallow: /collections/\nDisallow: /company/*.rss\nDisallow: /company/search\nDisallow: /comments/*\nDisallow: /courier\nDisallow: /conflagexp\nDisallow: /cy\nDisallow: /cycounter\nDisallow: /dzen\nDisallow: /experiments.xml\nDisallow: /formfeedback\nDisallow: /fresh\nDisallow: /goto_issue/\nDisallow: /goto_rubric/\nDisallow: /i/\nDisallow: /images-data\nDisallow: /images.html\nDisallow: /images/*\nDisallow: /images-apphost/*\nAllow: /images/$\nAllow: /images/smart/$\nAllow: /images/touch/$\nAllow: /images/touch/?\nAllow: /images/search?text=*&source=morda*\nAllow: /images/search?rpt=imageview\nAllow: /images/touch/search?text=*&source=morda*\nAllow: /images/touch/search?rpt=imageview\nAllow: /images/pad/search?text=*&source=morda*\nAllow: /images/pad/search?rpt=imageview\nDisallow: /index_m\nDisallow: /iznanka/\nDisallow: /int_search_summary\nDisallow: /delivery_tracking\nDisallow: /gundem\nDisallow: /jobs/skill-diagnostic/private/*\nDisallow: /keyboard_qwerty.html\nDisallow: /logotypes\nDisallow: /business/widget\nDisallow: /local/\nDisallow: /lab/neurodetector\nDisallow: /lab/habr-detector\nDisallow: /more_samples\nDisallow: /msearch\nDisallow: /map-constructor/loader*\nDisallow: /maps/print/*\nAllow: /maps/*?lang=ru$\nAllow: /maps/*?lang=uk$\nAllow: /maps/*?lang=kk$\nAllow: /maps/*?lang=uz$\nDisallow: /nmaps/*?\nDisallow: /mapeditor/*?\nDisallow: /metro/*?from\nDisallow: /masters\nDisallow: /services/executor/\nDisallow: /n?\nDisallow: /n$\nDisallow: /n/\nDisallow: /norobot\nDisallow: /opensearch.xml\nDisallow: /padsearch\nDisallow: /person\nDisallow: /podpiska/login.pl\nDisallow: /pogoda/search\nDisallow: /pogoda/404\nDisallow: /polling\nDisallow: /poll/\nDisallow: /portal/set/\nDisallow: /portal/tune/\nDisallow: /prefetch\nDisallow: /promo/*goodbye*\nDisallow: /promo/diskelement\nDisallow: /promo/launcher/mgoodbye/*\nDisallow: /promo/*welcome*\nDisallow: /promo/yobject/changelog*\nDisallow: /project/eda/partnership_services/$\nDisallow: /q/profile/*/answers\nDisallow: /q/profile/*/questions\nDisallow: /q/profile/*/verifies\nDisallow: /q/profile/*/articles\nDisallow: /q/profile/*/subscribers\nDisallow: /q/profile/*/org_answers\nDisallow: /q/user/*/answers\nDisallow: /q/user/*/questions\nDisallow: /q/user/*/verifies\nDisallow: /q/user/*/articles\nDisallow: /q/user/*/subscribers\nDisallow: /q/user/*/org_answers\nDisallow: /soft/bm/goodbye\nDisallow: /soft/chrome/ext-install/*\nDisallow: /soft/chrome/searchline-install/*\nDisallow: /soft/*goodbye*\nDisallow: /soft/punto/mac/uninstall/*\nDisallow: /soft/punto/win/uninstall/*\nDisallow: /soft/*welcome*\nDisallow: /soft/win/?*\nDisallow: /quotes\nDisallow: /realty*?*pinned_offer_id=\nDisallow: /realty*?*pinned_nb_id=\nDisallow: /realty/search\nDisallow: /realty*?*text=\nDisallow: /realty/result\nDisallow: /realty/map_pins\nDisallow: /realty/map_dust\nDisallow: /realty/callme\nDisallow: /redir\nDisallow: /region_map\nDisallow: /regions_list.xml\nDisallow: /regions.html?\nDisallow: /s/\nDisallow: /save\nDisallow: /safety/?\nDisallow: /safety?\nDisallow: /search\nAllow: /search/entity/touch/\nAllow: /search?products_mode=1\nAllow: /search/?products_mode=1\nAllow: /search?*&products_mode=1\nAllow: /search/?*&products_mode=1\nAllow: /search/touch?products_mode=1\nAllow: /search/touch/?products_mode=1\nAllow: /search/touch?*&products_mode=1\nAllow: /search/touch/?*&products_mode=1\nAllow: /search/products\nAllow: /products/$\nAllow: /products$\nDisallow: /setup\nDisallow: /showcaptcha\nDisallow: /sitesearch\nDisallow: /sl/*.html\nDisallow: /soft/extensions/goodbye\nDisallow: /sportagent\nDisallow: /storeclick\nDisallow: /storerequest\nDisallow: /subscribe/confirm.pl\nDisallow: /subscribe/view.pl\nAllow: /support/\nDisallow: /support/direct-images\nDisallow: /support/direct-tooltips\nDisallow: /support/distr\nDisallow: /support/dsp\nDisallow: /support/fe4be44a295cc679e19bf0b8f133083d\nDisallow: /support/maps-beta\nDisallow: /support/market-images\nDisallow: /support/security\nDisallow: /support/webmaster-images\nDisallow: /support/*zout_\nDisallow: /support/search-results/\nDisallow: /support/praktikum/flow.html\nDisallow: /support/praktikum-english/\nDisallow: /telsearch\nDisallow: /themes\nDisallow: /toggle-experiment\nDisallow: /touchsearch\nDisallow: /tune*retpath=\nAllow: /ugcpub/\nDisallow: /ugcpub/cabinet\nDisallow: /v$\nDisallow: /viewconfig$\nDisallow: /images/v$\nDisallow: /images/viewconfig$\nAllow: /tutor\nDisallow: /tutor/admin\nDisallow: /tutor/search/problems\nDisallow: /turbo\nDisallow: /versions\nDisallow: /video/*\nAllow: /video/$\nAllow: /video/?\nAllow: /video/sitemap\nAllow: /video/movies/entity\nAllow: /video/touch/movies/entity\nAllow: /video/touch/$\nAllow: /video/touch/?\nAllow: /video/preview\nAllow: /video/touch/preview\nAllow: /video/search?text=video$\nAllow: /video/touch/search?text=video$\nAllow: /video/search?text=%D0%B2%D0%B8%D0%B4%D0%B5%D0%BE$\nAllow: /video/touch/search?text=%D0%B2%D0%B8%D0%B4%D0%B5%D0%BE$\nAllow: /video/video_sitemap_*.xml\nAllow: /video/movies_sitemap_*.xml\nDisallow: /xmlsearch\nDisallow: /yandsearch\nDisallow: /yca/cy\nDisallow: /soft/distribution\nDisallow: /soft/?*\nDisallow: /promo/launcher/?*\nDisallow: /opera/?*\nDisallow: /firefox/?*\nDisallow: /soft/bm/?*\nDisallow: /soft/browsers/?*\nDisallow: /soft/punto/?*\nDisallow: /ie/?*\nDisallow: /soft/*goodbye*\nDisallow: /adult\nDisallow: /addappterless\nDisallow: /archive/api/image.jpg*\nAllow: /archive$\nAllow: /archive/project/sovsport$\nDisallow: /all?\nDisallow: /games/*?*reqId*\nDisallow: /games/*?*pageId*\nDisallow: /games/*?experimental=\nDisallow: /games/*?*app-id=\nDisallow: /games/*?*game-id=\nDisallow: /games/*?*div-rtx-reqid=\nDisallow: /games/*?*header=\nDisallow: /games/*?*from=\nDisallow: /games/*?*clid=\nDisallow: /games/*?*deviceid=\nDisallow: /games/*?*flags=\nDisallow: /games/*?*inapp_id=\nDisallow: /games/*?*k50id=\nDisallow: /games/*?*lang=\nDisallow: /games/*?*payload=\nDisallow: /games/*?*pfid=\nDisallow: /games/*?*promo_id=\nDisallow: /games/*?*promo_intent=\nDisallow: /games/*?*referrer=\nDisallow: /games/*?*utm_\nDisallow: /games/*?*ypr=\nDisallow: /products\nDisallow: /familysearch\nDisallow: /schoolsearch\nDisallow: /yandpage\nDisallow: *maps/covid19*\nDisallow: /promo/market/mobile/app*?from=\nDisallow: /support2/*zout_\nDisallow: /activity/\nDisallow: /sport\nDisallow: /support2/sport*\n\nAllow: /medicine\nDisallow: /medicine/search\nDisallow: /medicine?text=\nDisallow: /finance*?\nDisallow: /finance/assistant\n\nUser-agent: Yandex\nDisallow: /?\nDisallow: /403.html\nDisallow: /404.html\nDisallow: /500.html\nDisallow: /adddata\nDisallow: /adresa-segmentator\nDisallow: /advanced_engl.html\nDisallow: /advertising\nDisallow: /adv/news/vystavka-prodazha-na-poiske-poyavilas-tovarnaya-galereya\nDisallow: /adv?nw=\nDisallow: /adv?from=\nDisallow: /adv?email_form=\nDisallow: /adv?ref=\nDisallow: /alice/term/\nDisallow: /alice/*/term/\nDisallow: /alice/old-browser\nDisallow: /alice/chat/\nDisallow: /alice?share=\nDisallow: /alice*?\nDisallow: /all-supported-params\nDisallow: /articles\nDisallow: /aura/\nAllow: /aura/promo/\nDisallow: /blog/*?text=\nDisallow: /blog/*?tag=\nDisallow: /blog/*/*/*\nDisallow: /blog/punto\nDisallow: /MAPS/\nDisallow: /MaPS/\nDisallow: /MapS/\nDisallow: /Maps/\nDisallow: /mApS/\nDisallow: /catalog/?text=\nDisallow: /chat\nDisallow: /cgi-bin/\nDisallow: /cgi/\nDisallow: /clck\nDisallow: /collections/\nDisallow: /company/*.rss\nDisallow: /company/search\nDisallow: /business/widget\nDisallow: /comments/*\nDisallow: /courier\nDisallow: /products\nDisallow: /conflagexp\nDisallow: /cy\nDisallow: /cycounter\nDisallow: /dzen\nDisallow: /experiments.xml\nDisallow: /formfeedback\nDisallow: /fresh\nDisallow: /games/*?experimental=\nDisallow: /goto_issue/\nDisallow: /goto_rubric/\nDisallow: /i/\nDisallow: /images-data\nDisallow: /images.html\nDisallow: /images/*\nAllow: /images/$\nAllow: /images/smart/$\nAllow: /images/touch/$\nAllow: /images/touch/?\nAllow: /images/search?text=*&source=morda*\nAllow: /images/search?rpt=imageview\nAllow: /images/touch/search?text=*&source=morda*\nAllow: /images/touch/search?rpt=imageview\nAllow: /images/pad/search?text=*&source=morda*\nAllow: /images/pad/search?rpt=imageview\nDisallow: /index_m\nDisallow: /iznanka/\nDisallow: /int_search_summary\nDisallow: /delivery_tracking\nDisallow: /jobs/skill-diagnostic/private/*\nDisallow: /keyboard_qwerty.html\nDisallow: /logotypes\nDisallow: /local/\nDisallow: /lab/neurodetector\nDisallow: /lab/habr-detector\nDisallow: /more_samples\nDisallow: /msearch\nDisallow: /map-constructor/loader*\nDisallow: /maps/print/*\nDisallow: /harita/business/\nDisallow: /maps/business/\nDisallow: /maps-api/docs/js-api/reference-guide/instruction-gxs.html\nDisallow: /maps-api/docs/js-api/reference-guide/instruction-rs.html\nAllow: /maps/*?lang=ru$\nAllow: /maps/*?lang=uk$\nAllow: /maps/org/\nDisallow: /nmaps/*?\nDisallow: /mapeditor/*?\nDisallow: /metro/*?from\nDisallow: /masters\nDisallow: /services\nDisallow: /neuralsearch/api\nDisallow: /n?\nDisallow: /n$\nDisallow: /n/\nDisallow: /norobot\nDisallow: /opensearch.xml\nDisallow: /padsearch\nDisallow: /person\nDisallow: /podpiska/login.pl\nDisallow: /pogoda/amp/*\nDisallow: /pogoda/search\nDisallow: /pogoda/404\nDisallow: /pogoda/0\nDisallow: /polling\nDisallow: /poll/\nDisallow: /portal/set/\nDisallow: /portal/tune/\nDisallow: /promo/*goodbye*\nDisallow: /promo/diskelement\nDisallow: /promo/launcher/mgoodbye/*\nDisallow: /promo/*welcome*\nDisallow: /promo/yobject/changelog*\nDisallow: /project/eda/partnership_services/$\nDisallow: /q/profile/*/answers\nDisallow: /q/profile/*/questions\nDisallow: /q/profile/*/verifies\nDisallow: /q/profile/*/articles\nDisallow: /q/profile/*/subscribers\nDisallow: /q/profile/*/org_answers\nDisallow: /q/user/*/answers\nDisallow: /q/user/*/questions\nDisallow: /q/user/*/verifies\nDisallow: /q/user/*/articles\nDisallow: /q/user/*/subscribers\nDisallow: /q/user/*/org_answers\nDisallow: /soft/bm/goodbye\nDisallow: /soft/chrome/ext-install/*\nDisallow: /soft/chrome/searchline-install/*\nDisallow: /soft/*goodbye*\nDisallow: /soft/punto/mac/uninstall/*\nDisallow: /soft/punto/win/uninstall/*\nDisallow: /soft/*welcome*\nDisallow: /soft/win/?*\nDisallow: /quotes\nDisallow: /realty\nDisallow: /redir\nDisallow: /region_map\nDisallow: /regions_list.xml\nDisallow: /regions.html?\nDisallow: /s/\nDisallow: /save\nDisallow: /safety/?*\nDisallow: /search\nAllow: /search/entity/touch/\nDisallow: /setup\nDisallow: /showcaptcha\nDisallow: /sitesearch\nDisallow: /sl/*.html\nDisallow: /soft/extensions/goodbye\nDisallow: /sportagent\nDisallow: /storeclick\nDisallow: /storerequest\nDisallow: /subscribe/confirm.pl\nDisallow: /subscribe/view.pl\nAllow: /support/\nDisallow: /support/direct-images\nDisallow: /support/direct-tooltips\nDisallow: /support/distr\nDisallow: /support/dsp\nDisallow: /support/fe4be44a295cc679e19bf0b8f133083d\nDisallow: /support/maps-beta\nDisallow: /support/market-images\nDisallow: /support/security\nDisallow: /support/webmaster-images\nDisallow: /support/*zout_\nDisallow: /support/search-results/\nDisallow: /support/praktikum/flow.html\nDisallow: /support/praktikum-english/\nDisallow: /telsearch\nDisallow: /themes\nDisallow: /toggle-experiment\nDisallow: /touchsearch\nDisallow: /tune*retpath=\nDisallow: /8bit-fest/parents\nAllow: /tutor\nDisallow: /tutor/admin\nDisallow: /turbo\nAllow: /ugcpub/\nDisallow: /ugcpub/cabinet\nDisallow: /v$\nDisallow: /viewconfig$\nDisallow: /images/v$\nDisallow: /images/viewconfig$\nDisallow: /versions\nDisallow: /video/*\nAllow: /video/$\nAllow: /video/?\nAllow: /video/sitemap\nAllow: /video/touch/$\nAllow: /video/touch/?\nAllow: /video/preview\nAllow: /video/touch/preview\nAllow: /video/search?text=video$\nAllow: /video/touch/search?text=video$\nAllow: /video/search?text=%D0%B2%D0%B8%D0%B4%D0%B5%D0%BE$\nAllow: /video/touch/search?text=%D0%B2%D0%B8%D0%B4%D0%B5%D0%BE$\nAllow: /video/video_sitemap_*.xml\nAllow: /video/movies_sitemap_*.xml\nDisallow: /xmlsearch\nDisallow: /yandsearch\nDisallow: /yca/cy\nDisallow: /soft/distribution\nDisallow: /soft/?*\nDisallow: /promo/launcher/?*\nDisallow: /opera/?*\nDisallow: /firefox/?*\nDisallow: /soft/bm/?*\nDisallow: /soft/browsers/?*\nDisallow: /soft/punto/?*\nDisallow: /ie/?*\nDisallow: /soft/*goodbye*\nDisallow: /adult\nDisallow: /addappterless\nDisallow: /archive/api/image.jpg*\nAllow: /archive$\nAllow: /archive/project/sovsport$\nDisallow: /all?\nDisallow: /familysearch\nDisallow: /schoolsearch\nDisallow: /yandpage\nDisallow: *maps/covid19*\nDisallow: /promo/market/mobile/app*?from=\nDisallow: /support2/*zout_\nDisallow: /activity/\nDisallow: /sport\nDisallow: /support2/sport*\n\nDisallow: /medicine\nDisallow: /finance\n\nDisallow: /gundem\n\nUser-agent: Googlebot\nDisallow: /?\nDisallow: /403.html\nDisallow: /404.html\nDisallow: /500.html\nDisallow: /adddata\nDisallow: /adresa-segmentator\nDisallow: /advanced_engl.html\nDisallow: /advertising\nDisallow: /ads/\nDisallow: /adfox/\nDisallow: /an/\nDisallow: /alice/term/\nDisallow: /alice/*/term/\nDisallow: /alice/old-browser\nDisallow: /alice/store/orders/\nDisallow: /alice*?\nDisallow: /alice/*&\nDisallow: /alice/chat/\nDisallow: /alice?share=\nDisallow: /all-supported-params\nDisallow: /articles\nDisallow: /archive/search\nDisallow: /aura/\nAllow: /aura/promo/\nDisallow: /blog/*?text=\nDisallow: /blog/*?tag=\nDisallow: /blog/*/*/*\nDisallow: /blog/punto\nDisallow: /business/widget\nDisallow: /MAPS/\nDisallow: /MaPS/\nDisallow: /MapS/\nDisallow: /Maps/\nDisallow: /mApS/\nDisallow: /maps-api/docs/js-api/reference-guide/instruction-gxs.html\nDisallow: /maps-api/docs/js-api/reference-guide/instruction-rs.html\nDisallow: /catalog/?text=\nDisallow: /cgi-bin/\nDisallow: /cgi/\nDisallow: /clck\nDisallow: /collections/\nDisallow: /company/*.rss\nDisallow: /company/search\nDisallow: /comments/*\nDisallow: /courier\nDisallow: /conflagexp\nDisallow: /cy\nDisallow: /cycounter\nDisallow: /dzen\nDisallow: /experiments.xml\nDisallow: /formfeedback\nDisallow: /fresh\nDisallow: /goto_issue/\nDisallow: /goto_rubric/\nDisallow: /gundem\nDisallow: /i/\nDisallow: /images-data\nDisallow: /images.html\nDisallow: /images/\nDisallow: /images-apphost/\nAllow: /images$\nAllow: /images/$\nAllow: /images/?\nAllow: /images/smart/$\nAllow: /images/touch$\nAllow: /images/touch/$\nAllow: /images/touch/?\nAllow: /images/pad/search?text=*&source=morda*\nAllow: /images/pad/search?rpt=imageview\nAllow: /images/search?\nAllow: /images/touch/search?\n\nAllow: /images/sitemap\nDisallow: /index_m\nDisallow: /iznanka/\nDisallow: /int_search_summary\nDisallow: /delivery_tracking\nDisallow: /jobs/skill-diagnostic/private/*\nDisallow: /keyboard_qwerty.html\nDisallow: /logotypes\nDisallow: /local/\nDisallow: /lab/neurodetector\nDisallow: /lab/habr-detector\nDisallow: /more_samples\nDisallow: /msearch\nDisallow: /map-constructor/loader*\nDisallow: /maps/print/*\nDisallow: /harita/business/\nDisallow: /maps/business/\nAllow: /maps/*?lang=ru$\nAllow: /maps/*?lang=uk$\nAllow: /maps/*?lang=kk$\nAllow: /maps/*?lang=uz$\nDisallow: /nmaps/*?\nDisallow: /mapeditor/*?\nDisallow: /metro/*?from\nDisallow: /masters\nDisallow: /services/executor/\nDisallow: /neuralsearch/api\nDisallow: /n?\nDisallow: /n$\nDisallow: /n/\nDisallow: /norobot\nDisallow: /opensearch.xml\nDisallow: /padsearch\nAllow: /products/sitemaps/new/*\nDisallow: /person\nDisallow: /podpiska/login.pl\nDisallow: /pogoda/404\nDisallow: /polling\nDisallow: /poll/\nDisallow: /portal/set/\nDisallow: /portal/tune/\nDisallow: /prefetch\nDisallow: /products\nDisallow: /promo/*goodbye*\nDisallow: /promo/diskelement\nDisallow: /promo/launcher/mgoodbye/*\nDisallow: /promo/*welcome*\nDisallow: /promo/yobject/changelog*\nDisallow: /project/eda/partnership_services/$\nDisallow: /q/profile/*/answers\nDisallow: /q/profile/*/questions\nDisallow: /q/profile/*/verifies\nDisallow: /q/profile/*/articles\nDisallow: /q/profile/*/subscribers\nDisallow: /q/profile/*/org_answers\nDisallow: /q/user/*/answers\nDisallow: /q/user/*/questions\nDisallow: /q/user/*/verifies\nDisallow: /q/user/*/articles\nDisallow: /q/user/*/subscribers\nDisallow: /q/user/*/org_answers\nDisallow: /soft/bm/goodbye\nDisallow: /soft/chrome/ext-install/*\nDisallow: /soft/chrome/searchline-install/*\nDisallow: /soft/*goodbye*\nDisallow: /soft/punto/mac/uninstall/*\nDisallow: /soft/punto/win/uninstall/*\nDisallow: /soft/*welcome*\nDisallow: /soft/win/?*\nDisallow: /quotes\nDisallow: /realty*?*pinned_offer_id=\nDisallow: /realty*?*pinned_nb_id=\nDisallow: /realty/search\nDisallow: /realty*?*text=\nDisallow: /realty/result\nDisallow: /realty/map_pins\nDisallow: /realty/map_dust\nDisallow: /realty/callme\nDisallow: /redir\nDisallow: /region_map\nDisallow: /regions_list.xml\nDisallow: /regions.html?\nDisallow: /s/\nDisallow: /save\nDisallow: /safety/?\nDisallow: /safety?\nDisallow: /search\nAllow: /search/entity/touch/\nAllow: /search?products_mode=1\nAllow: /search/?products_mode=1\nAllow: /search?*&products_mode=1\nAllow: /search/?*&products_mode=1\nAllow: /search/touch?products_mode=1\nAllow: /search/touch/?products_mode=1\nAllow: /search/touch?*&products_mode=1\nAllow: /search/touch/?*&products_mode=1\nAllow: /search/products\nAllow: /products/$\nAllow: /products$\nDisallow: /setup\nDisallow: /showcaptcha\nDisallow: /sitesearch\nDisallow: /sl/*.html\nDisallow: /soft/extensions/goodbye\nDisallow: /sportagent\nDisallow: /storeclick\nDisallow: /storerequest\nDisallow: /subscribe/confirm.pl\nDisallow: /subscribe/view.pl\nAllow: /support/\nDisallow: /support/direct-images\nDisallow: /support/direct-tooltips\nDisallow: /support/distr\nDisallow: /support/dsp\nDisallow: /support/fe4be44a295cc679e19bf0b8f133083d\nDisallow: /support/maps-beta\nDisallow: /support/market-images\nDisallow: /support/security\nDisallow: /support/webmaster-images\nDisallow: /support/*zout_\nDisallow: /support/search-results/\nDisallow: /support/praktikum/flow.html\nDisallow: /support/praktikum-english/\nDisallow: /telsearch\nDisallow: /themes\nDisallow: /toggle-experiment\nDisallow: /touchsearch\nDisallow: /tune*retpath=\nDisallow: /turbo\nAllow: /ugcpub/\nDisallow: /ugcpub/cabinet\nDisallow: /v$\nDisallow: /viewconfig$\nDisallow: /images/v$\nDisallow: /images/viewconfig$\nAllow: /tutor\nDisallow: /tutor/admin\nDisallow: /tutor/search/problems\nDisallow: /versions\nDisallow: /video/\nAllow: /video/$\nAllow: /video/?\nAllow: /video/sitemap\nAllow: /video/movies/entity\nAllow: /video/touch/movies/entity\nAllow: /video/touch/$\nAllow: /video/touch/?\nAllow: /video/preview\nAllow: /video/touch/preview\nAllow: /video/search?text=video$\nAllow: /video/touch/search?text=video$\nAllow: /video/search?text=%D0%B2%D0%B8%D0%B4%D0%B5%D0%BE$\nAllow: /video/touch/search?text=%D0%B2%D0%B8%D0%B4%D0%B5%D0%BE$\nAllow: /video/movies/catalog\nAllow: /video/touch/movies/catalog\nAllow: /video/video_sitemap_*.xml\nAllow: /video/movies_sitemap_*.xml\nDisallow: /xmlsearch\nDisallow: /yandsearch\nDisallow: /yca/cy\nDisallow: /soft/distribution\nDisallow: /soft/?*\nDisallow: /promo/launcher/?*\nDisallow: /opera/?*\nDisallow: /firefox/?*\nDisallow: /soft/bm/?*\nDisallow: /soft/browsers/?*\nDisallow: /soft/punto/?*\nDisallow: /ie/?*\nDisallow: /soft/*goodbye*\nDisallow: /adult\nDisallow: /addappterless\nDisallow: /archive/api/image.jpg*\nAllow: /archive$\nAllow: /archive/project/sovsport$\nDisallow: /all?\nDisallow: /games/*?*reqId*\nDisallow: /games/*?*pageId*\nDisallow: /games/*?experimental=\nDisallow: /games/*?*app-id=\nDisallow: /games/*?*game-id=\nDisallow: /games/*?*div-rtx-reqid=\nDisallow: /games/*?*header=\nDisallow: /games/*?*from=\nDisallow: /games/*?*clid=\nDisallow: /games/*?*deviceid=\nDisallow: /games/*?*flags=\nDisallow: /games/*?*inapp_id=\nDisallow: /games/*?*k50id=\nDisallow: /games/*?*lang=\nDisallow: /games/*?*payload=\nDisallow: /games/*?*pfid=\nDisallow: /games/*?*promo_id=\nDisallow: /games/*?*promo_intent=\nDisallow: /games/*?*referrer=\nDisallow: /games/*?*utm_\nDisallow: /games/*?*ypr=\nDisallow: /familysearch\nDisallow: /schoolsearch\nDisallow: /yandpage\nDisallow: *maps/covid19*\nDisallow: /promo/market/mobile/app*?from=\nDisallow: /support2/*zout_\nDisallow: /activity/\nDisallow: /sport\nDisallow: /support2/sport*\n\nAllow: /medicine\nDisallow: /medicine/search\nDisallow: /medicine?text=\nDisallow: /finance*?\nDisallow: /finance/assistant\n\nUser-agent: OAI-SearchBot\nDisallow: /?\nDisallow: /403.html\nDisallow: /404.html\nDisallow: /500.html\nDisallow: /adddata\nDisallow: /adresa-segmentator\nDisallow: /advanced_engl.html\nDisallow: /advertising\nDisallow: /alice/term/\nDisallow: /alice/*/term/\nDisallow: /alice/old-browser\nDisallow: /alice/store/orders/\nDisallow: /alice*?\nDisallow: /alice/*&\nDisallow: /alice/chat/\nDisallow: /alice?share=\nDisallow: /all-supported-params\nDisallow: /articles\nDisallow: /aura/\nAllow: /aura/promo/\nDisallow: /blog/*?text=\nDisallow: /blog/*?tag=\nDisallow: /blog/*/*/*\nDisallow: /blog/punto\nDisallow: /MAPS/\nDisallow: /MaPS/\nDisallow: /MapS/\nDisallow: /Maps/\nDisallow: /mApS/\nDisallow: /harita/business/\nDisallow: /maps/business/\nDisallow: /maps-api/docs/js-api/reference-guide/instruction-gxs.html\nDisallow: /maps-api/docs/js-api/reference-guide/instruction-rs.html\nDisallow: /catalog/?text=\nDisallow: /chat\nDisallow: /cgi-bin/\nDisallow: /cgi/\nDisallow: /clck\nDisallow: /collections/\nDisallow: /company/*.rss\nDisallow: /company/search\nDisallow: /comments/*\nDisallow: /courier\nDisallow: /conflagexp\nDisallow: /cy\nDisallow: /cycounter\nDisallow: /dzen\nDisallow: /experiments.xml\nDisallow: /formfeedback\nDisallow: /fresh\nDisallow: /goto_issue/\nDisallow: /goto_rubric/\nDisallow: /gundem\nDisallow: /i/\nDisallow: /images-data\nDisallow: /images.html\nDisallow: /images/*\nDisallow: /images-apphost/*\nAllow: /images/$\nAllow: /images/smart/$\nAllow: /images/touch/$\nAllow: /images/touch/?\nAllow: /images/search?text=*&source=morda*\nAllow: /images/search?rpt=imageview\nAllow: /images/touch/search?text=*&source=morda*\nAllow: /images/touch/search?rpt=imageview\nAllow: /images/pad/search?text=*&source=morda*\nAllow: /images/pad/search?rpt=imageview\nDisallow: /index_m\nDisallow: /iznanka/\nDisallow: /int_search_summary\nDisallow: /delivery_tracking\nDisallow: /jobs/skill-diagnostic/private/*\nDisallow: /keyboard_qwerty.html\nDisallow: /logotypes\nDisallow: /business/widget\nDisallow: /local/\nDisallow: /lab/neurodetector\nDisallow: /lab/habr-detector\nDisallow: /more_samples\nDisallow: /msearch\nDisallow: /map-constructor/loader*\nDisallow: /maps/print/*\nAllow: /maps/*?lang=ru$\nAllow: /maps/*?lang=uk$\nAllow: /maps/*?lang=kk$\nAllow: /maps/*?lang=uz$\nDisallow: /nmaps/*?\nDisallow: /mapeditor/*?\nDisallow: /metro/*?from\nDisallow: /masters\nDisallow: /services/executor/\nDisallow: /n?\nDisallow: /n$\nDisallow: /n/\nDisallow: /neuralsearch/api\nDisallow: /norobot\nDisallow: /opensearch.xml\nDisallow: /padsearch\nDisallow: /person\nDisallow: /podpiska/login.pl\nDisallow: /pogoda/search\nDisallow: /pogoda/404\nDisallow: /pogoda/0\nDisallow: /polling\nDisallow: /poll/\nDisallow: /portal/set/\nDisallow: /portal/tune/\nDisallow: /prefetch\nDisallow: /promo/*goodbye*\nDisallow: /promo/diskelement\nDisallow: /promo/launcher/mgoodbye/*\nDisallow: /promo/*welcome*\nDisallow: /promo/yobject/changelog*\nDisallow: /project/eda/partnership_services/$\nDisallow: /q/profile/*/answers\nDisallow: /q/profile/*/questions\nDisallow: /q/profile/*/verifies\nDisallow: /q/profile/*/articles\nDisallow: /q/profile/*/subscribers\nDisallow: /q/profile/*/org_answers\nDisallow: /q/user/*/answers\nDisallow: /q/user/*/questions\nDisallow: /q/user/*/verifies\nDisallow: /q/user/*/articles\nDisallow: /q/user/*/subscribers\nDisallow: /q/user/*/org_answers\nDisallow: /soft/bm/goodbye\nDisallow: /soft/chrome/ext-install/*\nDisallow: /soft/chrome/searchline-install/*\nDisallow: /soft/*goodbye*\nDisallow: /soft/punto/mac/uninstall/*\nDisallow: /soft/punto/win/uninstall/*\nDisallow: /soft/*welcome*\nDisallow: /soft/win/?*\nDisallow: /quotes\nDisallow: /realty*?*pinned_offer_id=\nDisallow: /realty*?*pinned_nb_id=\nDisallow: /realty/search\nDisallow: /realty*?*text=\nDisallow: /realty/result\nDisallow: /realty/map_pins\nDisallow: /realty/map_dust\nDisallow: /realty/callme\nDisallow: /redir\nDisallow: /region_map\nDisallow: /regions_list.xml\nDisallow: /regions.html?\nDisallow: /s/\nDisallow: /save\nDisallow: /safety/?\nDisallow: /safety?\nDisallow: /search\nAllow: /search/entity/touch/\nDisallow: /setup\nDisallow: /showcaptcha\nDisallow: /sitesearch\nDisallow: /sl/*.html\nDisallow: /soft/extensions/goodbye\nDisallow: /sportagent\nDisallow: /storeclick\nDisallow: /storerequest\nDisallow: /subscribe/confirm.pl\nDisallow: /subscribe/view.pl\nAllow: /support/\nDisallow: /support/direct-images\nDisallow: /support/direct-tooltips\nDisallow: /support/distr\nDisallow: /support/dsp\nDisallow: /support/fe4be44a295cc679e19bf0b8f133083d\nDisallow: /support/maps-beta\nDisallow: /support/market-images\nDisallow: /support/security\nDisallow: /support/webmaster-images\nDisallow: /support/*zout_\nDisallow: /support/search-results/\nDisallow: /support/praktikum/flow.html\nDisallow: /support/praktikum-english/\nDisallow: /telsearch\nDisallow: /themes\nDisallow: /toggle-experiment\nDisallow: /touchsearch\nDisallow: /tune*retpath=\nDisallow: /turbo\nAllow: /ugcpub/\nDisallow: /ugcpub/cabinet\nDisallow: /v$\nDisallow: /viewconfig$\nDisallow: /images/v$\nDisallow: /images/viewconfig$\nAllow: /tutor\nDisallow: /tutor/admin\nDisallow: /tutor/search/problems\nDisallow: /versions\nDisallow: /video/*\nAllow: /video/$\nAllow: /video/?\nAllow: /video/sitemap\nAllow: /video/movies/entity\nAllow: /video/touch/movies/entity\nAllow: /video/touch/$\nAllow: /video/touch/?\nAllow: /video/preview\nAllow: /video/touch/preview\nAllow: /video/search?text=video$\nAllow: /video/touch/search?text=video$\nAllow: /video/search?text=%D0%B2%D0%B8%D0%B4%D0%B5%D0%BE$\nAllow: /video/touch/search?text=%D0%B2%D0%B8%D0%B4%D0%B5%D0%BE$\nAllow: /video/video_sitemap_*.xml\nAllow: /video/movies_sitemap_*.xml\nDisallow: /xmlsearch\nDisallow: /yandsearch\nDisallow: /yca/cy\nDisallow: /soft/distribution\nDisallow: /soft/?*\nDisallow: /promo/launcher/?*\nDisallow: /opera/?*\nDisallow: /firefox/?*\nDisallow: /soft/bm/?*\nDisallow: /soft/browsers/?*\nDisallow: /soft/punto/?*\nDisallow: /ie/?*\nDisallow: /soft/*goodbye*\nDisallow: /adult\nDisallow: /addappterless\nDisallow: /archive/api/image.jpg*\nAllow: /archive$\nAllow: /archive/project/sovsport$\nDisallow: /all?\nDisallow: /games/*?*reqId*\nDisallow: /games/*?*pageId*\nDisallow: /games/*?experimental=\nDisallow: /games/*?*app-id=\nDisallow: /games/*?*game-id=\nDisallow: /games/*?*div-rtx-reqid=\nDisallow: /games/*?*header=\nDisallow: /games/*?*from=\nDisallow: /games/*?*clid=\nDisallow: /games/*?*deviceid=\nDisallow: /games/*?*flags=\nDisallow: /games/*?*inapp_id=\nDisallow: /games/*?*k50id=\nDisallow: /games/*?*lang=\nDisallow: /games/*?*payload=\nDisallow: /games/*?*pfid=\nDisallow: /games/*?*promo_id=\nDisallow: /games/*?*promo_intent=\nDisallow: /games/*?*referrer=\nDisallow: /games/*?*utm_\nDisallow: /games/*?*ypr=\nDisallow: /familysearch\nDisallow: /schoolsearch\nDisallow: /yandpage\nDisallow: *maps/covid19*\nDisallow: /promo/market/mobile/app*?from=\nDisallow: /support2/*zout_\nDisallow: /activity/\nDisallow: /sport\nDisallow: /support2/sport\n\nAllow: /medicine\nDisallow: /medicine/search\nDisallow: /medicine?text=\nDisallow: /finance\nAllow: /finance/reviews\nAllow: /finance/banks/*/reviews\nDisallow: /finance/reviews*?*\nDisallow: /finance/banks/*/reviews*?\n\nSitemap: https://yandex.ru/pogoda/sitemap-index.xml\nSitemap: https://yandex.ru/support/sitemap.xml\nSitemap: https://yandex.ru/blog/sitemap.xml\nSitemap: https://frontend.s3.yandex.net/health/sitemap.xml\nSitemap: https://yandex.ru/uslugi/sitemap/sitemap.xml.gz\nSitemap: https://yandex.ru/turbo/public/sitemap.xml\nSitemap: https://yandex.ru/tutor/sitemap/?sitemap=sitemap.xml\nSitemap: https://yandex.ru/games/sitemaps/sitemap.index.xml\nSitemap: https://yandex.ru/q/sitemap/sitemap-index.xml\nSitemap: https://yandex.ru/images/sitemap.xml\nSitemap: https://yandex.ru/medicine/sitemaps/sitemap.xml\nSitemap: https://yandex.ru/yandexapp/ru/callerid/sitemap.xml\nSitemap: https://yandex.ru/video/sitemap_index.xml\nSitemap: https://yandex.ru/realty/sitemaps/human_friendly/sitemap_index_hfp-00001.xml\nSitemap: https://yandex.ru/images/sitemap_index.xml\nSitemap: https://yandex.ru/services/sitemaps/sitemap.xml\nSitemap: https://yandex.ru/archive/sitemap.xml\n\nClean-Param: ncrnd&redircnt&clid&_&win /\nClean-Param: utm_source&utm_medium&utm_campaign&utm_term&utm_content&utm_nooverride&utm_expid&gclid&_openstat&yclid&gsaid&_gs_rmark&_gs_ref&_gs_cttl&iframe /health/\nClean-Param: print=1 /tutor/\nClean-Param: w&w_question_id /q/question/*\nClean-Param: &appsearch_header=1&comments=1 /news/story/\nClean-Param: app-id&game-id&div-rtx-reqid&lang&platform&header&from&backQueryParams /games/\nClean-Param: ll&z&sll /maps/\nClean-param: oll&rtext&pt&source&via&from&origin&um&utm_medium&utm_source&portal&panorama%5Bspan%5D&panorama%5Bpoint%5D&panorama%5Bfull%5D&panorama%5Bdirection%5D&panorama%5Bair%5D&masstransit%5BstopId%5D&ol&poi%5Bpoint%5D&whatshere%5Bpoint%5D&whatshere%5Bzoom%5D&ouri&ruri&l&mode&tab&rtt&rtm&display-text&text&indoorLevel&trfm&trfst&no-distribution&distance&slices&scale&descy&descx&mapY&mapX&mapID&sctx&lay&where&results&map&id&spn&sspn&rtn&ost&profile-mode /maps/\nClean-Param: appsearch_header /pogoda\nClean-param: from&publication_id&email_form&id&page&test&year&month&lang&onlyWithCases&gcmes&cmnt_login&_aid&currentURL&res&cmnt_from&amp&comments&certId&mindbox-message-key&status&menu&block&version&ref&etext&ctime&nw&commentId&yqrid&erid&specializations&specializations[]&cities&workFormats&product&filters&autm_medium&industries&search_open&tasks&search_text&region&region_name&lastname /adv/\nClean-param: fl_checkpoint&clckid&begin&yprqee&products&yadclid&yadordid&direct&case_products&wvstest&modal&suggest_reqid&only-content&tb&source&p&m-message-key-id&journal&date&months[]&answer_email&url&intermedia&industry&service&country&K1[K2]&lr&budgetOrder&certificate&utm_source&utm_campaign&utm_medium&utm_term&utm_content /adv/\nClean-param: z&ll&mapTypeRoute&slug&uuid&app&is_user_location&app_with_layers&app_soyuz&from&dateRoute&clid&via&city&amp&topic&lat&lon&lang&le_Lightning&from&utm_source&utm_campaign&utm_medium&utm_content&utm_term /pogoda/\nClean-param: z&ll&mapTypeRoute&slug&uuid&app&is_user_location&app_with_layers&app_soyuz&from&dateRoute&clid&via&city&amp&topic&lat&lon&lang&le_Lightning&from&utm_source&utm_campaign&utm_medium&utm_content&utm_term /hava/\nClean-param: z&ll&mapTypeRoute&slug&uuid&app&is_user_location&app_with_layers&app_soyuz&from&dateRoute&clid&via&city&amp&topic&lat&lon&lang&le_Lightning&from&utm_source&utm_campaign&utm_medium&utm_content&utm_term /weather/\nClean-param: nw&from_block&mode&mar&photoType&regid /pogoda/\nClean-param: nw&from_block&mode&mar&photoType&regid /hava/\nClean-param: nw&from_block&mode&mar&photoType&regid /weather/\nClean-param: from_pumpkin&le_WindParticles&nowcast&yw_preset&gen_time&for_date&morda_geolocation&retpath&acc_maps&le_TemperatureBalloons&request&le_Borders&elmt&may&source&ids&june&july&referrer&PAGEN_1&from_render_fallback&preset&to&august&appsearch_ys&region&tm&banner-test-tags /pogoda/\nClean-param: stat-id&format-type&actual-format&april&test-tag&adgroup&campaign_id&c&nowcast_header&detailsType&utm_from&bar_exp&march&november&bounds&game&text&city_id&february&september&january&no_turbo&october&ref&december&weather_url&build&lp_source&tab&geoid&cameras&day&thndr_onbrd /pogoda/\nClean-param: from_pumpkin&le_WindParticles&nowcast&yw_preset&gen_time&for_date&morda_geolocation&retpath&acc_maps&le_TemperatureBalloons&request&le_Borders&elmt&may&source&ids&june&july&referrer&PAGEN_1&from_render_fallback&preset&to&august&appsearch_ys&region&tm&banner-test-tags&stat-id /hava/\nClean-param: format-type&actual-format&april&test-tag&adgroup&campaign_id&c&nowcast_header&detailsType&utm_from&bar_exp&march&november&bounds&game&text&city_id&february&september&january&no_turbo&october&ref&december&weather_url&build&lp_source&tab&geoid&cameras&day&thndr_onbrd /hava/\nClean-param: from_pumpkin&le_WindParticles&nowcast&yw_preset&gen_time&for_date&morda_geolocation&retpath&acc_maps&le_TemperatureBalloons&request&le_Borders&elmt&may&source&ids&june&july&referrer&PAGEN_1&from_render_fallback&preset&to&august&appsearch_ys&region&tm&banner-test-tags&stat-id /weather/\nClean-param: format-type&actual-format&april&test-tag&adgroup&campaign_id&c&nowcast_header&detailsType&utm_from&bar_exp&march&november&bounds&game&text&city_id&february&september&january&no_turbo&october&ref&december&weather_url&build&lp_source&tab&geoid&cameras&day&thndr_onbrd /weather/\nClean-param: utm_source&utm_medium&utm_term&utm_content&utm_campaign&yclid&referrer&ref&from&adj_campaign&win&lp_source&openstat /alice/\n","agents":{"GPTBot":true,"OAI-SearchBot":true,"ChatGPT-User":true,"ClaudeBot":true,"Claude-SearchBot":true,"PerplexityBot":true,"Google-Extended":true,"Applebot-Extended":true,"CCBot":true,"Bytespider":true,"Meta-ExternalAgent":true,"Amazonbot":true}},"sitemap":{"status":200,"content_type":"text/xml","bytes":13477,"sha256":"f3b052a7fadac11365e892fc42211ddd3053fa4edc9e90ea49f24521e2eed41e","final_host":"yandex.ru","final_path":"/support/sitemap.xml","ms":331,"url":"https://yandex.ru/support/sitemap.xml","declared":true,"xml":true},"files":{"llms_txt":{"status":404,"content_type":"text/plain","bytes":9,"sha256":"e3ebaa16dd9d9b9fc107c42183fb6cf9d22927e1af03dbbdfa0ccc38e4e4ac31","final_host":"yandex.ru","final_path":"/llms.txt","ms":283,"path":"/llms.txt","html":false,"json":null,"present":false},"agents_md":{"status":404,"content_type":"text/plain","bytes":9,"sha256":"e3ebaa16dd9d9b9fc107c42183fb6cf9d22927e1af03dbbdfa0ccc38e4e4ac31","final_host":"yandex.ru","final_path":"/agents.md","ms":735,"path":"/agents.md","html":false,"json":null,"present":false},"agent_skills":{"status":404,"content_type":null,"bytes":9,"sha256":"0019dfc4b32d63c1392aa264aed2253c1e0c2fb09216f8e2cc269bbfb8bb49b5","final_host":"yandex.ru","final_path":"/.well-known/agent-skills/index.json","ms":291,"path":"/.well-known/agent-skills/index.json","html":false,"json":null,"present":false},"mcp_card":{"status":404,"content_type":null,"bytes":9,"sha256":"0019dfc4b32d63c1392aa264aed2253c1e0c2fb09216f8e2cc269bbfb8bb49b5","final_host":"yandex.ru","final_path":"/.well-known/mcp/server-card.json","ms":284,"path":"/.well-known/mcp/server-card.json","html":false,"json":null,"present":false},"api_catalog":{"status":404,"content_type":null,"bytes":9,"sha256":"0019dfc4b32d63c1392aa264aed2253c1e0c2fb09216f8e2cc269bbfb8bb49b5","final_host":"yandex.ru","final_path":"/.well-known/api-catalog","ms":285,"path":"/.well-known/api-catalog","html":false,"json":null,"present":false},"a2a_card":{"status":404,"content_type":null,"bytes":9,"sha256":"0019dfc4b32d63c1392aa264aed2253c1e0c2fb09216f8e2cc269bbfb8bb49b5","final_host":"yandex.ru","final_path":"/.well-known/agent-card.json","ms":417,"path":"/.well-known/agent-card.json","html":false,"json":null,"present":false}},"rings":{"v":1,"candidates":[],"mcp":[],"a2a":null,"webmcp":{"model_context":false,"bridge":false,"mcp_json":false,"link_rel":false,"any":false},"handshakes":0,"get_path":{"status":404,"content_type":"application/json","ms":555,"error":null},"get_host":{"status":404,"content_type":"application/json","ms":803,"error":null}},"sf":[4,0,0,0,0,0,0,0,0],"derived":{"reachable":true,"home_class":"answers","robots_served":true,"robots_sitemap":true,"content_signal":false,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":[],"blocks_any_ai":false,"files":{"llms_txt":false,"agents_md":false,"agent_skills":false,"mcp_card":false,"api_catalog":false,"a2a_card":false},"any_capability":false,"ai_refused":[],"ai_thin":[],"aipref":0,"aipref_measured":true,"sf":[4,0,0,0,0,0,0,0,0],"rings":{"mcp_declared":false,"mcp_live":false,"mcp_auth":false,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":3,"fh":"sso.passport.yandex.ru","fp_other_host":[]},"derive_v":5,"frame":"top1k","shard":2},{"v":1,"kind":"crawlcheck-census-record","run":43,"domain":"wordpress.com","rank":100,"measured_at":"2026-11-13T07:59:45.450Z","home":{"browser":{"status":403,"content_type":"text/html","bytes":1762,"sha256":"fc89e2a14f48c43f7c85082a9360f737c7b13e2180a76370e7ae7a4ed6e3355f","final_host":"wordpress.com","final_path":"/","ms":59,"identity":"Mobile browser","html":true},"gptbot":{"status":403,"content_type":"text/html","bytes":1780,"sha256":"9288554aae934d49582257310e647b1da6118a37e253c9944e515c62d0cb8870","final_host":"wordpress.com","final_path":"/","ms":6,"identity":"GPTBot","html":true},"claudebot":{"status":403,"content_type":"text/html","bytes":1780,"sha256":"9288554aae934d49582257310e647b1da6118a37e253c9944e515c62d0cb8870","final_host":"wordpress.com","final_path":"/","ms":6,"identity":"ClaudeBot","html":true},"oaisearch":{"status":403,"content_type":"text/html","bytes":1780,"sha256":"9288554aae934d49582257310e647b1da6118a37e253c9944e515c62d0cb8870","final_host":"wordpress.com","final_path":"/","ms":6,"identity":"OAI-SearchBot","html":true},"perplexity":{"status":403,"content_type":"text/html","bytes":1780,"sha256":"9288554aae934d49582257310e647b1da6118a37e253c9944e515c62d0cb8870","final_host":"wordpress.com","final_path":"/","ms":25,"identity":"PerplexityBot","html":true}},"robots":{"status":200,"content_type":"text/plain","bytes":1387,"sha256":"4c6624558c79ed5e3defff9aa06f727c1a35337a3295b7106a49d7796d96df43","final_host":"wordpress.com","final_path":"/robots.txt","ms":10,"html":false,"text":"# If you are regularly crawling WordPress.com sites, please use our firehose to receive real-time push updates instead.\n# Please see https://developer.wordpress.com/docs/firehose/ for more details.\n\nSitemap: https://wordpress.com/sitemap.xml\nSitemap: https://wordpress.com/news-sitemap.xml\n\nSitemap: https://wordpress.com/go/sitemap.xml\nSitemap: https://wordpress.com/blog/sitemap.xml\nSitemap: https://wordpress.com/support/sitemap.xml\n\nUser-agent: *\nDisallow: /wp-admin/\nAllow: /wp-admin/admin-ajax.php\nDisallow: /typo/?subdomain=\nDisallow: /read/\nDisallow: /*/read/\nAllow: /forums/topic-tag/reader/\nAllow: /*/forums/topic-tag/reader/\nAllow: /support/reader/\nAllow: /*/support/reader/\nDisallow: /reader/\nDisallow: /*/reader/\nAllow: /log-in/\nAllow: /log-in$\nDisallow: /log-in?\nDisallow: /abuse/?*\nDisallow: /abuse?*\nDisallow: /plugins/?s=\nDisallow: /*/plugins/?s=\nDisallow: /*?aff=\nDisallow: /*&aff=\nDisallow: /*?affiliate=\nDisallow: /*&affiliate=\nDisallow: /*?cid=\nDisallow: /*&cid=\nDisallow: /*?irclickid=\nDisallow: /*&irclickid=\nDisallow: /*/?like_comment=\nDisallow: /*?retry=\nDisallow: /*?sid=\nDisallow: /*?action=\nDisallow: /wp-login.php\nDisallow: /wp-signup.php\nDisallow: /press-this.php\nDisallow: /remote-login.php\nDisallow: /activate/\nDisallow: /cgi-bin/\nDisallow: /mshots/v1/\nDisallow: /next/\nDisallow: /public.api/\n\n# This file was generated on Thu, 12 Nov 2026 21:07:27 +0000\n","agents":{"GPTBot":true,"OAI-SearchBot":true,"ChatGPT-User":true,"ClaudeBot":true,"Claude-SearchBot":true,"PerplexityBot":true,"Google-Extended":true,"Applebot-Extended":true,"CCBot":true,"Bytespider":true,"Meta-ExternalAgent":true,"Amazonbot":true}},"sitemap":null,"files":null,"derived":{"reachable":false,"home_class":"refused_us","robots_served":false,"robots_sitemap":false,"content_signal":false,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":[],"blocks_any_ai":false,"files":{"llms_txt":false,"agents_md":false,"agent_skills":false,"mcp_card":false,"api_catalog":false,"a2a_card":false},"any_capability":false,"ai_refused":[],"ai_thin":[],"rings":{"mcp_declared":false,"mcp_live":false,"mcp_auth":false,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":3,"fh":"wordpress.com","fp_other_host":[]},"derive_v":5,"frame":"top1k","shard":3}]}