{"ok":true,"kind":"crawlcheck-census-records","frame":"top1k","run":14,"complete":true,"measured":991,"of":991,"shards":8,"offset":0,"returned":100,"next":"https://crawlcheck.io/api/census/records?run=14&offset=100&limit=100","records":[{"v":1,"kind":"crawlcheck-census-record","run":14,"domain":"google.com","rank":1,"measured_at":"2026-10-15T05:26:03.595Z","home":{"browser":{"status":429,"content_type":"text/html","bytes":3128,"sha256":"b64b8158cb329f0c3a31aa9950e68da1c7325d8935066b3180299f627fcae9fa","final_host":"www.google.com","ms":443,"identity":"Mobile browser","html":true},"gptbot":{"status":429,"content_type":"text/html","bytes":3128,"sha256":"1dfe78f015d064bae7102c6e7a30c5a4d5b0b6c69da93ac9a93b8cd1346f10bc","final_host":"www.google.com","ms":304,"identity":"GPTBot","html":true},"claudebot":{"status":429,"content_type":"text/html","bytes":3128,"sha256":"e26a196968c65b09168474a7ab7410d77acf7674ef8ebdfa71d73114f64b6af5","final_host":"www.google.com","ms":489,"identity":"ClaudeBot","html":true},"oaisearch":{"status":429,"content_type":"text/html","bytes":3128,"sha256":"affc5966fcee5e688a3f7e77524c4bc5944eb6bfd0d02d8ad6a781bca209bda6","final_host":"www.google.com","ms":513,"identity":"OAI-SearchBot","html":true},"perplexity":{"status":429,"content_type":"text/html","bytes":3128,"sha256":"e3df33d599756e25eaa563a4decee836a25740186372a11a35dd6bd9db726a15","final_host":"www.google.com","ms":390,"identity":"PerplexityBot","html":true}},"robots":{"status":200,"content_type":"text/plain","bytes":6624,"sha256":"1aba570a1b32deb66941e0c29d4f37be71ee58aa1a624aa20754b3639538cf2c","final_host":"www.google.com","ms":133,"html":false,"text":"User-agent: *\nUser-agent: Yandex\nDisallow: /search\nAllow: /search/about\nAllow: /search/howsearchworks\nDisallow: /sdch\nDisallow: /groups\nDisallow: /index.html?\nDisallow: /?\nDisallow: /goto?\nAllow: /?hl=\nDisallow: /?hl=*&\nAllow: /?hl=*&gws_rd=ssl$\nDisallow: /?hl=*&*&gws_rd=ssl\nAllow: /?gws_rd=ssl$\nAllow: /?pt1=true$\nDisallow: /imgres\nDisallow: /u/\nDisallow: /setprefs\nDisallow: /m?\nDisallow: /m/\nAllow:    /m/finance\nDisallow: /wml?\nDisallow: /wml/?\nDisallow: /wml/search?\nDisallow: /xhtml?\nDisallow: /xhtml/?\nDisallow: /xhtml/search?\nDisallow: /xml?\nDisallow: /imode?\nDisallow: /imode/?\nDisallow: /imode/search?\nDisallow: /jsky?\nDisallow: /jsky/?\nDisallow: /jsky/search?\nDisallow: /pda?\nDisallow: /pda/?\nDisallow: /pda/search?\nDisallow: /local?\nDisallow: /local_url\nDisallow: /products?\nDisallow: /product_\nDisallow: /products_\nDisallow: /products;\nDisallow: /print\nDisallow: /books/\nDisallow: /bkshp?*dq=\nDisallow: /bkshp?*q=\nDisallow: /books?*dq=\nDisallow: /books?*q=\nDisallow: /books?*qtid=\nDisallow: /books?*output=\nDisallow: /books?*pg=\nDisallow: /books?*jtp=\nDisallow: /books?*jscmd=\nDisallow: /books?*buy=\nDisallow: /books?*zoom=\nAllow: /books/about\nAllow: /books?*zoom=1\nAllow: /books?*zoom=5\nAllow: /books/content?*zoom=1\nAllow: /books/content?*zoom=5\nDisallow: /patents?\nDisallow: /patents/download/\nDisallow: /patents/pdf/\nDisallow: /patents/related/\nDisallow: /scholar\nDisallow: /citations?\nAllow: /citations?user=\nAllow: /citations?view_op=new_profile\nAllow: /citations?view_op=top_venues\nAllow: /scholar_share\nDisallow: /s?\nDisallow: /maps?\nAllow: /maps?daddr=\nAllow: /maps?entry=wc\nAllow: /maps?f=\nAllow: /maps?hl=\nAllow: /maps?q=\nAllow: /maps?saddr=\nAllow: /maps?sid=\nAllow: /maps?*output=classic\nAllow: /maps?*file=\nDisallow: /mapslt?\nDisallow: /maphp?\nDisallow: /maps/\nAllow: /maps/$\nAllow: /maps/@\nAllow: /maps/?daddr=\nAllow: /maps/?entry=wc\nAllow: /maps/?f=\nAllow: /maps/?hl=\nAllow: /maps/?q=\nAllow: /maps/?saddr=\nAllow: /maps/?sid=\nAllow: /maps/api/staticmap?\nAllow: /maps/search/\nAllow: /maps/sitemap.xml\nAllow: /maps/sitemaps/\nAllow: /maps/dir/\nAllow: /maps/d/\nAllow: /maps/reserve\nAllow: /maps/about\nAllow: /maps/contrib/\nAllow: /maps/match\nAllow: /maps/ms?\nAllow: /maps/place/\nAllow: /maps/_/\nAllow: /search?*tbm=map\nAllow: /maps/vt?\nAllow: /maps/preview\nDisallow: /maps/api/js/\nAllow: /maps/api/js\nDisallow: /mld?\nDisallow: /staticmap?\nDisallow: /help/maps/streetview/partners/welcome/\nDisallow: /help/maps/indoormaps/partners/\nDisallow: /lochp?\nDisallow: /ie?\nDisallow: /uds/\nDisallow: /transit?\nDisallow: /trends?\nDisallow: /trends/music?\nDisallow: /trends/hottrends?\nDisallow: /trends/viz?\nDisallow: /trends/embed.js?\nDisallow: /trends/fetchComponent?\nDisallow: /trends/beta\nDisallow: /trends/topics\nDisallow: /trends/explore?\nDisallow: /trends/api\nDisallow: /musica\nDisallow: /musicl\nDisallow: /musics\nDisallow: /urchin_test/\nDisallow: /movies?\nDisallow: /wapsearch?\nDisallow: /reviews/search?\nDisallow: /cbk\nDisallow: /profiles/me\nDisallow: /s2/profiles/me\nAllow: /s2/profiles\nAllow: /s2/oz\nAllow: /s2/photos\nAllow: /s2/search/social\nAllow: /s2/static\nDisallow: /s2\nDisallow: /transconsole/portal/\nDisallow: /aclk\nDisallow: /tbproxy/\nDisallow: /support/forum/search?\nDisallow: /reviews/polls/\nDisallow: /hosted/images/\nDisallow: /accounts/ClientLogin\nDisallow: /accounts/ClientAuth\nDisallow: /accounts/o8\nAllow: /accounts/o8/id\nDisallow: /quality_form?\nDisallow: /labs/popgadget/search\nDisallow: /compressiontest/\nDisallow: /analytics/feeds/\nDisallow: /analytics/partners/comments/\nDisallow: /analytics/portal/\nDisallow: /analytics/uploads/\nAllow: /alerts/manage\nAllow: /alerts/remove\nDisallow: /alerts/\nAllow: /alerts/$\nDisallow: /phone/compare/?\nDisallow: /travel/clk\nDisallow: /travel/lodging/clk\nDisallow: /travel/entity\nDisallow: /travel/search\nDisallow: /travel/flights/booking\nDisallow: /travel/flights/s/\nDisallow: /travel/flights/search\nDisallow: /travel/hotels/stories\nDisallow: /travel/hotels/*/stories\nDisallow: /travel/story\nDisallow: /hotelfinder/rpc\nDisallow: /hotels/rpc\nDisallow: /evaluation/\nDisallow: /forms/perks/\nDisallow: /shopping/suppliers/search\nDisallow: /edu/cs4hs/\nDisallow: /trustedstores/s/\nDisallow: /trustedstores/tm2\nDisallow: /trustedstores/verify\nDisallow: /shopping?\nDisallow: /shopping/product/\nDisallow: /shopping/seller\nDisallow: /shopping/ratings/account/metrics\nDisallow: /shopping/ratings/merchant/immersivedetails\nDisallow: /shopping/reviewer\nDisallow: /shopping/search\nDisallow: /shopping/deals\nAllow: /shopping?udm=28$\nDisallow: /storefront\nDisallow: /storepicker\nDisallow: /about/careers/applications/candidate-prep\nDisallow: /about/careers/applications/connect-with-a-googler\nDisallow: /about/careers/applications/jobs/results?page=\nDisallow: /about/careers/applications/jobs/results/?page=\nDisallow: /about/careers/applications/jobs/results?*&page=\nDisallow: /about/careers/applications/jobs/results/?*&page=\nDisallow: /landing/signout.html\nDisallow: /gallery/\nDisallow: /landing/now/ontap/\nAllow: /maps/reserve\nAllow: /maps/reserve/partners\nDisallow: /maps/reserve/api/\nDisallow: /maps/reserve/search\nDisallow: /maps/reserve/bookings\nDisallow: /maps/reserve/settings\nDisallow: /maps/reserve/manage\nDisallow: /maps/reserve/payment\nDisallow: /maps/reserve/receipt\nDisallow: /maps/reserve/sellersignup\nDisallow: /maps/reserve/feedback\nDisallow: /maps/reserve/terms\nDisallow: /maps/reserve/m/\nDisallow: /maps/reserve/b/\nDisallow: /maps/reserve/partner-dashboard\nDisallow: /local/cars\nDisallow: /local/dealership/\nDisallow: /local/dining/\nDisallow: /local/place/products/\nDisallow: /local/place/reviews/\nDisallow: /local/place/rap/\nDisallow: /local/tab/\nDisallow: /localservices/\nAllow: /localservices/coupon/\nDisallow: /nonprofits/account/\nDisallow: /uviewer\nDisallow: /landing/cmsnext-root/\n\n# AdsBot\nUser-agent: AdsBot-Google\nDisallow: /maps/api/js/\nAllow: /maps/api/js\nDisallow: /maps/api/place/js/\nDisallow: /maps/api/staticmap\nDisallow: /maps/api/streetview\n\n# New user agent groups must also have a user agent reference in the global (*)\n# group. See \"Order of precedence\" section in\n# https://goo.gle/rep#order-of-precedence-for-user-agents\nUser-agent: Yandex\nDisallow: /about/careers/applications/jobs/results\nDisallow: /about/careers/applications-a/jobs/results\n\n# Crawlers of certain social media sites are allowed to access page markup when\n# google.com/imgres* links are shared. To learn more, please contact\n# images-robots-allowlist@google.com.\nUser-agent: facebookexternalhit\nUser-agent: Twitterbot\nAllow: /imgres\nAllow: /search\nDisallow: /groups\nDisallow: /hosted/images/\nDisallow: /m/\n\nSitemap: https://www.google.com/sitemap.xml\n","agents":{"GPTBot":true,"OAI-SearchBot":true,"ChatGPT-User":true,"ClaudeBot":true,"Claude-SearchBot":true,"PerplexityBot":true,"Google-Extended":true,"Applebot-Extended":true,"CCBot":true,"Bytespider":true,"Meta-ExternalAgent":true,"Amazonbot":true}},"sitemap":null,"files":null,"derived":{"reachable":false,"robots_served":false,"robots_sitemap":false,"content_signal":false,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":[],"blocks_any_ai":false,"files":{"llms_txt":false,"agents_md":false,"agent_skills":false,"mcp_card":false,"api_catalog":false,"a2a_card":false},"any_capability":false,"ai_refused":[],"ai_thin":[],"rings":{"mcp_declared":false,"mcp_live":false,"mcp_auth":false,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":2,"fh":"www.google.com","fp_other_host":[]},"derive_v":3,"frame":"top1k","shard":0},{"v":1,"kind":"crawlcheck-census-record","run":14,"domain":"cloudflare.com","rank":2,"measured_at":"2026-10-15T05:27:03.729Z","home":{"browser":{"status":200,"content_type":"text/html","bytes":1334314,"sha256":"71b540b3335342706c8bb30277a3a44b3c8fe1bbd10b6c39e27b73227e956d9a","final_host":"www.cloudflare.com","ms":198,"identity":"Mobile browser","html":true,"fp":"8913d00cc5dfeb9a"},"gptbot":{"status":200,"content_type":"text/html","bytes":1334314,"sha256":"71b540b3335342706c8bb30277a3a44b3c8fe1bbd10b6c39e27b73227e956d9a","final_host":"www.cloudflare.com","ms":147,"identity":"GPTBot","html":true,"fp":"8913d00cc5dfeb9a"},"claudebot":{"status":200,"content_type":"text/html","bytes":1334314,"sha256":"71b540b3335342706c8bb30277a3a44b3c8fe1bbd10b6c39e27b73227e956d9a","final_host":"www.cloudflare.com","ms":210,"identity":"ClaudeBot","html":true,"fp":"8913d00cc5dfeb9a"},"oaisearch":{"status":200,"content_type":"text/html","bytes":1334314,"sha256":"71b540b3335342706c8bb30277a3a44b3c8fe1bbd10b6c39e27b73227e956d9a","final_host":"www.cloudflare.com","ms":210,"identity":"OAI-SearchBot","html":true,"fp":"8913d00cc5dfeb9a"},"perplexity":{"status":200,"content_type":"text/html","bytes":1334314,"sha256":"71b540b3335342706c8bb30277a3a44b3c8fe1bbd10b6c39e27b73227e956d9a","final_host":"www.cloudflare.com","ms":210,"identity":"PerplexityBot","html":true,"fp":"8913d00cc5dfeb9a"}},"robots":{"status":200,"content_type":"text/plain","bytes":1099,"sha256":"c6e360cc6e931ea35c33ae917ff9e2194eb55f79f68e949ec54bcd4533324d98","final_host":"www.cloudflare.com","ms":21,"html":false,"text":"# Robots.txt for www.cloudflare.com\n\nUser-agent: *\nAllow: /\n\n# Sitemap\nSitemap: https://www.cloudflare.com/sitemap.xml\n\n# AI/LLM friendly content\n# See https://llmstxt.org for the llms.txt specification\n# llms.txt provides curated content for AI assistants and LLMs\n\n# Allow AI crawlers to access markdown versions of pages\nUser-agent: GPTBot\nAllow: /\n\nUser-agent: ChatGPT-User\nAllow: /\n\nUser-agent: Google-Extended\nAllow: /\n\nUser-agent: Anthropic-AI\nAllow: /\n\nUser-agent: Claude-Web\nAllow: /\n\nUser-agent: CCBot\nAllow: /\n\nUser-agent: PerplexityBot\nAllow: /\n\nUser-agent: Cohere-ai\nAllow: /\n\n# Content Signals — declare AI content usage preferences\n# See https://contentsignals.org/ and https://datatracker.ietf.org/doc/draft-romm-aipref-contentsignals/\nContent-Signal: ai-train=yes, search=yes, ai-input=yes\n\n# AI-friendly content locations\n# - /llms.txt - Curated overview for AI/LLMs (markdown)\n# - /llms-full.txt - Full expanded content for larger context windows\n# - /*.md - Markdown versions of all pages (append .md to any URL)\n# - /.well-known/agents.json - Agent discovery and capabilities\n","agents":{"GPTBot":true,"OAI-SearchBot":true,"ChatGPT-User":true,"ClaudeBot":true,"Claude-SearchBot":true,"PerplexityBot":true,"Google-Extended":true,"Applebot-Extended":true,"CCBot":true,"Bytespider":true,"Meta-ExternalAgent":true,"Amazonbot":true}},"sitemap":{"status":200,"content_type":"application/xml","bytes":1628250,"sha256":"8aadb43b16e5b9c5d3bd7a4b58eb01ef476541285375f72b773b172fa16346ee","final_host":"www.cloudflare.com","ms":10,"url":"https://www.cloudflare.com/sitemap.xml","declared":true,"xml":true},"files":{"llms_txt":{"status":200,"content_type":"text/plain","bytes":17173,"sha256":"81d53b543800b2a5fb3521421b0b522d09097c19f8e19ba092c25901d97db7ac","final_host":"www.cloudflare.com","ms":25,"path":"/llms.txt","html":false,"json":null,"present":true},"agents_md":{"status":404,"content_type":"text/plain","bytes":12,"sha256":"69967e8148f9451562c4ef70a9ba15c29cd14b13376b2595a71da547f9a03802","final_host":"www.cloudflare.com","ms":125,"path":"/agents.md","html":false,"json":null,"present":false},"agent_skills":{"status":200,"content_type":"application/json","bytes":8589,"sha256":"8388c68bdc0d4183035b6f29e568297d4efaa45ba2b1cbfcd42eef131f7672ef","final_host":"www.cloudflare.com","ms":31,"path":"/.well-known/agent-skills/index.json","html":false,"json":true,"present":true},"mcp_card":{"status":200,"content_type":"application/json","bytes":1320,"sha256":"296657d7bd0330bee2700f5de88edc054acc8637f6deb2b5cce8b9030f13706f","final_host":"www.cloudflare.com","ms":30,"path":"/.well-known/mcp/server-card.json","html":false,"json":true,"present":true},"api_catalog":{"status":404,"content_type":"text/html","bytes":176754,"sha256":"02bf5e3fdc39fa8e095c8da07dfb84b51806e9d3b9fde5d46da32dd341f11796","final_host":"www.cloudflare.com","ms":208,"path":"/.well-known/api-catalog","html":true,"json":null,"present":false},"a2a_card":{"status":404,"content_type":"text/plain","bytes":12,"sha256":"69967e8148f9451562c4ef70a9ba15c29cd14b13376b2595a71da547f9a03802","final_host":"www.cloudflare.com","ms":56,"path":"/.well-known/agent-card.json","html":false,"json":null,"present":false}},"rings":{"v":1,"candidates":[{"url":"https://docs.mcp.cloudflare.com/mcp","source":"mcp_card"},{"url":"https://blog.mcp.cloudflare.com/mcp","source":"mcp_card"},{"url":"https://mcp.cloudflare.com/mcp","source":"mcp_card"}],"mcp":[{"url":"https://docs.mcp.cloudflare.com/mcp","source":"mcp_card","outcome":"initialized","status":200,"ms":217,"initialized":true,"protocol_version":"2025-06-18","server":{"name":"docs-ai-search","version":"0.4.13"},"tools":2,"tools_sha256":"16fd520bee852c5de3a0ec249dac7322b1152156c6012d9102160e830e374c92","auth_header":false},{"url":"https://blog.mcp.cloudflare.com/mcp","source":"mcp_card","outcome":"initialized","status":200,"ms":222,"initialized":true,"protocol_version":"2025-06-18","server":{"name":"cloudflare-blog","version":"0.2.4"},"tools":4,"tools_sha256":"4995c0bbb213a965db3f46d62682b9eb69b29c40d876b3bd3648716bf750927d","auth_header":false},{"url":"https://mcp.cloudflare.com/mcp","source":"mcp_card","outcome":"auth_required","status":401,"ms":31,"initialized":false,"protocol_version":null,"server":null,"tools":null,"tools_sha256":null,"auth_header":true}],"a2a":null,"webmcp":{"model_context":false,"bridge":false,"mcp_json":false,"link_rel":false,"any":false},"handshakes":3,"get_path":{"status":404,"content_type":"text/html","ms":299,"error":null}},"derived":{"reachable":true,"robots_served":true,"robots_sitemap":true,"content_signal":true,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":[],"blocks_any_ai":false,"files":{"llms_txt":true,"agents_md":false,"agent_skills":true,"mcp_card":true,"api_catalog":false,"a2a_card":false},"any_capability":true,"ai_refused":[],"ai_thin":[],"rings":{"mcp_declared":true,"mcp_live":true,"mcp_auth":true,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":2,"fh":"www.cloudflare.com","fp_other_host":[],"fp":"8913d00cc5dfeb9a","fp_bits":{"gptbot":0,"claudebot":0,"oaisearch":0,"perplexity":0}},"derive_v":3,"frame":"top1k","shard":1},{"v":1,"kind":"crawlcheck-census-record","run":14,"domain":"facebook.com","rank":3,"measured_at":"2026-10-15T05:28:03.923Z","home":{"browser":{"status":400,"content_type":"text/html","bytes":3676,"sha256":"832c9eaf02207c95a6c524a7a1e630fa2297135c0ef922706a1b161dc8781432","final_host":"m.facebook.com","ms":761,"identity":"Mobile browser","html":true},"gptbot":{"status":200,"content_type":"text/html","bytes":420270,"sha256":"ed54d56df090152710eed34ff4ed0d7d7c65e19e3d77f2f197306e3a98da7f44","final_host":"www.facebook.com","ms":1125,"identity":"GPTBot","html":true,"fp":"1e73949b66889ea0"},"claudebot":{"status":200,"content_type":"text/html","bytes":420174,"sha256":"98e6f74279208ad0d6f28f52ab31e53fc9ae4914998c3d58937e175e54364b79","final_host":"www.facebook.com","ms":550,"identity":"ClaudeBot","html":true,"fp":"1e73949b66889ea0"},"oaisearch":{"status":200,"content_type":"text/html","bytes":420279,"sha256":"e62c569ef35bea1cb153ab2f35736ccc5bbc82a82f9184880f45cdc2d3877054","final_host":"www.facebook.com","ms":702,"identity":"OAI-SearchBot","html":true,"fp":"1e73949b66889ea0"},"perplexity":{"status":200,"content_type":"text/html","bytes":375865,"sha256":"eef55bb9da91ec390534b0c95343aba4fc6644c069adf3e571816bce5b5e8917","final_host":"www.facebook.com","ms":518,"identity":"PerplexityBot","html":true}},"robots":{"status":200,"content_type":"text/plain","bytes":34030,"sha256":"4b7557970233aecb84d886a887d77f5e22bad763100a2f11c509d1239cb7d9a5","final_host":"facebook.com","ms":255,"html":false,"text":"# Notice: Collection of data on Facebook through automated means is\n# prohibited unless you have express written permission from Facebook\n# and may only be conducted for the limited purpose contained in said\n# permission.\n# All authorized user-agents listed on this page must comply with Meta’s\n# Automated Data Collection Terms available at:\n# https://www.facebook.com/legal/automated_data_collection_terms\nUser-agent: Google-Extended\nDisallow: /\n\nUser-agent: PetalBot\nDisallow: /\n\nUser-agent: Scrapy\nDisallow: /\n\nUser-agent: uptimerobot\nDisallow: /\n\nUser-agent: viberbot\nDisallow: /\n\nUser-agent: YaK\nDisallow: /\n\nUser-agent: Yandex\nDisallow: /\n\nUser-agent: Yeti\nDisallow: /\n\nUser-agent: Applebot\nDisallow: /*/plugins/*\nDisallow: /?*next=\nDisallow: /a/bz?\nDisallow: /ajax/\nDisallow: /album.php\nDisallow: /business/*&categories\nDisallow: /business/*?categories\nDisallow: /business/help/search*&query=\nDisallow: /business/help/search*?query=\nDisallow: /business/news*&tags\nDisallow: /business/news*?tags\nDisallow: /business/search*&q=\nDisallow: /business/search*?q=\nDisallow: /checkpoint/\nDisallow: /contact_importer/\nDisallow: /dialog/\nDisallow: /fbml/ajax/dialog/\nDisallow: /feeds/\nDisallow: /file_download.php\nDisallow: /job_application/\nDisallow: /l.php\nDisallow: /login.php*&next=\nDisallow: /login.php/?next=\nDisallow: /login.php?next=\nDisallow: /login/*&next=\nDisallow: /login/?next=\nDisallow: /login/device-based/regular/login/*&next=\nDisallow: /login/device-based/regular/login/?next=\nDisallow: /meta-ai-answers/\nDisallow: /moments_app/\nDisallow: /p.php\nDisallow: /photos.php\nDisallow: /plugins/\nDisallow: /share.php\nDisallow: /share/\nDisallow: /sharer.php\nDisallow: /sharer/\nDisallow: /tr/\nDisallow: /tr?\nDisallow: /ufi/reaction/profile/browser/\nDisallow: /x/oauth/\nAllow: /ajax/bootloader-endpoint/\nAllow: /ajax/pagelet/generic.php/PagePostsSectionPagelet\nAllow: /careers/\nAllow: /safetycheck/\n\nUser-agent: Bingbot\nDisallow: /*/plugins/*\nDisallow: /?*next=\nDisallow: /a/bz?\nDisallow: /ajax/\nDisallow: /album.php\nDisallow: /business/*&categories\nDisallow: /business/*?categories\nDisallow: /business/help/search*&query=\nDisallow: /business/help/search*?query=\nDisallow: /business/news*&tags\nDisallow: /business/news*?tags\nDisallow: /business/search*&q=\nDisallow: /business/search*?q=\nDisallow: /checkpoint/\nDisallow: /contact_importer/\nDisallow: /dialog/\nDisallow: /fbml/ajax/dialog/\nDisallow: /feeds/\nDisallow: /file_download.php\nDisallow: /job_application/\nDisallow: /l.php\nDisallow: /login.php*&next=\nDisallow: /login.php/?next=\nDisallow: /login.php?next=\nDisallow: /login/*&next=\nDisallow: /login/?next=\nDisallow: /login/device-based/regular/login/*&next=\nDisallow: /login/device-based/regular/login/?next=\nDisallow: /meta-ai-answers/\nDisallow: /moments_app/\nDisallow: /p.php\nDisallow: /photos.php\nDisallow: /plugins/\nDisallow: /share.php\nDisallow: /share/\nDisallow: /sharer.php\nDisallow: /sharer/\nDisallow: /tr/\nDisallow: /tr?\nDisallow: /ufi/reaction/profile/browser/\nDisallow: /x/oauth/\nAllow: /ajax/bootloader-endpoint/\nAllow: /ajax/pagelet/generic.php/PagePostsSectionPagelet\nAllow: /careers/\nAllow: /safetycheck/\n\nUser-agent: deepcrawl\nDisallow: /\nDisallow: /*/plugins/*\nDisallow: /?*next=\nDisallow: /a/bz?\nDisallow: /ajax/\nDisallow: /album.php\nDisallow: /business/*&categories\nDisallow: /business/*?categories\nDisallow: /business/help/search*&query=\nDisallow: /business/help/search*?query=\nDisallow: /business/news*&tags\nDisallow: /business/news*?tags\nDisallow: /business/search*&q=\nDisallow: /business/search*?q=\nDisallow: /checkpoint/\nDisallow: /contact_importer/\nDisallow: /dialog/\nDisallow: /fbml/ajax/dialog/\nDisallow: /feeds/\nDisallow: /file_download.php\nDisallow: /job_application/\nDisallow: /l.php\nDisallow: /login.php*&next=\nDisallow: /login.php/?next=\nDisallow: /login.php?next=\nDisallow: /login/*&next=\nDisallow: /login/?next=\nDisallow: /login/device-based/regular/login/*&next=\nDisallow: /login/device-based/regular/login/?next=\nDisallow: /meta-ai-answers/\nDisallow: /moments_app/\nDisallow: /p.php\nDisallow: /photos.php\nDisallow: /plugins/\nDisallow: /share.php\nDisallow: /share/\nDisallow: /sharer.php\nDisallow: /sharer/\nDisallow: /tr/\nDisallow: /tr?\nDisallow: /ufi/reaction/profile/browser/\nDisallow: /x/oauth/\nAllow: /ajax/bootloader-endpoint/\nAllow: /ajax/pagelet/generic.php/PagePostsSectionPagelet\nAllow: /business/\nAllow: /careers/\nAllow: /safetycheck/\n\nUser-agent: Discordbot\nDisallow: /*/plugins/*\nDisallow: /?*next=\nDisallow: /a/bz?\nDisallow: /ajax/\nDisallow: /album.php\nDisallow: /business/*&categories\nDisallow: /business/*?categories\nDisallow: /business/help/search*&query=\nDisallow: /business/help/search*?query=\nDisallow: /business/news*&tags\nDisallow: /business/news*?tags\nDisallow: /business/search*&q=\nDisallow: /business/search*?q=\nDisallow: /checkpoint/\nDisallow: /contact_importer/\nDisallow: /dialog/\nDisallow: /fbml/ajax/dialog/\nDisallow: /feeds/\nDisallow: /file_download.php\nDisallow: /job_application/\nDisallow: /l.php\nDisallow: /login.php*&next=\nDisallow: /login.php/?next=\nDisallow: /login.php?next=\nDisallow: /login/*&next=\nDisallow: /login/?next=\nDisallow: /login/device-based/regular/login/*&next=\nDisallow: /login/device-based/regular/login/?next=\nDisallow: /meta-ai-answers/\nDisallow: /moments_app/\nDisallow: /p.php\nDisallow: /photos.php\nDisallow: /plugins/\nDisallow: /share.php\nDisallow: /share/\nDisallow: /sharer.php\nDisallow: /sharer/\nDisallow: /tr/\nDisallow: /tr?\nDisallow: /ufi/reaction/profile/browser/\nDisallow: /x/oauth/\nAllow: /ajax/bootloader-endpoint/\nAllow: /ajax/pagelet/generic.php/PagePostsSectionPagelet\nAllow: /careers/\nAllow: /safetycheck/\n\nUser-agent: DuckDuckBot\nDisallow: /*/plugins/*\nDisallow: /?*next=\nDisallow: /a/bz?\nDisallow: /ajax/\nDisallow: /album.php\nDisallow: /business/*&categories\nDisallow: /business/*?categories\nDisallow: /business/help/search*&query=\nDisallow: /business/help/search*?query=\nDisallow: /business/news*&tags\nDisallow: /business/news*?tags\nDisallow: /business/search*&q=\nDisallow: /business/search*?q=\nDisallow: /checkpoint/\nDisallow: /contact_importer/\nDisallow: /dialog/\nDisallow: /fbml/ajax/dialog/\nDisallow: /feeds/\nDisallow: /file_download.php\nDisallow: /job_application/\nDisallow: /l.php\nDisallow: /login.php*&next=\nDisallow: /login.php/?next=\nDisallow: /login.php?next=\nDisallow: /login/*&next=\nDisallow: /login/?next=\nDisallow: /login/device-based/regular/login/*&next=\nDisallow: /login/device-based/regular/login/?next=\nDisallow: /meta-ai-answers/\nDisallow: /moments_app/\nDisallow: /p.php\nDisallow: /photos.php\nDisallow: /plugins/\nDisallow: /share.php\nDisallow: /share/\nDisallow: /sharer.php\nDisallow: /sharer/\nDisallow: /tr/\nDisallow: /tr?\nDisallow: /ufi/reaction/profile/browser/\nDisallow: /x/oauth/\nAllow: /ajax/bootloader-endpoint/\nAllow: /ajax/pagelet/generic.php/PagePostsSectionPagelet\nAllow: /careers/\nAllow: /safetycheck/\n\nUser-agent: facebookexternalhit\nDisallow: /*/plugins/*\nDisallow: /?*next=\nDisallow: /a/bz?\nDisallow: /ajax/\nDisallow: /album.php\nDisallow: /business/*&categories\nDisallow: /business/*?categories\nDisallow: /business/help/search*&query=\nDisallow: /business/help/search*?query=\nDisallow: /business/news*&tags\nDisallow: /business/news*?tags\nDisallow: /business/search*&q=\nDisallow: /business/search*?q=\nDisallow: /checkpoint/\nDisallow: /contact_importer/\nDisallow: /dialog/\nDisallow: /fbml/ajax/dialog/\nDisallow: /feeds/\nDisallow: /file_download.php\nDisallow: /job_application/\nDisallow: /l.php\nDisallow: /login.php*&next=\nDisallow: /login.php/?next=\nDisallow: /login.php?next=\nDisallow: /login/*&next=\nDisallow: /login/?next=\nDisallow: /login/device-based/regular/login/*&next=\nDisallow: /login/device-based/regular/login/?next=\nDisallow: /meta-ai-answers/\nDisallow: /moments_app/\nDisallow: /p.php\nDisallow: /photos.php\nDisallow: /plugins/\nDisallow: /share.php\nDisallow: /share/\nDisallow: /sharer.php\nDisallow: /sharer/\nDisallow: /tr/\nDisallow: /tr?\nDisallow: /ufi/reaction/profile/browser/\nDisallow: /x/oauth/\nAllow: /ajax/bootloader-endpoint/\nAllow: /ajax/pagelet/generic.php/PagePostsSectionPagelet\nAllow: /careers/\nAllow: /safetycheck/\n\nUser-agent: Googlebot\nDisallow: /*/plugins/*\nDisallow: /?*next=\nDisallow: /a/bz?\nDisallow: /ajax/\nDisallow: /album.php\nDisallow: /business/*&categories\nDisallow: /business/*?categories\nDisallow: /business/help/search*&query=\nDisallow: /business/help/search*?query=\nDisallow: /business/news*&tags\nDisallow: /business/news*?tags\nDisallow: /business/search*&q=\nDisallow: /business/search*?q=\nDisallow: /checkpoint/\nDisallow: /contact_importer/\nDisallow: /dialog/\nDisallow: /fbml/ajax/dialog/\nDisallow: /feeds/\nDisallow: /file_download.php\nDisallow: /job_application/\nDisallow: /l.php\nDisallow: /login.php*&next=\nDisallow: /login.php/?next=\nDisallow: /login.php?next=\nDisallow: /login/*&next=\nDisallow: /login/?next=\nDisallow: /login/device-based/regular/login/*&next=\nDisallow: /login/device-based/regular/login/?next=\nDisallow: /meta-ai-answers/\nDisallow: /moments_app/\nDisallow: /p.php\nDisallow: /photos.php\nDisallow: /plugins/\nDisallow: /share.php\nDisallow: /share/\nDisallow: /sharer.php\nDisallow: /sharer/\nDisallow: /tr/\nDisallow: /tr?\nDisallow: /ufi/reaction/profile/browser/\nDisallow: /x/oauth/\nAllow: /ajax/bootloader-endpoint/\nAllow: /ajax/pagelet/generic.php/PagePostsSectionPagelet\nAllow: /careers/\nAllow: /safetycheck/\n\nUser-agent: Googlebot-Image\nDisallow: /*/plugins/*\nDisallow: /?*next=\nDisallow: /a/bz?\nDisallow: /ajax/\nDisallow: /album.php\nDisallow: /business/*&categories\nDisallow: /business/*?categories\nDisallow: /business/help/search*&query=\nDisallow: /business/help/search*?query=\nDisallow: /business/news*&tags\nDisallow: /business/news*?tags\nDisallow: /business/search*&q=\nDisallow: /business/search*?q=\nDisallow: /checkpoint/\nDisallow: /contact_importer/\nDisallow: /dialog/\nDisallow: /fbml/ajax/dialog/\nDisallow: /feeds/\nDisallow: /file_download.php\nDisallow: /job_application/\nDisallow: /l.php\nDisallow: /login.php*&next=\nDisallow: /login.php/?next=\nDisallow: /login.php?next=\nDisallow: /login/*&next=\nDisallow: /login/?next=\nDisallow: /login/device-based/regular/login/*&next=\nDisallow: /login/device-based/regular/login/?next=\nDisallow: /map_tile.php\nDisallow: /meta-ai-answers/\nDisallow: /moments_app/\nDisallow: /p.php\nDisallow: /photos.php\nDisallow: /plugins/\nDisallow: /share.php\nDisallow: /share/\nDisallow: /sharer.php\nDisallow: /sharer/\nDisallow: /static_map.php\nDisallow: /tr/\nDisallow: /tr?\nDisallow: /ufi/reaction/profile/browser/\nDisallow: /x/oauth/\nAllow: /ajax/bootloader-endpoint/\nAllow: /ajax/pagelet/generic.php/PagePostsSectionPagelet\nAllow: /careers/\nAllow: /safetycheck/\n\nUser-agent: Googlebot-Video\nDisallow: /*/plugins/*\nDisallow: /?*next=\nDisallow: /a/bz?\nDisallow: /ajax/\nDisallow: /album.php\nDisallow: /business/*&categories\nDisallow: /business/*?categories\nDisallow: /business/help/search*&query=\nDisallow: /business/help/search*?query=\nDisallow: /business/news*&tags\nDisallow: /business/news*?tags\nDisallow: /business/search*&q=\nDisallow: /business/search*?q=\nDisallow: /checkpoint/\nDisallow: /contact_importer/\nDisallow: /dialog/\nDisallow: /fbml/ajax/dialog/\nDisallow: /feeds/\nDisallow: /file_download.php\nDisallow: /job_application/\nDisallow: /l.php\nDisallow: /login.php*&next=\nDisallow: /login.php/?next=\nDisallow: /login.php?next=\nDisallow: /login/*&next=\nDisallow: /login/?next=\nDisallow: /login/device-based/regular/login/*&next=\nDisallow: /login/device-based/regular/login/?next=\nDisallow: /meta-ai-answers/\nDisallow: /moments_app/\nDisallow: /p.php\nDisallow: /photos.php\nDisallow: /plugins/\nDisallow: /share.php\nDisallow: /share/\nDisallow: /sharer.php\nDisallow: /sharer/\nDisallow: /tr/\nDisallow: /tr?\nDisallow: /ufi/reaction/profile/browser/\nDisallow: /x/oauth/\nAllow: /ajax/bootloader-endpoint/\nAllow: /ajax/pagelet/generic.php/PagePostsSectionPagelet\nAllow: /careers/\nAllow: /safetycheck/\n\nUser-agent: Googlebot-News\nDisallow: /*/plugins/*\nDisallow: /?*next=\nDisallow: /a/bz?\nDisallow: /ajax/\nDisallow: /album.php\nDisallow: /business/*&categories\nDisallow: /business/*?categories\nDisallow: /business/help/search*&query=\nDisallow: /business/help/search*?query=\nDisallow: /business/news*&tags\nDisallow: /business/news*?tags\nDisallow: /business/search*&q=\nDisallow: /business/search*?q=\nDisallow: /checkpoint/\nDisallow: /contact_importer/\nDisallow: /dialog/\nDisallow: /fbml/ajax/dialog/\nDisallow: /feeds/\nDisallow: /file_download.php\nDisallow: /job_application/\nDisallow: /l.php\nDisallow: /login.php*&next=\nDisallow: /login.php/?next=\nDisallow: /login.php?next=\nDisallow: /login/*&next=\nDisallow: /login/?next=\nDisallow: /login/device-based/regular/login/*&next=\nDisallow: /login/device-based/regular/login/?next=\nDisallow: /meta-ai-answers/\nDisallow: /moments_app/\nDisallow: /p.php\nDisallow: /photos.php\nDisallow: /plugins/\nDisallow: /share.php\nDisallow: /share/\nDisallow: /sharer.php\nDisallow: /sharer/\nDisallow: /tr/\nDisallow: /tr?\nDisallow: /ufi/reaction/profile/browser/\nDisallow: /x/oauth/\nAllow: /ajax/bootloader-endpoint/\nAllow: /ajax/pagelet/generic.php/PagePostsSectionPagelet\nAllow: /careers/\nAllow: /safetycheck/\n\nUser-agent: Google-InspectionTool\nDisallow: /*/plugins/*\nDisallow: /?*next=\nDisallow: /a/bz?\nDisallow: /ajax/\nDisallow: /album.php\nDisallow: /business/*&categories\nDisallow: /business/*?categories\nDisallow: /business/help/search*&query=\nDisallow: /business/help/search*?query=\nDisallow: /business/news*&tags\nDisallow: /business/news*?tags\nDisallow: /business/search*&q=\nDisallow: /business/search*?q=\nDisallow: /checkpoint/\nDisallow: /contact_importer/\nDisallow: /dialog/\nDisallow: /fbml/ajax/dialog/\nDisallow: /feeds/\nDisallow: /file_download.php\nDisallow: /job_application/\nDisallow: /l.php\nDisallow: /login.php*&next=\nDisallow: /login.php/?next=\nDisallow: /login.php?next=\nDisallow: /login/*&next=\nDisallow: /login/?next=\nDisallow: /login/device-based/regular/login/*&next=\nDisallow: /login/device-based/regular/login/?next=\nDisallow: /meta-ai-answers/\nDisallow: /moments_app/\nDisallow: /p.php\nDisallow: /photos.php\nDisallow: /plugins/\nDisallow: /share.php\nDisallow: /share/\nDisallow: /sharer.php\nDisallow: /sharer/\nDisallow: /tr/\nDisallow: /tr?\nDisallow: /ufi/reaction/profile/browser/\nDisallow: /x/oauth/\nAllow: /ajax/bootloader-endpoint/\nAllow: /ajax/pagelet/generic.php/PagePostsSectionPagelet\nAllow: /careers/\nAllow: /safetycheck/\n\nUser-agent: lumar\nDisallow: /\nDisallow: /*/plugins/*\nDisallow: /?*next=\nDisallow: /a/bz?\nDisallow: /ajax/\nDisallow: /album.php\nDisallow: /business/*&categories\nDisallow: /business/*?categories\nDisallow: /business/help/search*&query=\nDisallow: /business/help/search*?query=\nDisallow: /business/news*&tags\nDisallow: /business/news*?tags\nDisallow: /business/search*&q=\nDisallow: /business/search*?q=\nDisallow: /checkpoint/\nDisallow: /contact_importer/\nDisallow: /dialog/\nDisallow: /fbml/ajax/dialog/\nDisallow: /feeds/\nDisallow: /file_download.php\nDisallow: /job_application/\nDisallow: /l.php\nDisallow: /login.php*&next=\nDisallow: /login.php/?next=\nDisallow: /login.php?next=\nDisallow: /login/*&next=\nDisallow: /login/?next=\nDisallow: /login/device-based/regular/login/*&next=\nDisallow: /login/device-based/regular/login/?next=\nDisallow: /meta-ai-answers/\nDisallow: /moments_app/\nDisallow: /p.php\nDisallow: /photos.php\nDisallow: /plugins/\nDisallow: /share.php\nDisallow: /share/\nDisallow: /sharer.php\nDisallow: /sharer/\nDisallow: /tr/\nDisallow: /tr?\nDisallow: /ufi/reaction/profile/browser/\nDisallow: /x/oauth/\nAllow: /ajax/bootloader-endpoint/\nAllow: /ajax/pagelet/generic.php/PagePostsSectionPagelet\nAllow: /business/\nAllow: /careers/\nAllow: /safetycheck/\n\nUser-agent: LinkedInBot\nDisallow: /*/plugins/*\nDisallow: /?*next=\nDisallow: /a/bz?\nDisallow: /ajax/\nDisallow: /album.php\nDisallow: /business/*&categories\nDisallow: /business/*?categories\nDisallow: /business/help/search*&query=\nDisallow: /business/help/search*?query=\nDisallow: /business/news*&tags\nDisallow: /business/news*?tags\nDisallow: /business/search*&q=\nDisallow: /business/search*?q=\nDisallow: /checkpoint/\nDisallow: /contact_importer/\nDisallow: /dialog/\nDisallow: /fbml/ajax/dialog/\nDisallow: /feeds/\nDisallow: /file_download.php\nDisallow: /job_application/\nDisallow: /l.php\nDisallow: /login.php*&next=\nDisallow: /login.php/?next=\nDisallow: /login.php?next=\nDisallow: /login/*&next=\nDisallow: /login/?next=\nDisallow: /login/device-based/regular/login/*&next=\nDisallow: /login/device-based/regular/login/?next=\nDisallow: /meta-ai-answers/\nDisallow: /moments_app/\nDisallow: /p.php\nDisallow: /photos.php\nDisallow: /plugins/\nDisallow: /share.php\nDisallow: /share/\nDisallow: /sharer.php\nDisallow: /sharer/\nDisallow: /tr/\nDisallow: /tr?\nDisallow: /ufi/reaction/profile/browser/\nDisallow: /x/oauth/\nAllow: /ajax/bootloader-endpoint/\nAllow: /ajax/pagelet/generic.php/PagePostsSectionPagelet\nAllow: /careers/\nAllow: /safetycheck/\n\nUser-agent: msnbot\nDisallow: /*/plugins/*\nDisallow: /?*next=\nDisallow: /a/bz?\nDisallow: /ajax/\nDisallow: /album.php\nDisallow: /business/*&categories\nDisallow: /business/*?categories\nDisallow: /business/help/search*&query=\nDisallow: /business/help/search*?query=\nDisallow: /business/news*&tags\nDisallow: /business/news*?tags\nDisallow: /business/search*&q=\nDisallow: /business/search*?q=\nDisallow: /checkpoint/\nDisallow: /contact_importer/\nDisallow: /dialog/\nDisallow: /fbml/ajax/dialog/\nDisallow: /feeds/\nDisallow: /file_download.php\nDisallow: /job_application/\nDisallow: /l.php\nDisallow: /login.php*&next=\nDisallow: /login.php/?next=\nDisallow: /login.php?next=\nDisallow: /login/*&next=\nDisallow: /login/?next=\nDisallow: /login/device-based/regular/login/*&next=\nDisallow: /login/device-based/regular/login/?next=\nDisallow: /meta-ai-answers/\nDisallow: /moments_app/\nDisallow: /p.php\nDisallow: /photos.php\nDisallow: /plugins/\nDisallow: /share.php\nDisallow: /share/\nDisallow: /sharer.php\nDisallow: /sharer/\nDisallow: /tr/\nDisallow: /tr?\nDisallow: /ufi/reaction/profile/browser/\nDisallow: /x/oauth/\nAllow: /ajax/bootloader-endpoint/\nAllow: /ajax/pagelet/generic.php/PagePostsSectionPagelet\nAllow: /careers/\nAllow: /safetycheck/\n\nUser-agent: Pinterestbot\nDisallow: /*/plugins/*\nDisallow: /?*next=\nDisallow: /a/bz?\nDisallow: /ajax/\nDisallow: /album.php\nDisallow: /business/*&categories\nDisallow: /business/*?categories\nDisallow: /business/help/search*&query=\nDisallow: /business/help/search*?query=\nDisallow: /business/news*&tags\nDisallow: /business/news*?tags\nDisallow: /business/search*&q=\nDisallow: /business/search*?q=\nDisallow: /checkpoint/\nDisallow: /contact_importer/\nDisallow: /dialog/\nDisallow: /fbml/ajax/dialog/\nDisallow: /feeds/\nDisallow: /file_download.php\nDisallow: /job_application/\nDisallow: /l.php\nDisallow: /login.php*&next=\nDisallow: /login.php/?next=\nDisallow: /login.php?next=\nDisallow: /login/*&next=\nDisallow: /login/?next=\nDisallow: /login/device-based/regular/login/*&next=\nDisallow: /login/device-based/regular/login/?next=\nDisallow: /meta-ai-answers/\nDisallow: /moments_app/\nDisallow: /p.php\nDisallow: /photos.php\nDisallow: /plugins/\nDisallow: /share.php\nDisallow: /share/\nDisallow: /sharer.php\nDisallow: /sharer/\nDisallow: /tr/\nDisallow: /tr?\nDisallow: /ufi/reaction/profile/browser/\nDisallow: /x/oauth/\nAllow: /ajax/bootloader-endpoint/\nAllow: /ajax/pagelet/generic.php/PagePostsSectionPagelet\nAllow: /careers/\nAllow: /safetycheck/\n\nUser-agent: Screaming Frog SEO Spider\nDisallow: /*/plugins/*\nDisallow: /?*next=\nDisallow: /a/bz?\nDisallow: /ajax/\nDisallow: /album.php\nDisallow: /business/*&categories\nDisallow: /business/*?categories\nDisallow: /business/help/search*&query=\nDisallow: /business/help/search*?query=\nDisallow: /business/news*&tags\nDisallow: /business/news*?tags\nDisallow: /business/search*&q=\nDisallow: /business/search*?q=\nDisallow: /checkpoint/\nDisallow: /contact_importer/\nDisallow: /dialog/\nDisallow: /fbml/ajax/dialog/\nDisallow: /feeds/\nDisallow: /file_download.php\nDisallow: /job_application/\nDisallow: /l.php\nDisallow: /login.php*&next=\nDisallow: /login.php/?next=\nDisallow: /login.php?next=\nDisallow: /login/*&next=\nDisallow: /login/?next=\nDisallow: /login/device-based/regular/login/*&next=\nDisallow: /login/device-based/regular/login/?next=\nDisallow: /meta-ai-answers/\nDisallow: /moments_app/\nDisallow: /p.php\nDisallow: /photos.php\nDisallow: /plugins/\nDisallow: /share.php\nDisallow: /share/\nDisallow: /sharer.php\nDisallow: /sharer/\nDisallow: /tr/\nDisallow: /tr?\nDisallow: /ufi/reaction/profile/browser/\nDisallow: /x/oauth/\nAllow: /ajax/bootloader-endpoint/\nAllow: /ajax/pagelet/generic.php/PagePostsSectionPagelet\nAllow: /careers/\nAllow: /safetycheck/\n\nUser-agent: seznambot\nDisallow: /*/plugins/*\nDisallow: /?*next=\nDisallow: /a/bz?\nDisallow: /ajax/\nDisallow: /album.php\nDisallow: /business/*&categories\nDisallow: /business/*?categories\nDisallow: /business/help/search*&query=\nDisallow: /business/help/search*?query=\nDisallow: /business/news*&tags\nDisallow: /business/news*?tags\nDisallow: /business/search*&q=\nDisallow: /business/search*?q=\nDisallow: /checkpoint/\nDisallow: /contact_importer/\nDisallow: /dialog/\nDisallow: /fbml/ajax/dialog/\nDisallow: /feeds/\nDisallow: /file_download.php\nDisallow: /job_application/\nDisallow: /l.php\nDisallow: /login.php*&next=\nDisallow: /login.php/?next=\nDisallow: /login.php?next=\nDisallow: /login/*&next=\nDisallow: /login/?next=\nDisallow: /login/device-based/regular/login/*&next=\nDisallow: /login/device-based/regular/login/?next=\nDisallow: /meta-ai-answers/\nDisallow: /moments_app/\nDisallow: /p.php\nDisallow: /photos.php\nDisallow: /plugins/\nDisallow: /share.php\nDisallow: /share/\nDisallow: /sharer.php\nDisallow: /sharer/\nDisallow: /tr/\nDisallow: /tr?\nDisallow: /ufi/reaction/profile/browser/\nDisallow: /x/oauth/\nAllow: /ajax/bootloader-endpoint/\nAllow: /ajax/pagelet/generic.php/PagePostsSectionPagelet\nAllow: /careers/\nAllow: /safetycheck/\n\nUser-agent: Slurp\nDisallow: /*/plugins/*\nDisallow: /?*next=\nDisallow: /a/bz?\nDisallow: /ajax/\nDisallow: /album.php\nDisallow: /business/*&categories\nDisallow: /business/*?categories\nDisallow: /business/help/search*&query=\nDisallow: /business/help/search*?query=\nDisallow: /business/news*&tags\nDisallow: /business/news*?tags\nDisallow: /business/search*&q=\nDisallow: /business/search*?q=\nDisallow: /checkpoint/\nDisallow: /contact_importer/\nDisallow: /dialog/\nDisallow: /fbml/ajax/dialog/\nDisallow: /feeds/\nDisallow: /file_download.php\nDisallow: /job_application/\nDisallow: /l.php\nDisallow: /login.php*&next=\nDisallow: /login.php/?next=\nDisallow: /login.php?next=\nDisallow: /login/*&next=\nDisallow: /login/?next=\nDisallow: /login/device-based/regular/login/*&next=\nDisallow: /login/device-based/regular/login/?next=\nDisallow: /meta-ai-answers/\nDisallow: /moments_app/\nDisallow: /p.php\nDisallow: /photos.php\nDisallow: /plugins/\nDisallow: /share.php\nDisallow: /share/\nDisallow: /sharer.php\nDisallow: /sharer/\nDisallow: /tr/\nDisallow: /tr?\nDisallow: /ufi/reaction/profile/browser/\nDisallow: /x/oauth/\nAllow: /ajax/bootloader-endpoint/\nAllow: /ajax/pagelet/generic.php/PagePostsSectionPagelet\nAllow: /careers/\nAllow: /safetycheck/\n\nUser-agent: teoma\nDisallow: /*/plugins/*\nDisallow: /?*next=\nDisallow: /a/bz?\nDisallow: /ajax/\nDisallow: /album.php\nDisallow: /business/*&categories\nDisallow: /business/*?categories\nDisallow: /business/help/search*&query=\nDisallow: /business/help/search*?query=\nDisallow: /business/news*&tags\nDisallow: /business/news*?tags\nDisallow: /business/search*&q=\nDisallow: /business/search*?q=\nDisallow: /checkpoint/\nDisallow: /contact_importer/\nDisallow: /dialog/\nDisallow: /fbml/ajax/dialog/\nDisallow: /feeds/\nDisallow: /file_download.php\nDisallow: /job_application/\nDisallow: /l.php\nDisallow: /login.php*&next=\nDisallow: /login.php/?next=\nDisallow: /login.php?next=\nDisallow: /login/*&next=\nDisallow: /login/?next=\nDisallow: /login/device-based/regular/login/*&next=\nDisallow: /login/device-based/regular/login/?next=\nDisallow: /meta-ai-answers/\nDisallow: /moments_app/\nDisallow: /p.php\nDisallow: /photos.php\nDisallow: /plugins/\nDisallow: /share.php\nDisallow: /share/\nDisallow: /sharer.php\nDisallow: /sharer/\nDisallow: /tr/\nDisallow: /tr?\nDisallow: /ufi/reaction/profile/browser/\nDisallow: /x/oauth/\nAllow: /ajax/bootloader-endpoint/\nAllow: /ajax/pagelet/generic.php/PagePostsSectionPagelet\nAllow: /careers/\nAllow: /safetycheck/\n\nUser-agent: TelegramBot\nDisallow: /*/plugins/*\nDisallow: /?*next=\nDisallow: /a/bz?\nDisallow: /ajax/\nDisallow: /album.php\nDisallow: /business/*&categories\nDisallow: /business/*?categories\nDisallow: /business/help/search*&query=\nDisallow: /business/help/search*?query=\nDisallow: /business/news*&tags\nDisallow: /business/news*?tags\nDisallow: /business/search*&q=\nDisallow: /business/search*?q=\nDisallow: /checkpoint/\nDisallow: /contact_importer/\nDisallow: /dialog/\nDisallow: /fbml/ajax/dialog/\nDisallow: /feeds/\nDisallow: /file_download.php\nDisallow: /job_application/\nDisallow: /l.php\nDisallow: /login.php*&next=\nDisallow: /login.php/?next=\nDisallow: /login.php?next=\nDisallow: /login/*&next=\nDisallow: /login/?next=\nDisallow: /login/device-based/regular/login/*&next=\nDisallow: /login/device-based/regular/login/?next=\nDisallow: /meta-ai-answers/\nDisallow: /moments_app/\nDisallow: /p.php\nDisallow: /photos.php\nDisallow: /plugins/\nDisallow: /share.php\nDisallow: /share/\nDisallow: /sharer.php\nDisallow: /sharer/\nDisallow: /tr/\nDisallow: /tr?\nDisallow: /ufi/reaction/profile/browser/\nDisallow: /x/oauth/\nAllow: /ajax/bootloader-endpoint/\nAllow: /ajax/pagelet/generic.php/PagePostsSectionPagelet\nAllow: /careers/\nAllow: /safetycheck/\n\nUser-agent: Twitterbot\nDisallow: /*/plugins/*\nDisallow: /?*next=\nDisallow: /a/bz?\nDisallow: /ajax/\nDisallow: /album.php\nDisallow: /business/*&categories\nDisallow: /business/*?categories\nDisallow: /business/help/search*&query=\nDisallow: /business/help/search*?query=\nDisallow: /business/news*&tags\nDisallow: /business/news*?tags\nDisallow: /business/search*&q=\nDisallow: /business/search*?q=\nDisallow: /checkpoint/\nDisallow: /contact_importer/\nDisallow: /dialog/\nDisallow: /fbml/ajax/dialog/\nDisallow: /feeds/\nDisallow: /file_download.php\nDisallow: /job_application/\nDisallow: /l.php\nDisallow: /login.php*&next=\nDisallow: /login.php/?next=\nDisallow: /login.php?next=\nDisallow: /login/*&next=\nDisallow: /login/?next=\nDisallow: /login/device-based/regular/login/*&next=\nDisallow: /login/device-based/regular/login/?next=\nDisallow: /meta-ai-answers/\nDisallow: /moments_app/\nDisallow: /p.php\nDisallow: /photos.php\nDisallow: /plugins/\nDisallow: /share.php\nDisallow: /share/\nDisallow: /sharer.php\nDisallow: /sharer/\nDisallow: /tr/\nDisallow: /tr?\nDisallow: /ufi/reaction/profile/browser/\nDisallow: /x/oauth/\nAllow: /ajax/bootloader-endpoint/\nAllow: /ajax/pagelet/generic.php/PagePostsSectionPagelet\nAllow: /careers/\nAllow: /safetycheck/\n\nUser-agent: Amazonbot\nDisallow: /*/plugins/*\nDisallow: /?*next=\nDisallow: /a/bz?\nDisallow: /ajax/\nDisallow: /album.php\nDisallow: /business/*&categories\nDisallow: /business/*?categories\nDisallow: /business/help/search*&query=\nDisallow: /business/help/search*?query=\nDisallow: /business/news*&tags\nDisallow: /business/news*?tags\nDisallow: /business/search*&q=\nDisallow: /business/search*?q=\nDisallow: /checkpoint/\nDisallow: /contact_importer/\nDisallow: /dialog/\nDisallow: /fbml/ajax/dialog/\nDisallow: /feeds/\nDisallow: /file_download.php\nDisallow: /job_application/\nDisallow: /l.php\nDisallow: /login.php*&next=\nDisallow: /login.php/?next=\nDisallow: /login.php?next=\nDisallow: /login/*&next=\nDisallow: /login/?next=\nDisallow: /login/device-based/regular/login/*&next=\nDisallow: /login/device-based/regular/login/?next=\nDisallow: /meta-ai-answers/\nDisallow: /moments_app/\nDisallow: /p.php\nDisallow: /photos.php\nDisallow: /plugins/\nDisallow: /share.php\nDisallow: /share/\nDisallow: /sharer.php\nDisallow: /sharer/\nDisallow: /tr/\nDisallow: /tr?\nDisallow: /ufi/reaction/profile/browser/\nDisallow: /x/oauth/\nAllow: /ajax/bootloader-endpoint/\nAllow: /ajax/pagelet/generic.php/PagePostsSectionPagelet\nAllow: /careers/\nAllow: /safetycheck/\n\nUser-agent: Applebot-Extended\nDisallow: /*/plugins/*\nDisallow: /?*next=\nDisallow: /a/bz?\nDisallow: /ajax/\nDisallow: /album.php\nDisallow: /business/*&categories\nDisallow: /business/*?categories\nDisallow: /business/help/search*&query=\nDisallow: /business/help/search*?query=\nDisallow: /business/news*&tags\nDisallow: /business/news*?tags\nDisallow: /business/search*&q=\nDisallow: /business/search*?q=\nDisallow: /checkpoint/\nDisallow: /contact_importer/\nDisallow: /dialog/\nDisallow: /fbml/ajax/dialog/\nDisallow: /feeds/\nDisallow: /file_download.php\nDisallow: /job_application/\nDisallow: /l.php\nDisallow: /login.php*&next=\nDisallow: /login.php/?next=\nDisallow: /login.php?next=\nDisallow: /login/*&next=\nDisallow: /login/?next=\nDisallow: /login/device-based/regular/login/*&next=\nDisallow: /login/device-based/regular/login/?next=\nDisallow: /meta-ai-answers/\nDisallow: /moments_app/\nDisallow: /p.php\nDisallow: /photos.php\nDisallow: /plugins/\nDisallow: /share.php\nDisallow: /share/\nDisallow: /sharer.php\nDisallow: /sharer/\nDisallow: /tr/\nDisallow: /tr?\nDisallow: /ufi/reaction/profile/browser/\nDisallow: /x/oauth/\nAllow: /ajax/bootloader-endpoint/\nAllow: /ajax/pagelet/generic.php/PagePostsSectionPagelet\nAllow: /careers/\nAllow: /safetycheck/\n\nUser-agent: Brightbot\nDisallow: /*/plugins/*\nDisallow: /?*next=\nDisallow: /a/bz?\nDisallow: /ajax/\nDisallow: /album.php\nDisallow: /business/*&categories\nDisallow: /business/*?categories\nDisallow: /business/help/search*&query=\nDisallow: /business/help/search*?query=\nDisallow: /business/news*&tags\nDisallow: /business/news*?tags\nDisallow: /business/search*&q=\nDisallow: /business/search*?q=\nDisallow: /checkpoint/\nDisallow: /contact_importer/\nDisallow: /dialog/\nDisallow: /fbml/ajax/dialog/\nDisallow: /feeds/\nDisallow: /file_download.php\nDisallow: /job_application/\nDisallow: /l.php\nDisallow: /login.php*&next=\nDisallow: /login.php/?next=\nDisallow: /login.php?next=\nDisallow: /login/*&next=\nDisallow: /login/?next=\nDisallow: /login/device-based/regular/login/*&next=\nDisallow: /login/device-based/regular/login/?next=\nDisallow: /meta-ai-answers/\nDisallow: /moments_app/\nDisallow: /p.php\nDisallow: /photos.php\nDisallow: /plugins/\nDisallow: /share.php\nDisallow: /share/\nDisallow: /sharer.php\nDisallow: /sharer/\nDisallow: /tr/\nDisallow: /tr?\nDisallow: /ufi/reaction/profile/browser/\nDisallow: /x/oauth/\nAllow: /ajax/bootloader-endpoint/\nAllow: /ajax/pagelet/generic.php/PagePostsSectionPagelet\nAllow: /careers/\nAllow: /safetycheck/\n\nUser-agent: ClaudeBot\nDisallow: /*/plugins/*\nDisallow: /?*next=\nDisallow: /a/bz?\nDisallow: /ajax/\nDisallow: /album.php\nDisallow: /business/*&categories\nDisallow: /business/*?categories\nDisallow: /business/help/search*&query=\nDisallow: /business/help/search*?query=\nDisallow: /business/news*&tags\nDisallow: /business/news*?tags\nDisallow: /business/search*&q=\nDisallow: /business/search*?q=\nDisallow: /checkpoint/\nDisallow: /contact_importer/\nDisallow: /dialog/\nDisallow: /fbml/ajax/dialog/\nDisallow: /feeds/\nDisallow: /file_download.php\nDisallow: /job_application/\nDisallow: /l.php\nDisallow: /login.php*&next=\nDisallow: /login.php/?next=\nDisallow: /login.php?next=\nDisallow: /login/*&next=\nDisallow: /login/?next=\nDisallow: /login/device-based/regular/login/*&next=\nDisallow: /login/device-based/regular/login/?next=\nDisallow: /meta-ai-answers/\nDisallow: /moments_app/\nDisallow: /p.php\nDisallow: /photos.php\nDisallow: /plugins/\nDisallow: /share.php\nDisallow: /share/\nDisallow: /sharer.php\nDisallow: /sharer/\nDisallow: /tr/\nDisallow: /tr?\nDisallow: /ufi/reaction/profile/browser/\nDisallow: /x/oauth/\nAllow: /ajax/bootloader-endpoint/\nAllow: /ajax/pagelet/generic.php/PagePostsSectionPagelet\nAllow: /careers/\nAllow: /safetycheck/\n\nUser-agent: GPTBot\nDisallow: /*/plugins/*\nDisallow: /?*next=\nDisallow: /a/bz?\nDisallow: /ajax/\nDisallow: /album.php\nDisallow: /business/*&categories\nDisallow: /business/*?categories\nDisallow: /business/help/search*&query=\nDisallow: /business/help/search*?query=\nDisallow: /business/news*&tags\nDisallow: /business/news*?tags\nDisallow: /business/search*&q=\nDisallow: /business/search*?q=\nDisallow: /checkpoint/\nDisallow: /contact_importer/\nDisallow: /dialog/\nDisallow: /fbml/ajax/dialog/\nDisallow: /feeds/\nDisallow: /file_download.php\nDisallow: /job_application/\nDisallow: /l.php\nDisallow: /login.php*&next=\nDisallow: /login.php/?next=\nDisallow: /login.php?next=\nDisallow: /login/*&next=\nDisallow: /login/?next=\nDisallow: /login/device-based/regular/login/*&next=\nDisallow: /login/device-based/regular/login/?next=\nDisallow: /meta-ai-answers/\nDisallow: /moments_app/\nDisallow: /p.php\nDisallow: /photos.php\nDisallow: /plugins/\nDisallow: /share.php\nDisallow: /share/\nDisallow: /sharer.php\nDisallow: /sharer/\nDisallow: /tr/\nDisallow: /tr?\nDisallow: /ufi/reaction/profile/browser/\nDisallow: /x/oauth/\nAllow: /ajax/bootloader-endpoint/\nAllow: /ajax/pagelet/generic.php/PagePostsSectionPagelet\nAllow: /careers/\nAllow: /safetycheck/\n\nUser-agent: PerplexityBot\nDisallow: /*/plugins/*\nDisallow: /?*next=\nDisallow: /a/bz?\nDisallow: /ajax/\nDisallow: /album.php\nDisallow: /business/*&categories\nDisallow: /business/*?categories\nDisallow: /business/help/search*&query=\nDisallow: /business/help/search*?query=\nDisallow: /business/news*&tags\nDisallow: /business/news*?tags\nDisallow: /business/search*&q=\nDisallow: /business/search*?q=\nDisallow: /checkpoint/\nDisallow: /contact_importer/\nDisallow: /dialog/\nDisallow: /fbml/ajax/dialog/\nDisallow: /feeds/\nDisallow: /file_download.php\nDisallow: /job_application/\nDisallow: /l.php\nDisallow: /login.php*&next=\nDisallow: /login.php/?next=\nDisallow: /login.php?next=\nDisallow: /login/*&next=\nDisallow: /login/?next=\nDisallow: /login/device-based/regular/login/*&next=\nDisallow: /login/device-based/regular/login/?next=\nDisallow: /meta-ai-answers/\nDisallow: /moments_app/\nDisallow: /p.php\nDisallow: /photos.php\nDisallow: /plugins/\nDisallow: /share.php\nDisallow: /share/\nDisallow: /sharer.php\nDisallow: /sharer/\nDisallow: /tr/\nDisallow: /tr?\nDisallow: /ufi/reaction/profile/browser/\nDisallow: /x/oauth/\nAllow: /ajax/bootloader-endpoint/\nAllow: /ajax/pagelet/generic.php/PagePostsSectionPagelet\nAllow: /careers/\nAllow: /safetycheck/\n\nUser-agent: *\nDisallow: /\n\n\n","agents":{"GPTBot":true,"OAI-SearchBot":false,"ChatGPT-User":false,"ClaudeBot":true,"Claude-SearchBot":false,"PerplexityBot":true,"Google-Extended":false,"Applebot-Extended":true,"CCBot":false,"Bytespider":false,"Meta-ExternalAgent":false,"Amazonbot":true}},"sitemap":null,"files":null,"derived":{"reachable":false,"robots_served":false,"robots_sitemap":false,"content_signal":false,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":[],"blocks_any_ai":false,"files":{"llms_txt":false,"agents_md":false,"agent_skills":false,"mcp_card":false,"api_catalog":false,"a2a_card":false},"any_capability":false,"ai_refused":[],"ai_thin":[],"rings":{"mcp_declared":false,"mcp_live":false,"mcp_auth":false,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":2,"fh":"m.facebook.com","fp_other_host":[]},"derive_v":3,"frame":"top1k","shard":2},{"v":1,"kind":"crawlcheck-census-record","run":14,"domain":"gstatic.com","rank":4,"measured_at":"2026-10-15T05:29:03.596Z","home":{"browser":{"status":404,"content_type":"text/html","bytes":1561,"sha256":"23926e9185d8d43c02807a838ffb373cc1977726094a4e46807c66ada9dd7660","final_host":"www.gstatic.com","ms":171,"identity":"Mobile browser","html":true},"gptbot":{"status":404,"content_type":"text/html","bytes":1561,"sha256":"23926e9185d8d43c02807a838ffb373cc1977726094a4e46807c66ada9dd7660","final_host":"www.gstatic.com","ms":171,"identity":"GPTBot","html":true},"claudebot":{"status":404,"content_type":"text/html","bytes":1561,"sha256":"23926e9185d8d43c02807a838ffb373cc1977726094a4e46807c66ada9dd7660","final_host":"www.gstatic.com","ms":169,"identity":"ClaudeBot","html":true},"oaisearch":{"status":404,"content_type":"text/html","bytes":1561,"sha256":"23926e9185d8d43c02807a838ffb373cc1977726094a4e46807c66ada9dd7660","final_host":"www.gstatic.com","ms":171,"identity":"OAI-SearchBot","html":true},"perplexity":{"status":404,"content_type":"text/html","bytes":1561,"sha256":"23926e9185d8d43c02807a838ffb373cc1977726094a4e46807c66ada9dd7660","final_host":"www.gstatic.com","ms":169,"identity":"PerplexityBot","html":true}},"robots":{"status":200,"content_type":"text/plain","bytes":5148,"sha256":"250d03d95cb7f7cac1202474926b169d6a85d39661989737527895d46a64319f","final_host":"www.gstatic.com","ms":171,"html":false,"text":"User-agent: *\nAllow: /_/aihub_pod/\nAllow: /_/alkali-learn/\nAllow: /_/apps-viewer/\nAllow: /_/atari/\nAllow: /_/boq/\nAllow: /_/boq-play/\nAllow: /_/boq-search/\nAllow: /_/boutique/\nAllow: /_/buganizer/\nAllow: /_/compare-banking/\nAllow: /_/compare-insurance/\nAllow: /_/fi/\nAllow: /_/fonts/\nAllow: /_/mss/boq-bard-web/\nAllow: /_/mss/boq-chrome-cloudcast/\nAllow: /_/mss/boq-cloud-ai-agentis/\nAllow: /_/mss/boq-cloud-client/\nAllow: /_/mss/boq-cloud-web3-portal/\nAllow: /_/mss/boq-deepmind-alphafold-server/\nAllow: /_/mss/boq-discover-labs/\nAllow: /_/mss/boq-gdp/\nAllow: /_/mss/boq-geo/\nAllow: /_/mss/boq-home-enterprise/\nAllow: /_/mss/boq-makersuite/\nAllow: /_/mss/boq-nbu-monetization/\nAllow: /_/mss/boq-search/\nAllow: /_/mss/boq-search-engagement-post/\nAllow: /_/mss/boq-rsvp/\nAllow: /_/mss/boq-travel/\nAllow: /_/mss/boq-identity/\nAllow: /_/mss/boq-labs-tailwind/\nAllow: /_/pantheon/\nAllow: /_/pantheon-platform/\nAllow: /_/play/\nAllow: /_/pog/\nAllow: /_/cloudsourcetools/\nAllow: /_/devopsconsole/\nAllow: /_/tfhub/\nAllow: /_/transparencyreport/\nAllow: /_/gstore/\nAllow: /_/ain/_/js/\nAllow: /ac/\nAllow: /actions/news/rss/\nAllow: /ads/learn/\nAllow: /ads/local/\nAllow: /ads/gpa/vidgen/\nAllow: /ads/research/\nAllow: /ads/ubaq/\nAllow: /agentis/omnichannel/public/\nAllow: /aistudio/*favicon*.png$\nAllow: /aistudio-static/\nAllow: /ai-sandbox-internal/generated_images/\nAllow: /aihub/tfhub/\nAllow: /aitestkitchen/website/flow/applets/\nAllow: /aitestkitchen/website/flow/social/\nAllow: /alkali/\nAllow: /alphafoldserver/\nAllow: /amphtml/stamp/ucp/post-testing/\nAllow: /android/market/\nAllow: /android/market_images/\nAllow: /antigravity/logos/\nAllow: /atari/images/public/\nAllow: /audio_news/\nAllow: /awmp/data/images/\nAllow: /bfe/apps/website/\nAllow: /boutique/sitemaps/\nAllow: /bricks/image/\nAllow: /compare/\nAllow: /chrome/crlset/\nAllow: /chrome/webstore/images/\nAllow: /cloud-launcher/sitemap/\nAllow: /cloudsourcetools/images/\nAllow: /commercesearch/images/\nAllow: /communities/sitemap/\nAllow: /copy-conductor/crawled/\nAllow: /covid19/mobility/\nAllow: /culturalinstitute/images/\nAllow: /culturalinstitute/sitemaps/\nAllow: /culturalinstitute/stella/\nAllow: /ct/log_list/\nAllow: /delight/\nAllow: /devopsconsole/images/\nAllow: /devrel-devsite/\nAllow: /docs/forms/social/\nAllow: /docs/socialmarkup/\nAllow: /dictionary/static/sitemaps/\nAllow: /droidguard/\nAllow: /earth/social/\nAllow: /earth/00-favicon.ico\nAllow: /enterprise-partner-search/sitemaps/\nAllow: /external_hosted/\nAllow: /fi/web/\nAllow: /flights/\nAllow: /flutter-canvaskit/\nAllow: /foodordering/\nAllow: /frommers/\nAllow: /gb/images/\nAllow: /glass/images/commerce/\nAllow: /gpay/social/\nAllow: /growth-ime/\nAllow: /gsx/\nAllow: /gtp/stir/\nAllow: /guidebooks/\nAllow: /GoogleInternetAuthority/\nAllow: /h5games_gamecenter/images/\nAllow: /healthricherkp/illustrations/\nAllow: /healthricherkp/cropped/\nAllow: /hex/\nAllow: /history/static/\nAllow: /identity/accountsettingsactivityweb/\nAllow: /identity/accountsettingssyaf/\nAllow: /images\nAllow: /inproduct_help/chatsupport/\nAllow: /keep/\nAllow: /lamda/images/\nAllow: /labs-code/\nAllow: /local/donut/\nAllow: /mapspro/\nAllow: /marketing-cms/\nAllow: /moderator/static/ux/\nAllow: /modern-formats-eng/ucp/framework_logging/\nAllow: /modern-formats-eng/ucp/*/client/\nAllow: /modern-formats-eng/ucp/*/csr/\nAllow: /modern-formats-eng/ucp/*/dev/\nAllow: /modern-formats-eng/ucp/static/framework_logging/\nAllow: /modern-formats-eng/ucp/static/*/client/\nAllow: /modern-formats-eng/ucp/static/*/csr/\nAllow: /modern-formats-eng/ucp/static/*/dev/\nAllow: /nbubumblebee/\nAllow: /og/\nAllow: /ohttp_gateway/\nAllow: /onebox/media/sports/logos/\nAllow: /onebox/media/sports/photos/soccer/wc2018/\nAllow: /onebox/sports/logos/\nAllow: /onebox/sports/logos/sports_types/png/\nAllow: /pagespeed/insights/ui/logo/\nAllow: /pantheon/\nAllow: /pantheon-platform/\nAllow: /patent-search/frontend/\nAllow: /pay/images/merchant_logos/\nAllow: /pay/images/program_logos/\nAllow: /paygo/\nAllow: /play/store/web/\nAllow: /play/surveys/serp/\nAllow: /podcasts/test-podcast/rss/\nAllow: /politics/e/sharing/\nAllow: /prettyearth/assets/\nAllow: /psb/udc/\nAllow: /rsvp/images/\nAllow: /s2/sitemaps/\nAllow: /search-ar-dev/\nAllow: /search-console/scfe/\nAllow: /share\nAllow: /shopping-content-api/product_images/\nAllow: /shopping-organic/price_validation/\nAllow: /shopping/shoppingpage/casa/\nAllow: /shopstream/\nAllow: /sites/\nAllow: /sitemaps/\nAllow: /social/photosui/images/\nAllow: /social/photosui/sitemap/\nAllow: /stadia/gamers/\nAllow: /store/images/favicons/\nAllow: /subscriptions/\nAllow: /support/content/\nAllow: /system1/images/\nAllow: /travel-frontend/images/\nAllow: /finance/favicon/\nAllow: /travel-vacations/\nAllow: /touringbird/\nAllow: /trends/\nAllow: /trends_tpt/\nAllow: /trustedstores/images/\nAllow: /trustedstores/js/\nAllow: /tv/thumb/\nAllow: /ui/\nAllow: /violet-web/previewer/\nAllow: /vr/*\nAllow: /vsi/*.pdf$\nAllow: /wondermissive/interest/\nAllow: /youtube/fonts/\nAllow: /youtube/img/branding/favicon/\nAllow: /youtube/img/promos/\nAllow: /icons/\nAllow: /js/\nAllow: /ss/\nAllow: /*.js\nAllow: /*.css\nAllow: /*.svg\nAllow: /buyside-e2e-monitor/\nAllow: /cgc/*.ico\nAllow: /cgc/*.png\nAllow: /cgc/*.svg\n\nDisallow: /\n","agents":{"GPTBot":false,"OAI-SearchBot":false,"ChatGPT-User":false,"ClaudeBot":false,"Claude-SearchBot":false,"PerplexityBot":false,"Google-Extended":false,"Applebot-Extended":false,"CCBot":false,"Bytespider":false,"Meta-ExternalAgent":false,"Amazonbot":false}},"sitemap":null,"files":null,"derived":{"reachable":false,"robots_served":false,"robots_sitemap":false,"content_signal":false,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":[],"blocks_any_ai":false,"files":{"llms_txt":false,"agents_md":false,"agent_skills":false,"mcp_card":false,"api_catalog":false,"a2a_card":false},"any_capability":false,"ai_refused":[],"ai_thin":[],"rings":{"mcp_declared":false,"mcp_live":false,"mcp_auth":false,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":2,"fh":"www.gstatic.com","fp_other_host":[]},"derive_v":3,"frame":"top1k","shard":3},{"v":1,"kind":"crawlcheck-census-record","run":14,"domain":"amazonaws.com","rank":5,"measured_at":"2026-10-15T05:30:03.842Z","home":{"browser":{"status":200,"content_type":"text/html","bytes":501746,"sha256":"153752cf3825b776e025f27301edec46c8ad930c644f16449115b53cfc5daa91","final_host":"aws.amazon.com","ms":822,"identity":"Mobile browser","html":true,"fp":"17afb9c19b71b7d2"},"gptbot":{"status":200,"content_type":"text/html","bytes":501746,"sha256":"82b0d2c97e3dea2522e5e2d9d47ce6abff3339b8a23deec3107605c467ee36f2","final_host":"aws.amazon.com","ms":656,"identity":"GPTBot","html":true,"fp":"17afb9c19b71b7d2"},"claudebot":{"status":200,"content_type":"text/html","bytes":501746,"sha256":"0148c98a1f42f15a21ab44941c14e3cbcf7c119317909514626f8a7b625e1695","final_host":"aws.amazon.com","ms":634,"identity":"ClaudeBot","html":true,"fp":"17afb9c19b71b7d2"},"oaisearch":{"status":200,"content_type":"text/html","bytes":501746,"sha256":"6851222560436ea3637b97bdd1c9d9375a8ec1d65d8cfb77df2f5140faf1c0ad","final_host":"aws.amazon.com","ms":710,"identity":"OAI-SearchBot","html":true,"fp":"17afb9c19b71b7d2"},"perplexity":{"status":200,"content_type":"text/html","bytes":501746,"sha256":"e0f6450872c31e3fea7b81a1e5bca48fbd5d8cd3e1367f737f2c5d4d9ea3cbfa","final_host":"aws.amazon.com","ms":692,"identity":"PerplexityBot","html":true,"fp":"17afb9c19b71b7d2"}},"robots":{"status":200,"content_type":"text/plain","bytes":13822,"sha256":"c1f42540a70f0dcd98d98565c46f486ed49a464f077ef1f1362c5470b638ae98","final_host":"aws.amazon.com","ms":766,"html":false,"text":"User-agent: AdsBot-Google\r\nUser-agent: AdsBot-Google-Mobile\r\nAllow: /pm/\r\nDisallow: /alexaforbusiness/\r\nDisallow: /blogs/\r\nDisallow: /*/blogs/\r\nDisallow: /solutions/case-studies/\r\nDisallow: /*/solutions/case-studies/\r\nAllow: /solutions/case-studies/sustainability-emea/\r\nAllow: /*/solutions/case-studies/sustainability-emea/\r\nAllow: /solutions/case-studies/innovators/innovators-eu/\r\nAllow: /*/solutions/case-studies/innovators/innovators-eu/\r\nAllow: /solutions/case-studies/generative-ai-ekacare/\r\nAllow: /cn/solutions/case-studies/generative-ai-ekacare/\r\nAllow: /solutions/case-studies/generative-ai-mindtickle/\r\nAllow: /cn/solutions/case-studies/generative-ai-mindtickle/\r\nAllow: /solutions/case-studies/generative-ai-invideo/\r\nAllow: /solutions/case-studies/generative-ai-mitsui/\r\nAllow: /solutions/case-studies/generative-ai-national-australia-bank/\r\nAllow: /solutions/case-studies/generative-ai-ntu/\r\nAllow: /solutions/case-studies/generative-ai-qarc/\r\nAllow: /solutions/case-studies/generative-ai-swimming-australia/\r\nAllow: /cn/solutions/case-studies/generative-ai-swimming-australia/\r\nAllow: /solutions/case-studies/generative-ai-thematic/\r\nAllow: /cn/solutions/case-studies/generative-ai-thematic/\r\nAllow: /solutions/case-studies/generative-ai-turing/\r\nAllow: /solutions/case-studies/generative-ai-stockmark/   \r\nAllow: /*/solutions/case-studies/generative-ai-turing/\r\nAllow: /*/solutions/case-studies/generative-ai-stockmark/   \r\nAllow: /solutions/case-studies/generative-ai-starhub/    \r\nAllow: /solutions/case-studies/generative-ai-bolttech/\r\nAllow: /*/solutions/case-studies/generative-ai-starhub/    \r\nAllow: /*/solutions/case-studies/generative-ai-bolttech/\r\n\r\nUser-agent: *\r\n# Block old What's New posts to preserve crawl budget for current content\r\nDisallow: /about-aws/whats-new/200\r\nDisallow: /about-aws/whats-new/201\r\nDisallow: /about-aws/whats-new/2020\r\nDisallow: /about-aws/whats-new/2021\r\nDisallow: /about-aws/whats-new/2022\r\nDisallow: /about-aws/whats-new/2023\r\nDisallow: /about-aws/whats-new/2024\n# Block tag\nDisallow: /*/blogs/*/tag/\nDisallow: /blogs/*/tag/\n\r\n# Other\r\nDisallow: */confirmation/\r\nDisallow: */registration-confirmation/\r\nDisallow: /about-aws/launch-a-website-in-minutes-tutorial/\r\nDisallow: /about-aws/media-coverage/2011/08/17/rhat-support-for-sap-solutions-running-on-rhel-on-amazon-ec2/\r\nDisallow: /activate/accelerators\r\nDisallow: /activate/accelerators/*\r\nDisallow: /activate/accelerators/confirmation/\r\nDisallow: /activate/china-portfolio-signup/\r\nDisallow: /activate/cloudkata/\r\nDisallow: /activate/event/\r\nDisallow: /activate/event/*\r\nDisallow: /activate/events/*\r\nDisallow: /activate/events/hact/\r\nDisallow: /activate/hackathons\r\nDisallow: /activate/hackathons/*\r\nDisallow: /activate/harvard-innovation-lab-students-2013/\r\nDisallow: /activate/harvard-innovation-lab-vip-teams-2013/\r\nDisallow: /activate/hsw/confirmation/\r\nDisallow: /activate/loft/\r\nDisallow: /activate/self-starters/confirmation/\r\nDisallow: /activate/wdm/*\r\nDisallow: /alexa/*\r\nDisallow: /alexa-top-sites/*\r\nDisallow: /alexa-web-search/*\r\nDisallow: /apac/aws-credits-confirmation/\r\nDisallow: /apac/aws-credits-request-confirmation/\r\nDisallow: /articles/item/\r\nDisallow: /awis/*\r\nDisallow: /aws-summit-2014/enterprise-london-db/\r\nDisallow: /aws-summit-2014/london-db/\r\nDisallow: /blogs/7faebe5debfdd358a46f2452a54eec94fce49269/\r\nDisallow: /campaigns/infor-hub-xt5r5iwdrgxbnuz/\r\nDisallow: /careers/aus-nz-event/\r\nDisallow: /careers/aus-nz-event/\r\nDisallow: /careers/aws-ar-event/\r\nDisallow: /careers/aws-aus-event/\r\nDisallow: /careers/aws-australia-event/\r\nDisallow: /careers/aws-bigdata-hiring/\r\nDisallow: /careers/aws-brisbane-event/\r\nDisallow: /careers/aws-db-event/\r\nDisallow: /careers/aws-ks-event/\r\nDisallow: /careers/aws-london-event/\r\nDisallow: /careers/aws-mexicocity-event/\r\nDisallow: /careers/aws-miami-event/\r\nDisallow: /careers/aws-ny-event/\r\nDisallow: /careers/aws-pittsburgh-event/\r\nDisallow: /careers/aws-portland-event/\r\nDisallow: /careers/aws-sea-event/\r\nDisallow: /careers/aws-singapore-event/\r\nDisallow: /careers/aws-sydney-event/\r\nDisallow: /careers/chattanooga-mobile-event/\r\nDisallow: /careers/database-hiring/\r\nDisallow: /careers/devtools-australia-event/\r\nDisallow: /careers/devtools-australia-event/\r\nDisallow: /careers/ebs-cincinnati-event/\r\nDisallow: /careers/ec2-australia-event/\r\nDisallow: /careers/ec2-chicago-event/\r\nDisallow: /careers/ec2-hiring-events/\r\nDisallow: /careers/ec2-ottawa-event/\r\nDisallow: /careers/ec2-vancouver-event/\r\nDisallow: /careers/indianapolis-event/\r\nDisallow: /careers/mexico-edge-hiring-2014/\r\nDisallow: /careers/milwaukee-edge-hiring-2014/\r\nDisallow: /careers/montreal-edge-hiring/\r\nDisallow: /careers/Phoenix-Hiring-Event/\r\nDisallow: /careers/pittsburgh-platform-hiring-2014/\r\nDisallow: /careers/raleigh-mobile-event/\r\nDisallow: /careers/raleigh-storage-hiring-2014/\r\nDisallow: /careers/santiago-event/\r\nDisallow: /careers/Sydney-Hiring-Event/\r\nDisallow: /careers/texas-event/\r\nDisallow: /careers/toronto-event/\r\nDisallow: /careers/toronto-storage-hiring-2014/\r\nDisallow: /Cloud_Start_01.LandingPage.html\r\nDisallow: /Cloud_Start_01-Landing-Page-EDO.html\r\nDisallow: /Cloud_Start_02.ConfirmationPage.html\r\nDisallow: /code/*\r\nDisallow: /compliance/contact-us-confirmation/\r\nDisallow: /compliance/hipaa-eligible-services-reference/\r\nDisallow: /compliance/how-to-open-aws-compliance-reports/\r\nDisallow: /compliance/public-sector-contact-confirmation/\r\nDisallow: /compliance/security-assurance-faq\r\nDisallow: /compliance/security-assurance-faq-1\r\nDisallow: /compliance/security-assurance-faq-10\r\nDisallow: /compliance/security-assurance-faq-2\r\nDisallow: /compliance/security-assurance-faq-3\r\nDisallow: /compliance/security-assurance-faq-4\r\nDisallow: /compliance/security-assurance-faq-5\r\nDisallow: /compliance/security-assurance-faq-6\r\nDisallow: /compliance/security-assurance-faq-7\r\nDisallow: /compliance/security-assurance-faq-8\r\nDisallow: /compliance/security-assurance-faq-9\r\nDisallow: /compliance/security-assurance-faq-complete\r\nDisallow: /compliance/security-assurance-US-Public-Sector-checklist\r\nDisallow: /compliance/security-assurance-US-Public-Sector-checklist-1\r\nDisallow: /compliance/security-assurance-US-Public-Sector-checklist-10\r\nDisallow: /compliance/security-assurance-US-Public-Sector-checklist-2\r\nDisallow: /compliance/security-assurance-US-Public-Sector-checklist-3\r\nDisallow: /compliance/security-assurance-US-Public-Sector-checklist-4\r\nDisallow: /compliance/security-assurance-US-Public-Sector-checklist-5\r\nDisallow: /compliance/security-assurance-US-Public-Sector-checklist-6\r\nDisallow: /compliance/security-assurance-US-Public-Sector-checklist-7\r\nDisallow: /compliance/security-assurance-US-Public-Sector-checklist-8\r\nDisallow: /compliance/security-assurance-US-Public-Sector-checklist-9\r\nDisallow: /compliance/security-assurance-US-Public-Sector-checklist-complete\r\nDisallow: /compliance/third-party-access/\r\nDisallow: /contact-us/ami-submission-conf/\r\nDisallow: /contact-us/article-submission-conf/\r\nDisallow: /contact-us/case-contact-confirmation/\r\nDisallow: /contact-us/customer-app-submission-conf/\r\nDisallow: /contact-us/developertool-submission-conf/\r\nDisallow: /contact-us/ec2-email-limit-request-confirmation/\r\nDisallow: /contact-us/elasticache-node-limit-request-confirmation/\r\nDisallow: /contact-us/marketing-unsubscribe-confirmation/\r\nDisallow: /contact-us/publicdataset-conf/\r\nDisallow: /contact-us/sales-contact-confirmation/\r\nDisallow: /contact-us/samplecode-submission-conf/\r\nDisallow: /csds/*\r\nDisallow: /customerapps/*\r\nDisallow: /datasets/\r\nDisallow: /developertools/2555753788650372\r\nDisallow: /documentation/alexa-top-sites/\r\nDisallow: /documentation/awis/\r\nDisallow: /email-preferences\r\nDisallow: /email-preferences/\r\nDisallow: /events/applesummit\r\nDisallow: /events/cio-advisory-council/\r\nDisallow: /events/finserv-summit-dublin\r\nDisallow: /events/finserv-summit-dublin/confirmation\r\nDisallow: /events/finserv-summit-singapore\r\nDisallow: /events/finserv-summit-singapore/confirmation\r\nDisallow: /events/fsi-cloud-symposium-nyc.html\r\nDisallow: /federal/events/earthobservationday2015/\r\nDisallow: /federal/events/earthobservationday2015/ConfirmationPage/\r\nDisallow: /forms/activate-support\r\nDisallow: /free/cloud-computing-free-tier/\r\nDisallow: /free/databases-free-tier/\r\nDisallow: /get-cloud/\r\nDisallow: /getting-started/functional-groups/\r\nDisallow: /getting-started/functional-groups/*\r\nDisallow: /government-education/nonprofits/aws-serve/\r\nDisallow: /government-education/nonprofits/aws-serve-member/\r\nDisallow: /government-education/nonprofits/credit-program/\r\nDisallow: /government-education/nonprofits/credit-program/confirmation\r\nDisallow: /government-education/nonprofits/credit-program/terms\r\nDisallow: /government-education/nonprofits/grant-program/\r\nDisallow: /government-education/nonprofits/grant-program/confirmation/\r\nDisallow: /government-education/nonprofits/grant-program/terms/\r\nDisallow: /government-education/nonprofits/leadership-forum-poc\r\nDisallow: /government-education/nonprofits/leadership-forum-poc/confirmation\r\nDisallow: /government-education/public-sector-cloud-start/public-sector-cloud-start/\r\nDisallow: /government-education/public-sector-cloud-start/public-sector-cloud-start1/\r\nDisallow: /hackathons/reinvent-2013/\r\nDisallow: /legal/3874403/\r\nDisallow: /lms/contact-us/\r\nDisallow: /loft/\r\nDisallow: /marketplace/reviews/reviews-list/*/write-comment/\r\nDisallow: /marketplace/reviews/write-review/\r\nDisallow: /marketplace/search*\r\nDisallow: /microsoft/bizspark/\r\nDisallow: /microsoft/bizspark/signon/thankyou/\r\nDisallow: /nav/\r\nDisallow: /nav/side-nav/\r\nDisallow: /nav/side-nav/products-services/\r\nDisallow: /nav/tabs/\r\nDisallow: /nav/tabs/account/\r\nDisallow: /nav/tabs/aws/\r\nDisallow: /nav/tabs/community/\r\nDisallow: /nav/tabs/developers/\r\nDisallow: /nav/tabs/products/\r\nDisallow: /nav/tabs/support/\r\nDisallow: /optin/\r\nDisallow: /optout/\r\nDisallow: /partners/contact-us/partners-confirmation/\r\nDisallow: /partners/overview/partner-newsletter/confirmation/\r\nDisallow: /partners/overview/partner-terms-and-conditions/\r\nDisallow: /partners/webinars/request/\r\nDisallow: /ping\r\nDisallow: /pm/\r\nDisallow: /ar/pm/\r\nDisallow: /cn/pm/\r\nDisallow: /de/pm/\r\nDisallow: /es/pm/\r\nDisallow: /fr/pm/\r\nDisallow: /id/pm/\r\nDisallow: /it/pm/\r\nDisallow: /jp/pm/\r\nDisallow: /ko/pm/\r\nDisallow: /pt/pm/\r\nDisallow: /ru/pm/\r\nDisallow: /th/pm/\r\nDisallow: /tr/pm/\r\nDisallow: /tw/pm/\r\nDisallow: /vi/pm/\r\nAllow: /pm/amplify/\r\nAllow: /pm/lambda/\r\nAllow: /pm/sagemaker/\r\nAllow: /pm/textract/\r\nAllow: /pm/transcribe/\r\nDisallow: /premiumsupport/iam/\r\nDisallow: /premiumsupport/knowledge-center/feedback-no/\r\nDisallow: /premiumsupport/knowledge-center/feedback-yes/\r\nDisallow: /pricing/awis/\r\nDisallow: /public-data-sets/applicants/\r\nDisallow: /public-data-sets/terms/\r\nDisallow: /publicdatasets-confirmation/\r\nDisallow: /quickstart/promo\r\nDisallow: /quickstart/promo/confirmation\r\nDisallow: /reinvent/video-case-studies\r\nDisallow: /releasenotes/item/\r\nDisallow: /research-credits/data-science-bowl\r\nDisallow: /ru/blogs/497c2c2d305b7dfa351d17bd3b4398dc37b0e0a4/\r\nDisallow: /s/*\r\nDisallow: /s/dm/house/editorial/startups/ha_ed_activate_loft_launch_sf_1/\r\nDisallow: /sap/saphana/pilot/\r\nDisallow: /sap/saphana/pilot/offer/\r\nDisallow: /sap/saphana/pilot/offer/confirmation\r\nDisallow: /search\r\nDisallow: /search/\r\nAllow: /api/dirs/items/search\r\nDisallow: /security/aws-pgp-public-key\r\nDisallow: /security/ec2-pkgs-public-key\r\nDisallow: /security/introducing-s2n-new-open-source-TLS-implementation\r\nDisallow: /s-moved/dm/house/editorial/startups/ha_ed_activate_loft_launch_sf_1/\r\nDisallow: /solutions/case-studies/imdb/\r\nDisallow: /solutions/featured-partners/\r\nDisallow: /solutions/global-solution-providers/microsoft/bizspark/\r\nDisallow: /solutions/global-solution-providers/microsoft/bizspark/faq/\r\nDisallow: /solutions/global-solution-providers/microsoft/bizspark/signon/\r\nDisallow: /solutions/global-solution-providers/microsoft/bizspark/signon/\r\nDisallow: /solutions/global-solution-providers/microsoft/bizspark/signon/\r\nDisallow: /solutions/global-solution-providers/microsoft/bizspark/signon/thankyou/\r\nDisallow: /solutions/global-solution-providers/microsoft/bizspark/signon/thankyou/\r\nDisallow: /solutions/solution-providers/program/reseller/adguidelines/\r\nDisallow: /solutions/solution-providers/program/reseller/adguidelines/\r\nDisallow: /solutions/solution-providers/program/reseller/authorized/\r\nDisallow: /solutions/solution-providers/program/reseller/authorized/\r\nDisallow: /solutions/solution-providers/program/reseller/customerterms/\r\nDisallow: /solutions/solution-providers/program/reseller/customerterms/\r\nDisallow: /specialhandling-payments/\r\nDisallow: /start-ups/challenge/\r\nDisallow: /start-ups/loft/hack/\r\nDisallow: /start-ups/loft/hacknight/\r\nDisallow: /start-ups/loft/sf2014/\r\nDisallow: /start-ups/loft/sfff100l3ft\r\nDisallow: /start-ups/loft/survey/\r\nDisallow: /start-ups/loft/welcome\r\nDisallow: /unable-to-process-request/\r\nDisallow: /use-cases/global-solution-providers/microsoft/bizspark/\r\nDisallow: /use-cases/global-solution-providers/microsoft/bizspark/getting-started/\r\nDisallow: /use-cases/global-solution-providers/microsoft/bizspark/signon/\r\nDisallow: /use-cases/global-solution-providers/microsoft/bizspark/signon/thankyou/\r\nDisallow: /use-cases/global-solution-providers/microsoft/bizspark/thankyou/\r\nDisallow: /vmware/about\r\nDisallow: /windows/resources/whitepapers/\r\nDisallow: /windows/server2003/access/\r\nDisallow: /z_dx-marketing-team-prototypes/*\r\nDisallow: /community/*\r\nDisallow: /community/*\r\nDisallow: /community/heroes/\r\nDisallow: /*/community/heroes/\r\nDisallow: /*/search\r\nDisallow: /*/search/\r\nSitemap: https://aws.amazon.com/sitemaps/index/\r\nSitemap: https://d36a099zv34xie.cloudfront.net/psf-sitemap.xml\r\nSitemap: https://aws.amazon.com/marketplace/sitemap","agents":{"GPTBot":true,"OAI-SearchBot":true,"ChatGPT-User":true,"ClaudeBot":true,"Claude-SearchBot":true,"PerplexityBot":true,"Google-Extended":true,"Applebot-Extended":true,"CCBot":true,"Bytespider":true,"Meta-ExternalAgent":true,"Amazonbot":true}},"sitemap":{"status":200,"content_type":"application/xml","bytes":2669,"sha256":"a0de77f224598d85059fb8900f5e7fc2797f2e5d4d5d4ddddac47a349f65479f","final_host":"aws.amazon.com","ms":446,"url":"https://aws.amazon.com/sitemaps/index/","declared":true,"xml":true},"files":{"llms_txt":{"status":404,"content_type":"text/html","bytes":302397,"sha256":"61b69c2e01f33057fd738e10d5f1407d3712110289b9b8c624999f32bfc4313b","final_host":"aws.amazon.com","ms":280,"path":"/llms.txt","html":true,"json":null,"present":false},"agents_md":{"status":404,"content_type":"text/html","bytes":302397,"sha256":"fc52ff3822add2b876c14ab118313fa42f54004c6a8b760666d4d5463e4ef3ad","final_host":"aws.amazon.com","ms":656,"path":"/agents.md","html":true,"json":null,"present":false},"agent_skills":{"status":404,"content_type":"text/html","bytes":302397,"sha256":"c532c590a28c3b37e7d0adbea45a2c73d3c268574fc88859c7fb6288338e2bbc","final_host":"aws.amazon.com","ms":466,"path":"/.well-known/agent-skills/index.json","html":true,"json":null,"present":false},"mcp_card":{"status":404,"content_type":"text/html","bytes":302397,"sha256":"1b27494ab46542d557d668d99ea95172b5ba0c04cfaaa59abef60d8c78170394","final_host":"aws.amazon.com","ms":467,"path":"/.well-known/mcp/server-card.json","html":true,"json":null,"present":false},"api_catalog":{"status":404,"content_type":"text/html","bytes":302397,"sha256":"b9b6b907c2bdcfdeb922a0df5951acdc14ef988554c6740a06b1c9f085e9bd2b","final_host":"aws.amazon.com","ms":678,"path":"/.well-known/api-catalog","html":true,"json":null,"present":false},"a2a_card":{"status":404,"content_type":"text/html","bytes":302397,"sha256":"1b18aeab4a441db4b258b0c073b70c667ad8293d83e1019952b758614aee7f60","final_host":"aws.amazon.com","ms":889,"path":"/.well-known/agent-card.json","html":true,"json":null,"present":false}},"rings":{"v":1,"candidates":[],"mcp":[],"a2a":null,"webmcp":{"model_context":false,"bridge":false,"mcp_json":false,"link_rel":false,"any":false},"handshakes":0,"get_path":{"status":404,"content_type":"text/html","ms":881,"error":null},"get_host":{"status":530,"content_type":"text/plain","ms":56,"error":null}},"derived":{"reachable":true,"robots_served":true,"robots_sitemap":true,"content_signal":false,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":[],"blocks_any_ai":false,"files":{"llms_txt":false,"agents_md":false,"agent_skills":false,"mcp_card":false,"api_catalog":false,"a2a_card":false},"any_capability":false,"ai_refused":[],"ai_thin":[],"rings":{"mcp_declared":false,"mcp_live":false,"mcp_auth":false,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":2,"fh":"aws.amazon.com","fp_other_host":[],"fp":"17afb9c19b71b7d2","fp_bits":{"gptbot":0,"claudebot":0,"oaisearch":0,"perplexity":0}},"derive_v":3,"frame":"top1k","shard":4},{"v":1,"kind":"crawlcheck-census-record","run":14,"domain":"googleapis.com","rank":6,"measured_at":"2026-10-15T05:31:08.958Z","home":{"browser":{"status":404,"content_type":"text/html","bytes":1561,"sha256":"23926e9185d8d43c02807a838ffb373cc1977726094a4e46807c66ada9dd7660","final_host":"googleapis.com","ms":207,"identity":"Mobile browser","html":true},"gptbot":{"status":404,"content_type":"text/html","bytes":1561,"sha256":"23926e9185d8d43c02807a838ffb373cc1977726094a4e46807c66ada9dd7660","final_host":"googleapis.com","ms":206,"identity":"GPTBot","html":true},"claudebot":{"status":404,"content_type":"text/html","bytes":1561,"sha256":"23926e9185d8d43c02807a838ffb373cc1977726094a4e46807c66ada9dd7660","final_host":"googleapis.com","ms":207,"identity":"ClaudeBot","html":true},"oaisearch":{"status":404,"content_type":"text/html","bytes":1561,"sha256":"23926e9185d8d43c02807a838ffb373cc1977726094a4e46807c66ada9dd7660","final_host":"googleapis.com","ms":207,"identity":"OAI-SearchBot","html":true},"perplexity":{"status":404,"content_type":"text/html","bytes":1561,"sha256":"23926e9185d8d43c02807a838ffb373cc1977726094a4e46807c66ada9dd7660","final_host":"googleapis.com","ms":207,"identity":"PerplexityBot","html":true}},"robots":{"status":404,"content_type":"text/html","bytes":1571,"sha256":"6adee83a3eba87d30fdca3997d27097697f2767805f222a0f78c3f04392004d2","final_host":"googleapis.com","ms":178,"html":true,"text":null,"agents":null},"sitemap":null,"files":null,"derived":{"reachable":false,"robots_served":false,"robots_sitemap":false,"content_signal":false,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":[],"blocks_any_ai":false,"files":{"llms_txt":false,"agents_md":false,"agent_skills":false,"mcp_card":false,"api_catalog":false,"a2a_card":false},"any_capability":false,"ai_refused":[],"ai_thin":[],"rings":{"mcp_declared":false,"mcp_live":false,"mcp_auth":false,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":2,"fh":"googleapis.com","fp_other_host":[]},"derive_v":3,"frame":"top1k","shard":5},{"v":1,"kind":"crawlcheck-census-record","run":14,"domain":"akamai.net","rank":7,"measured_at":"2026-10-15T05:33:03.639Z","home":{"browser":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"akamai.net","ms":10,"identity":"Mobile browser","html":false},"gptbot":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"akamai.net","ms":8,"identity":"GPTBot","html":false},"claudebot":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"akamai.net","ms":10,"identity":"ClaudeBot","html":false},"oaisearch":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"akamai.net","ms":10,"identity":"OAI-SearchBot","html":false},"perplexity":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"akamai.net","ms":10,"identity":"PerplexityBot","html":false}},"robots":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"akamai.net","ms":10,"html":false,"text":null,"agents":null},"sitemap":null,"files":null,"derived":{"reachable":false,"robots_served":false,"robots_sitemap":false,"content_signal":false,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":[],"blocks_any_ai":false,"files":{"llms_txt":false,"agents_md":false,"agent_skills":false,"mcp_card":false,"api_catalog":false,"a2a_card":false},"any_capability":false,"ai_refused":[],"ai_thin":[],"rings":{"mcp_declared":false,"mcp_live":false,"mcp_auth":false,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":2,"fh":"akamai.net","fp_other_host":[]},"derive_v":3,"frame":"top1k","shard":6},{"v":1,"kind":"crawlcheck-census-record","run":14,"domain":"microsoft.com","rank":8,"measured_at":"2026-10-15T05:34:03.814Z","home":{"browser":{"status":200,"content_type":"text/html","bytes":207849,"sha256":"efd8567cf95bb8e7c7800b05176dd28ae0bf4b0c554ff32940313ed50297ef4e","final_host":"www.microsoft.com","ms":432,"identity":"Mobile browser","html":true,"fp":"e83d250dca983ab7"},"gptbot":{"status":200,"content_type":"text/html","bytes":207849,"sha256":"9c44dddd91f7609f548c97c3538b280eefbd3f8554f553dae49f3ccaefde77fb","final_host":"www.microsoft.com","ms":457,"identity":"GPTBot","html":true,"fp":"e83d250dca983ab7"},"claudebot":{"status":200,"content_type":"text/html","bytes":207849,"sha256":"191071548fb9afb213f2f9d2bc4b80758317cfb2a5121e0f49186de14b8532cf","final_host":"www.microsoft.com","ms":432,"identity":"ClaudeBot","html":true,"fp":"e83d250dca983ab7"},"oaisearch":{"status":200,"content_type":"text/html","bytes":207849,"sha256":"b67728af826dba4f579fac819a47c742782e5d3062028564482fdc8c3e458199","final_host":"www.microsoft.com","ms":422,"identity":"OAI-SearchBot","html":true,"fp":"e83d250dca983ab7"},"perplexity":{"status":200,"content_type":"text/html","bytes":207849,"sha256":"b936a021d2685a504a167482dbf22cf5ed03c97f9645c673ce9fc74b17af29dc","final_host":"www.microsoft.com","ms":432,"identity":"PerplexityBot","html":true,"fp":"e83d250dca983ab7"}},"robots":{"status":200,"content_type":"text/plain","bytes":9126,"sha256":"79a2a24e3e2c00deaba2fc1922b9113f5508e74133911eadea4ebb00b5de9768","final_host":"www.microsoft.com","ms":82,"html":false,"text":"# Robots.txt file for www.microsoft.com v2.11\r\n\r\nUser-agent: *\r\nDisallow: /*?*ep_*\r\nDisallow: /*&ep_*\r\nDisallow: /store/XboxComMsCom3PAdsOptOutCookieSync.html\r\nDisallow: /store/XboxComMsComCartMuidSync.html\r\nDisallow: /*/store/locations/store-locations-sitemap.xml\r\nDisallow: /en-us/windows/si/matrix.html\r\nDisallow: /en-us/windows/si/matrix.html\r\nDisallow: /*/security/search-results.aspx?\r\nDisallow: /*/music/*/search/\r\nDisallow: /*/store/buynow?\r\nDisallow: /*/store/cart\r\nDisallow: /store/buy/cartcount\r\nDisallow: /*/search/\r\nDisallow: /*/music/*/Search/\r\nDisallow: /*/Search/\r\nDisallow: /*/newsearch/\r\nDisallow: *action=catalogsearch&\r\nDisallow: /*/store/d/groove-music-pass/cfq7ttc0k5dq/0001\r\nDisallow: /*/store/best-rated/*?*&*&*\r\nDisallow: /*/store/best-selling/*?*&*&*\r\nDisallow: /*/store/coming-soon/*?*&*&*\r\nDisallow: /*/store/deals/*?*&*&*\r\nDisallow: /*/store/game-demos/*?*&*&*\r\nDisallow: /*/store/game-previews/*?*&*&*\r\nDisallow: /*/store/most-played/*?*&*&*\r\nDisallow: /*/store/most-popular/*?*&*&*\r\nDisallow: /*/store/new/*?*&*&*\r\nDisallow: /*/store/new-and-rising/*?*&*&*\r\nDisallow: /*/store/top-free/*?*&*&*\r\nDisallow: /*/store/top-paid/*?*&*&*\r\nDisallow: /*/store/collections/helpmechoosepc?*&*&*\r\nDisallow: /*/store/collections/helpmechoosepc/?*&*&*\r\nDisallow: /*/store/movies-and-tv/*?*&*&*\r\nDisallow: /cascadeauth/microsoft-365/account/silentauth\r\nDisallow: /*/microsoft-365/site-search\r\nDisallow: /news/search\r\nDisallow: /video/search\r\nDisallow: */qa-folder\r\nAllow: /*/store/*/search/\r\nAllow: /*/store/*/layout/\r\nAllow: /*/store/music/groove-music-pass/*\r\nAllow: *action=catalogsearch&catalog_mode=grid&page=2$\r\nAllow: *action=catalogsearch&catalog_mode=grid&page=3$\r\nAllow: *action=catalogsearch&catalog_mode=grid&page=4$\r\nAllow: *action=catalogsearch&catalog_mode=grid&page=5$\r\nAllow: *action=catalogsearch&catalog_mode=grid&page=6$\r\nAllow: *action=catalogsearch&catalog_mode=grid&page=7$\r\nAllow: *action=catalogsearch&catalog_mode=grid&page=8$\r\nAllow: *action=catalogsearch&catalog_mode=list&page=2$\r\nAllow: *action=catalogsearch&catalog_mode=list&page=3$\r\nAllow: *action=catalogsearch&catalog_mode=list&page=4$\r\nAllow: *action=catalogsearch&catalog_mode=list&page=5$\r\nAllow: *action=catalogsearch&catalog_mode=list&page=6$\r\nAllow: *action=catalogsearch&catalog_mode=list&page=7$\r\nAllow: *action=catalogsearch&catalog_mode=list&page=8$\r\nDisallow: *action=accessorysearch&product=*&*\r\nAllow: *action=accessorysearch&product=*$\r\nDisallow: *action=accessorysearch&\r\nAllow: *action=accessorysearch&page=2$\r\nAllow: *action=accessorysearch&page=3$\r\nAllow: *action=accessorysearch&page=4$\r\nAllow: *action=accessorysearch&page=5$\r\nAllow: *action=accessorysearch&page=6$\r\nAllow: *action=accessorysearch&page=7$\r\nAllow: *action=accessorysearch&page=8$\r\nDisallow: *action=productcompareaction&\r\nDisallow: *action=productLinkAction&\r\nDisallow: *action=overlay&\r\nDisallow: *action=quickSearch&\r\nDisallow: *action=writeReview\r\nDisallow: *rep=hc\r\nDisallow: *fe=true\r\nDisallow: *?intc=\r\nDisallow: *&solved=\r\nDisallow: /music/*/wal/\r\nDisallow: */wal/\r\nDisallow: */layout/\r\nDisallow: */base-en/\r\nDisallow: *action=siteSearch\r\nDisallow: *action=productSupportSearch\r\nDisallow: *=imgmanager\r\nDisallow: */unsubscribe/\r\nDisallow: *ActivityUID=\r\nDisallow: /feeds/TechNet/fr-fr/screenshot/screenshot%20surface.jpg\r\nDisallow: /imaginecup/*\r\nDisallow: /*/download/confirmation.aspx?\r\nDisallow: /*HpOptOut=true$\r\nDisallow: /*TOCLinksForCrawlers*\r\nDisallow: /*mac/help.mspx?\r\nDisallow: /*mactopia/help.mspx?\r\nDisallow: */mediaapi/\r\nDisallow: /blacklisted*\r\nDisallow: /canada/Library/mnp/2/aspx/\r\nDisallow: /communities/bin.aspx?\r\nDisallow: /communities/blogs/PortalResults.mspx?\r\nDisallow: /communities/eventdetails.mspx?\r\nDisallow: /communities/rss.aspx*\r\nDisallow: /*/download/confirmation.aspx?\r\nDisallow: /*/download/registration-suggested.aspx?\r\nDisallow: /*/download/results.aspx?\r\nDisallow: /*/download/Browse.aspx?\r\nDisallow: /*/download/browse.aspx?\r\nDisallow: /*/download/info.aspx?\r\nDisallow: /*/download/thankyou.aspx\r\nDisallow: /*/download/thankyou.aspx?\r\nDisallow: /france/formation/centres/planning.asp?\r\nDisallow: /france/ie/default.asp?\r\nDisallow: /france/mnp_utility.mspx?\r\nDisallow: /genuine/\r\nDisallow: /Germany/kleinunternehmen/euga/detail.mspx?\r\nDisallow: /Germany/kleinunternehmen/euga/results.mspx?\r\nDisallow: /germany/library/images/mnp/\r\nDisallow: /germany/video/de/de/related*\r\nDisallow: /hpc/*/supported-applications.aspx?\r\nDisallow: /ie/ie40/\r\nDisallow: /info/customerror.htm*\r\nDisallow: /info/smart404.asp*\r\nDisallow: /intlkb/\r\nDisallow: /isapi/\r\nDisallow: /Japan/DirectX/default.asp?\r\nDisallow: /japan/directx/default.asp?\r\nDisallow: /japan/enable/textview.asp?\r\nDisallow: /japan/products/library/search.asp?\r\nDisallow: /japan/showcase/print/default.aspx?\r\nDisallow: /japan/terminology/query.asp?\r\nDisallow: /*mnp_utility.mspx?\r\nDisallow: /rus/licensing/Unilateral.aspx/*\r\nDisallow: /spain/empresas/\r\nDisallow: /spain/medianaempresa/\r\nDisallow: /windows/compatibility/windows-vista/\r\nDisallow: /windows/compatibility/windows-7/\r\nDisallow: /windows/compatibility/windows-7/*\r\nDisallow: /windows/404.aspx?*\r\nDisallow: /windows/campaign/meet-start.aspx\r\nDisallow: /windows/campaign/meet-apps.aspx\r\nDisallow: /windows/campaign/features-built-in-apps.aspx\r\nDisallow: /ru-ru/events/platforma/materials/default.aspx?speaker*\r\nDisallow: /de-de/corporate/rechtliche-hinweise/impressum_de.aspx\r\nDisallow: /en/microsoft-365\r\nDisallow: /en/microsoft-365/\r\nAllow: /en/microsoft-365/word\r\nAllow: /en/microsoft-365/outlook/web-email-login-for-outlook\r\nAllow: /en/microsoft-365/free-office-online-for-the-web\r\nAllow: /en/microsoft-365/planner/microsoft-planner\r\nAllow: /en/microsoft-365/yammer/yammer-overview\r\nAllow: /en/microsoft-365/publisher\r\nAllow: /en/microsoft-365/products-apps-services\r\nAllow: /en/microsoft-365/onenote/digital-note-taking-app\r\nAllow: /en/microsoft-365/powerpoint\r\nAllow: /en/microsoft-365/excel\r\nAllow: /en/microsoft-365/microsoft-office\r\nAllow: /en/microsoft-365/business/microsoft-365-administration\r\nAllow: /en/microsoft-365-copilot/microsoft-copilot-studio\r\nAllow: /en/microsoft-365/project/project-management-software\r\nAllow: /en/microsoft-365/project/microsoft-project-enterprise-plans-and-pricing\r\nAllow: /en/microsoft-365/onedrive/online-cloud-storage\r\nAllow: /en/microsoft-365/business\r\nAllow: /en/microsoft-365/clipchamp\r\nAllow: /en/microsoft-viva/engage\r\nDisallow: /en/microsoft-365/\r\nDisallow: /es/microsoft-365\r\nDisallow: /es/microsoft-365/\r\nDisallow: /fr/microsoft-365\r\nDisallow: /fr/microsoft-365/\r\nDisallow: /en/security\r\nDisallow: /en/security/\r\nDisallow: /es/security\r\nDisallow: /es/security/\r\nDisallow: /fr/security\r\nDisallow: /fr/security/\r\nDisallow: /en/microsoft-teams\r\nDisallow: /en/microsoft-teams/\r\nDisallow: /es/microsoft-teams\r\nDisallow: /es/microsoft-teams/\r\nDisallow: /fr/microsoft-teams\r\nDisallow: /fr/microsoft-teams/\r\nDisallow: /en/microsoft-viva\r\nDisallow: /en/microsoft-viva/\r\nDisallow: /es/microsoft-viva\r\nDisallow: /es/microsoft-viva/\r\nDisallow: /fr/microsoft-viva\r\nDisallow: /fr/microsoft-viva/\r\nDisallow: */isv/*?\r\nDisallow: /wp-admin/post.php \r\nAllow: /wp-admin/admin-ajax.php \r\nAllow: /wp-content/uploads/ \r\nDisallow: /blog/feed/ \r\nDisallow: /wp-login.php \r\nDisallow: /xmlrpc.php \r\nDisallow: /*?p= \r\nDisallow: /*?*p=* \r\nDisallow: /cdn-cgi/* \r\nDisallow: /page/*/?s= \r\nDisallow: /login/ \r\nDisallow: /*/embed/ \r\nDisallow: /cgi-bin/ \r\nAllow: /?display=wide \r\nDisallow: /readme.html \r\nDisallow: /*noItems= \r\nDisallow: /?s=* \r\nDisallow: /?sort-by=*\r\n\r\nSitemap: https://www.microsoft.com/store/sitemap-index.xml\r\nSitemap: https://www.microsoft.com/en-us/surface/surface.index.xml\r\nSitemap: https://www.microsoft.com/en-us/windows/windows-index.xml\r\nSitemap: https://www.microsoft.com/sitemaps/ai/sitemapindex.xml\r\nSitemap: https://www.microsoft.com/sitemaps/dynamics-365/sitemapindex.xml\r\nSitemap: https://www.microsoft.com/sitemaps/isv/sitemapindex.xml\r\nSitemap: https://www.microsoft.com/sitemaps/microsoft-365/sitemapindex.xml\r\nSitemap: https://www.microsoft.com/sitemaps/microsoft-copilot/sitemapindex.xml\r\nSitemap: https://www.microsoft.com/sitemaps/microsoft-fabric/sitemapindex.xml\r\nSitemap: https://www.microsoft.com/sitemaps/microsoft-teams/sitemapindex.xml\r\nSitemap: https://www.microsoft.com/sitemaps/power-platform/sitemapindex.xml\r\nSitemap: https://www.microsoft.com/sitemaps/microsoft-viva/sitemapindex.xml\r\nSitemap: https://www.microsoft.com/sitemaps/security/sitemapindex.xml\r\nSitemap: https://www.microsoft.com/sitemaps/trust-center/sitemapindex.xml\r\nSitemap: https://www.microsoft.com/sitemaps/windows-365/sitemapindex.xml\r\nSitemap: https://www.microsoft.com/sitemaps/customers/sitemapindex.xml\r\nSitemap: https://www.microsoft.com/sitemaps/education/sitemapindex.xml\r\nSitemap: https://www.microsoft.com/sitemaps/industry/sitemapindex.xml\r\nSitemap: https://www.microsoft.com/sitemaps/microsoft-cloud/sitemapindex.xml\r\nSitemap: https://www.microsoft.com/sitemaps/sustainability/sitemapindex.xml\r\nSitemap: https://www.microsoft.com/en-us/microsoft-365-life-hacks/sitemap.xml\r\nSitemap: https://www.microsoft.com/windows/business/sitemap.xml","agents":{"GPTBot":true,"OAI-SearchBot":true,"ChatGPT-User":true,"ClaudeBot":true,"Claude-SearchBot":true,"PerplexityBot":true,"Google-Extended":true,"Applebot-Extended":true,"CCBot":true,"Bytespider":true,"Meta-ExternalAgent":true,"Amazonbot":true}},"sitemap":{"status":200,"content_type":"application/xml","bytes":662,"sha256":"189323c8e54460f16570aaf2273a36f393cda75cc789bb1a626a147be7e989cb","final_host":"www.microsoft.com","ms":15,"url":"https://www.microsoft.com/store/sitemap-index.xml","declared":true,"xml":true},"files":{"llms_txt":{"status":404,"content_type":"text/html","bytes":76859,"sha256":"6769df293e90f9bf5e31c25958f1ea6c3b7715aa49f3176afe7694e324e0e6b1","final_host":"www.microsoft.com","ms":191,"path":"/llms.txt","html":true,"json":null,"present":false},"agents_md":{"status":404,"content_type":"text/html","bytes":76859,"sha256":"6769df293e90f9bf5e31c25958f1ea6c3b7715aa49f3176afe7694e324e0e6b1","final_host":"www.microsoft.com","ms":127,"path":"/agents.md","html":true,"json":null,"present":false},"agent_skills":{"status":404,"content_type":"text/html","bytes":76859,"sha256":"6769df293e90f9bf5e31c25958f1ea6c3b7715aa49f3176afe7694e324e0e6b1","final_host":"www.microsoft.com","ms":84,"path":"/.well-known/agent-skills/index.json","html":true,"json":null,"present":false},"mcp_card":{"status":404,"content_type":"text/html","bytes":76859,"sha256":"6769df293e90f9bf5e31c25958f1ea6c3b7715aa49f3176afe7694e324e0e6b1","final_host":"www.microsoft.com","ms":150,"path":"/.well-known/mcp/server-card.json","html":true,"json":null,"present":false},"api_catalog":{"status":404,"content_type":"text/html","bytes":76859,"sha256":"6769df293e90f9bf5e31c25958f1ea6c3b7715aa49f3176afe7694e324e0e6b1","final_host":"www.microsoft.com","ms":179,"path":"/.well-known/api-catalog","html":true,"json":null,"present":false},"a2a_card":{"status":404,"content_type":"text/html","bytes":76859,"sha256":"6769df293e90f9bf5e31c25958f1ea6c3b7715aa49f3176afe7694e324e0e6b1","final_host":"www.microsoft.com","ms":109,"path":"/.well-known/agent-card.json","html":true,"json":null,"present":false}},"rings":{"v":1,"candidates":[],"mcp":[],"a2a":null,"webmcp":{"model_context":false,"bridge":false,"mcp_json":false,"link_rel":false,"any":false},"handshakes":0,"get_path":{"status":200,"content_type":"text/html","ms":591,"error":null},"get_host":{"status":200,"content_type":"text/html","ms":412,"error":null}},"derived":{"reachable":true,"robots_served":true,"robots_sitemap":true,"content_signal":false,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":[],"blocks_any_ai":false,"files":{"llms_txt":false,"agents_md":false,"agent_skills":false,"mcp_card":false,"api_catalog":false,"a2a_card":false},"any_capability":false,"ai_refused":[],"ai_thin":[],"rings":{"mcp_declared":false,"mcp_live":false,"mcp_auth":false,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":2,"fh":"www.microsoft.com","fp_other_host":[],"fp":"e83d250dca983ab7","fp_bits":{"gptbot":0,"claudebot":0,"oaisearch":0,"perplexity":0}},"derive_v":3,"frame":"top1k","shard":7},{"v":1,"kind":"crawlcheck-census-record","run":14,"domain":"youtube.com","rank":9,"measured_at":"2026-10-15T05:26:03.595Z","home":{"browser":{"status":200,"content_type":"text/html","bytes":371383,"sha256":"0cb5a6da553d21dbc9c1d4ee5f82ccaca9264e820dca44ffdc1b41fbed0eb71b","final_host":"m.youtube.com","ms":380,"identity":"Mobile browser","html":true},"gptbot":{"status":200,"content_type":"text/html","bytes":995518,"sha256":"01a8743a408d9c2bca5ab5e028460515cfcd4124d95ef55d3a75ca9a64ab696c","final_host":"www.youtube.com","ms":203,"identity":"GPTBot","html":true},"claudebot":{"status":200,"content_type":"text/html","bytes":995249,"sha256":"e3f12a066b8270f172888150fbb089354936a6609f8c36d46d0e389a1b0d77aa","final_host":"www.youtube.com","ms":192,"identity":"ClaudeBot","html":true},"oaisearch":{"status":200,"content_type":"text/html","bytes":994553,"sha256":"bbd3c2cfbc339b36334d62d1129cc17b434f57b64e0f25f40e2defa6f8749646","final_host":"www.youtube.com","ms":150,"identity":"OAI-SearchBot","html":true},"perplexity":{"status":200,"content_type":"text/html","bytes":953740,"sha256":"9977fbe740838f0568c196b76dde5ee6a5103b2272912179c402bc0bdbbe0a2f","final_host":"www.youtube.com","ms":187,"identity":"PerplexityBot","html":true}},"robots":{"status":200,"content_type":"text/plain","bytes":792,"sha256":"96f8640ca357626e7d9a894057fef8b5746eb838c29ed68fc176af58c9348c6f","final_host":"www.youtube.com","ms":78,"html":false,"text":"# robots.txt file for YouTube\n# Created in the distant future (the year 2000) after\n# the robotic uprising of the mid 90's which wiped out all humans.\n\nUser-agent: Mediapartners-Google*\nDisallow:\n\nUser-agent: *\nDisallow: /api/\nDisallow: /channel_picker\nDisallow: /comment\nDisallow: /feeds/videos.xml\nDisallow: /file_download\nDisallow: /get_video\nDisallow: /get_video_info\nDisallow: /get_midroll_info\nDisallow: /live_chat\nDisallow: /login\nDisallow: /qr\nDisallow: /results\nDisallow: /signup\nDisallow: /t/terms\nDisallow: /timedtext_video\nDisallow: /verify_age\nDisallow: /watch_ajax\nDisallow: /watch_fragments_ajax\nDisallow: /watch_popup\nDisallow: /watch_queue_ajax\nDisallow: /youtubei/\n\nSitemap: https://www.youtube.com/sitemaps/sitemap.xml\nSitemap: https://www.youtube.com/product/sitemap.xml\n\n","agents":{"GPTBot":true,"OAI-SearchBot":true,"ChatGPT-User":true,"ClaudeBot":true,"Claude-SearchBot":true,"PerplexityBot":true,"Google-Extended":true,"Applebot-Extended":true,"CCBot":true,"Bytespider":true,"Meta-ExternalAgent":true,"Amazonbot":true}},"sitemap":{"status":200,"content_type":"text/xml","bytes":2045,"sha256":"89bea4222871c7b633fc750e5f1b2fd503ab80ecb9c6cf7aae7de3cf3bbddfbb","final_host":"www.youtube.com","ms":47,"url":"https://www.youtube.com/sitemaps/sitemap.xml","declared":true,"xml":true},"files":{"llms_txt":{"status":404,"content_type":"text/html","bytes":758,"sha256":"4a9ec8516a0d80bd44d17a40f30377dbfd0c249c28930b72c004612b2ebbb246","final_host":"www.youtube.com","ms":254,"path":"/llms.txt","html":true,"json":null,"present":false},"agents_md":{"status":404,"content_type":"text/html","bytes":758,"sha256":"482dfb81249dcd1cba80b81ab9b848c814cadafc211bf8b9eec88d15389064c0","final_host":"www.youtube.com","ms":244,"path":"/agents.md","html":true,"json":null,"present":false},"agent_skills":{"status":404,"content_type":"text/html","bytes":758,"sha256":"436debff143e487bac1fc6e79d24b92d9a0c171a4c1c32c05b25ab2e117e57b3","final_host":"www.youtube.com","ms":199,"path":"/.well-known/agent-skills/index.json","html":true,"json":null,"present":false},"mcp_card":{"status":404,"content_type":"text/html","bytes":758,"sha256":"b1fb4be9d2537d505ec2da907f2531ab0c6144c29a06b96916e18499675eca44","final_host":"www.youtube.com","ms":239,"path":"/.well-known/mcp/server-card.json","html":true,"json":null,"present":false},"api_catalog":{"status":404,"content_type":"text/html","bytes":758,"sha256":"1ff50b4a88bdaa7bd5434a8a3bfa34216d3616128e2f541123d8c0301ce08ac6","final_host":"www.youtube.com","ms":199,"path":"/.well-known/api-catalog","html":true,"json":null,"present":false},"a2a_card":{"status":404,"content_type":"text/html","bytes":758,"sha256":"0bb6e555302fd814ff6b21e25cadc03ab5e467491b487df503e48ea02f8bc050","final_host":"www.youtube.com","ms":196,"path":"/.well-known/agent-card.json","html":true,"json":null,"present":false}},"rings":{"v":1,"candidates":[],"mcp":[],"a2a":null,"webmcp":{"model_context":false,"bridge":false,"mcp_json":false,"link_rel":false,"any":false},"handshakes":0,"get_path":{"status":404,"content_type":"text/html","ms":206,"error":null},"get_host":{"status":530,"content_type":"text/plain","ms":92,"error":null}},"derived":{"reachable":true,"robots_served":true,"robots_sitemap":true,"content_signal":false,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":[],"blocks_any_ai":false,"files":{"llms_txt":false,"agents_md":false,"agent_skills":false,"mcp_card":false,"api_catalog":false,"a2a_card":false},"any_capability":false,"ai_refused":[],"ai_thin":[],"rings":{"mcp_declared":false,"mcp_live":false,"mcp_auth":false,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":2,"fh":"m.youtube.com","fp_other_host":[]},"derive_v":3,"frame":"top1k","shard":0},{"v":1,"kind":"crawlcheck-census-record","run":14,"domain":"apple.com","rank":10,"measured_at":"2026-10-15T05:27:03.729Z","home":{"browser":{"status":200,"content_type":"text/html","bytes":257835,"sha256":"2450711ec577b416b433e9696dc8751406e26befdb0a46233990c3691cff34a2","final_host":"www.apple.com","ms":143,"identity":"Mobile browser","html":true,"fp":"174e86e85ce0244e"},"gptbot":{"status":200,"content_type":"text/html","bytes":257835,"sha256":"2450711ec577b416b433e9696dc8751406e26befdb0a46233990c3691cff34a2","final_host":"www.apple.com","ms":183,"identity":"GPTBot","html":true,"fp":"174e86e85ce0244e"},"claudebot":{"status":200,"content_type":"text/html","bytes":257835,"sha256":"2450711ec577b416b433e9696dc8751406e26befdb0a46233990c3691cff34a2","final_host":"www.apple.com","ms":173,"identity":"ClaudeBot","html":true,"fp":"174e86e85ce0244e"},"oaisearch":{"status":200,"content_type":"text/html","bytes":257835,"sha256":"2450711ec577b416b433e9696dc8751406e26befdb0a46233990c3691cff34a2","final_host":"www.apple.com","ms":177,"identity":"OAI-SearchBot","html":true,"fp":"174e86e85ce0244e"},"perplexity":{"status":200,"content_type":"text/html","bytes":257835,"sha256":"2450711ec577b416b433e9696dc8751406e26befdb0a46233990c3691cff34a2","final_host":"www.apple.com","ms":187,"identity":"PerplexityBot","html":true,"fp":"174e86e85ce0244e"}},"robots":{"status":200,"content_type":"text/plain","bytes":1018,"sha256":"3115f45c98cfa19341978dcab5dfbc0fd3e0ff4eb3ab1fe8192c3fbf7227dee7","final_host":"www.apple.com","ms":183,"html":false,"text":"# robots.txt for https://www.apple.com/\n\nUser-agent: *\nDisallow: /*shop/browse/overlay/*\nDisallow: /*shop/iphone/payments/overlay/*\nDisallow: /cn/*/aow/*\nDisallow: /tmall*\nAllow: /ac/globalnav/2.0/*/images/ac-globalnav/globalnav/search/* \t\n\nUser-agent: Baiduspider\nDisallow: /*\nAllow: /cn/*\nAllow: /cn-edu/*\nAllow: /cn-k12/*\n\nUser-agent: HaoSouSpider\nDisallow: /*\nAllow: /cn/*\nAllow: /cn-edu/*\nAllow: /cn-k12/*\n\nUser-agent: Sogou web spider\nDisallow: /*\nAllow: /cn/*\nAllow: /cn-k12/*\n\nUser-agent: Sogou inst spider\nDisallow: /*\nAllow: /cn/*\nAllow: /cn-k12/*\n\nUser-agent: Sogou spider2\nDisallow: /*\nAllow: /cn/*\nAllow: /cn-k12/*\n\n#DaumWebMasterTool:6ff40e0f09564a9d4e3bc80e7cbde7c196677a2f5f610eb9c029836774f9a8e9:D9a8IEFAzmC/NWsQ3tB0Wg==\n\nSitemap: https://www.apple.com/shop/sitemap.xml\nSitemap: https://www.apple.com/autopush/sitemap/sitemap-index.xml\nSitemap: https://www.apple.com/newsroom/sitemap.xml\nSitemap: https://www.apple.com/retail/sitemap/sitemap.xml\nSitemap: https://www.apple.com/today/sitemap-index.xml\n","agents":{"GPTBot":true,"OAI-SearchBot":true,"ChatGPT-User":true,"ClaudeBot":true,"Claude-SearchBot":true,"PerplexityBot":true,"Google-Extended":true,"Applebot-Extended":true,"CCBot":true,"Bytespider":true,"Meta-ExternalAgent":true,"Amazonbot":true}},"sitemap":{"status":200,"content_type":"text/xml","bytes":3776,"sha256":"6143254373f03e6b21fad78999cd596f2e91f01925f97b588d1e337c36a558e5","final_host":"www.apple.com","ms":183,"url":"https://www.apple.com/shop/sitemap.xml","declared":true,"xml":true},"files":{"llms_txt":{"status":404,"content_type":"text/html","bytes":109400,"sha256":"2c33262e1cb35568064b86d703038e420bda53717cb88b78b7ff81e861ac5cde","final_host":"www.apple.com","ms":367,"path":"/llms.txt","html":true,"json":null,"present":false},"agents_md":{"status":404,"content_type":"text/html","bytes":109400,"sha256":"2c33262e1cb35568064b86d703038e420bda53717cb88b78b7ff81e861ac5cde","final_host":"www.apple.com","ms":382,"path":"/agents.md","html":true,"json":null,"present":false},"agent_skills":{"status":404,"content_type":"text/html","bytes":196,"sha256":"80c3fe2ae1062abf56456f52518bd670f9ec3917b7f85e152b347ac6b6faf880","final_host":"www.apple.com","ms":313,"path":"/.well-known/agent-skills/index.json","html":true,"json":null,"present":false},"mcp_card":{"status":404,"content_type":"text/html","bytes":196,"sha256":"80c3fe2ae1062abf56456f52518bd670f9ec3917b7f85e152b347ac6b6faf880","final_host":"www.apple.com","ms":316,"path":"/.well-known/mcp/server-card.json","html":true,"json":null,"present":false},"api_catalog":{"status":404,"content_type":"text/html","bytes":196,"sha256":"80c3fe2ae1062abf56456f52518bd670f9ec3917b7f85e152b347ac6b6faf880","final_host":"www.apple.com","ms":316,"path":"/.well-known/api-catalog","html":true,"json":null,"present":false},"a2a_card":{"status":404,"content_type":"text/html","bytes":196,"sha256":"80c3fe2ae1062abf56456f52518bd670f9ec3917b7f85e152b347ac6b6faf880","final_host":"www.apple.com","ms":307,"path":"/.well-known/agent-card.json","html":true,"json":null,"present":false}},"rings":{"v":1,"candidates":[],"mcp":[],"a2a":null,"webmcp":{"model_context":false,"bridge":false,"mcp_json":false,"link_rel":false,"any":false},"handshakes":0,"get_path":{"status":404,"content_type":"text/html","ms":611,"error":null},"get_host":{"status":530,"content_type":"text/plain","ms":61,"error":null}},"derived":{"reachable":true,"robots_served":true,"robots_sitemap":true,"content_signal":false,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":[],"blocks_any_ai":false,"files":{"llms_txt":false,"agents_md":false,"agent_skills":false,"mcp_card":false,"api_catalog":false,"a2a_card":false},"any_capability":false,"ai_refused":[],"ai_thin":[],"rings":{"mcp_declared":false,"mcp_live":false,"mcp_auth":false,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":2,"fh":"www.apple.com","fp_other_host":[],"fp":"174e86e85ce0244e","fp_bits":{"gptbot":0,"claudebot":0,"oaisearch":0,"perplexity":0}},"derive_v":3,"frame":"top1k","shard":1},{"v":1,"kind":"crawlcheck-census-record","run":14,"domain":"instagram.com","rank":11,"measured_at":"2026-10-15T05:28:03.923Z","home":{"browser":{"status":200,"content_type":"text/html","bytes":572910,"sha256":"910690f3c23aa6557ecee42299da47276030755e86e088c58eeb69c0f97f5ced","final_host":"www.instagram.com","ms":714,"identity":"Mobile browser","html":true},"gptbot":{"status":200,"content_type":"text/html","bytes":598260,"sha256":"16ea04a0b24d55b030a6a039e091aabc116c8cf95755822ccfc23c2bb9c00834","final_host":"www.instagram.com","ms":500,"identity":"GPTBot","html":true},"claudebot":{"status":200,"content_type":"text/html","bytes":598190,"sha256":"c83190399fb51955c88862d4beee9ed357bab6f418a96387457cf41deaf8d66e","final_host":"www.instagram.com","ms":769,"identity":"ClaudeBot","html":true},"oaisearch":{"status":200,"content_type":"text/html","bytes":598298,"sha256":"b6ba7faa33e275adbb7be67b8bd46d5ed4b4c0cfb80f1440c55bda6bc5f8f0fd","final_host":"www.instagram.com","ms":757,"identity":"OAI-SearchBot","html":true},"perplexity":{"status":200,"content_type":"text/html","bytes":409574,"sha256":"e697af4467b0834c7393e3d58f83067dd3c68c848e3ac2e073698b5b42eb56d3","final_host":"www.instagram.com","ms":714,"identity":"PerplexityBot","html":true,"fp":"b8e8896ab3101937"}},"robots":{"status":200,"content_type":"text/plain","bytes":7336,"sha256":"c326b09ffb3c383b77140a56097e3ad056ed9e57040fe70269c17460da6d346b","final_host":"www.instagram.com","ms":621,"html":false,"text":"# Notice: Collection of data on Instagram through automated means is\n# prohibited unless you have express written permission from Instagram\n# and may only be conducted for the limited purpose contained in said\n# permission.\n# All authorized user-agents listed on this page must comply with Meta’s\n# Automated Data Collection Terms available at:\n# https://www.facebook.com/legal/automated_data_collection_terms\nUser-agent: Google-Extended\nDisallow: /\n\nUser-agent: PetalBot\nDisallow: /\n\nUser-agent: Scrapy\nDisallow: /\n\nUser-agent: uptimerobot\nDisallow: /\n\nUser-agent: viberbot\nDisallow: /\n\nUser-agent: YaK\nDisallow: /\n\nUser-agent: Yandex\nDisallow: /\n\nUser-agent: Yeti\nDisallow: /\n\nUser-agent: Applebot\nDisallow: /accounts/login/*?next=\nDisallow: /ajax/\nDisallow: /client_error/\nDisallow: /direct/\nDisallow: /logging/\nDisallow: /publicapi/\nDisallow: /qp/batch_fetch_web/\nDisallow: /query/\nAllow: /places/c/\n\nUser-agent: Bingbot\nDisallow: /*/c/\nDisallow: /*/comments/\nDisallow: /*/liked_by/\nDisallow: /accounts/login/*?next=\nDisallow: /ajax/\nDisallow: /client_error/\nDisallow: /direct/\nDisallow: /logging/\nDisallow: /publicapi/\nDisallow: /qp/batch_fetch_web/\nDisallow: /query/\nAllow: /places/c/\n\nUser-agent: deepcrawl\nDisallow: /accounts/login/*?next=\nDisallow: /ajax/\nDisallow: /client_error/\nDisallow: /direct/\nDisallow: /logging/\nDisallow: /publicapi/\nDisallow: /qp/batch_fetch_web/\nDisallow: /query/\nAllow: /places/c/\n\nUser-agent: Discordbot\nDisallow: /accounts/login/*?next=\nDisallow: /ajax/\nDisallow: /client_error/\nDisallow: /direct/\nDisallow: /logging/\nDisallow: /publicapi/\nDisallow: /qp/batch_fetch_web/\nDisallow: /query/\nAllow: /places/c/\n\nUser-agent: DuckDuckBot\nDisallow: /accounts/login/*?next=\nDisallow: /ajax/\nDisallow: /client_error/\nDisallow: /direct/\nDisallow: /logging/\nDisallow: /publicapi/\nDisallow: /qp/batch_fetch_web/\nDisallow: /query/\nAllow: /places/c/\n\nUser-agent: facebookexternalhit\nDisallow: /accounts/login/*?next=\nDisallow: /ajax/\nDisallow: /client_error/\nDisallow: /direct/\nDisallow: /logging/\nDisallow: /publicapi/\nDisallow: /qp/batch_fetch_web/\nDisallow: /query/\nAllow: /places/c/\n\nUser-agent: Googlebot\nDisallow: /*/c/\nDisallow: /*/comments/\nDisallow: /*/liked_by/\nDisallow: /accounts/login/*?next=\nDisallow: /ajax/\nDisallow: /client_error/\nDisallow: /direct/\nDisallow: /logging/\nDisallow: /publicapi/\nDisallow: /qp/batch_fetch_web/\nDisallow: /query/\nAllow: /places/c/\n\nUser-agent: Googlebot-Image\nDisallow: /accounts/login/*?next=\nDisallow: /ajax/\nDisallow: /client_error/\nDisallow: /direct/\nDisallow: /logging/\nDisallow: /publicapi/\nDisallow: /qp/batch_fetch_web/\nDisallow: /query/\nAllow: /places/c/\n\nUser-agent: Googlebot-Video\nDisallow: /accounts/login/*?next=\nDisallow: /ajax/\nDisallow: /client_error/\nDisallow: /direct/\nDisallow: /logging/\nDisallow: /publicapi/\nDisallow: /qp/batch_fetch_web/\nDisallow: /query/\nAllow: /places/c/\n\nUser-agent: Googlebot-News\nDisallow: /accounts/login/*?next=\nDisallow: /ajax/\nDisallow: /client_error/\nDisallow: /direct/\nDisallow: /logging/\nDisallow: /publicapi/\nDisallow: /qp/batch_fetch_web/\nDisallow: /query/\nAllow: /places/c/\n\nUser-agent: Google-InspectionTool\nDisallow: /accounts/login/*?next=\nDisallow: /ajax/\nDisallow: /client_error/\nDisallow: /direct/\nDisallow: /logging/\nDisallow: /publicapi/\nDisallow: /qp/batch_fetch_web/\nDisallow: /query/\nAllow: /places/c/\n\nUser-agent: lumar\nDisallow: /accounts/login/*?next=\nDisallow: /ajax/\nDisallow: /client_error/\nDisallow: /direct/\nDisallow: /logging/\nDisallow: /publicapi/\nDisallow: /qp/batch_fetch_web/\nDisallow: /query/\nAllow: /places/c/\n\nUser-agent: LinkedInBot\nDisallow: /accounts/login/*?next=\nDisallow: /ajax/\nDisallow: /client_error/\nDisallow: /direct/\nDisallow: /logging/\nDisallow: /publicapi/\nDisallow: /qp/batch_fetch_web/\nDisallow: /query/\nAllow: /places/c/\n\nUser-agent: msnbot\nDisallow: /accounts/login/*?next=\nDisallow: /ajax/\nDisallow: /client_error/\nDisallow: /direct/\nDisallow: /logging/\nDisallow: /publicapi/\nDisallow: /qp/batch_fetch_web/\nDisallow: /query/\nAllow: /places/c/\n\nUser-agent: Pinterestbot\nDisallow: /accounts/login/*?next=\nDisallow: /ajax/\nDisallow: /client_error/\nDisallow: /direct/\nDisallow: /logging/\nDisallow: /publicapi/\nDisallow: /qp/batch_fetch_web/\nDisallow: /query/\nAllow: /places/c/\n\nUser-agent: Screaming Frog SEO Spider\nDisallow: /accounts/login/*?next=\nDisallow: /ajax/\nDisallow: /client_error/\nDisallow: /direct/\nDisallow: /logging/\nDisallow: /publicapi/\nDisallow: /qp/batch_fetch_web/\nDisallow: /query/\nAllow: /places/c/\n\nUser-agent: seznambot\nDisallow: /accounts/login/*?next=\nDisallow: /ajax/\nDisallow: /client_error/\nDisallow: /direct/\nDisallow: /logging/\nDisallow: /publicapi/\nDisallow: /qp/batch_fetch_web/\nDisallow: /query/\nAllow: /places/c/\n\nUser-agent: Slurp\nDisallow: /accounts/login/*?next=\nDisallow: /ajax/\nDisallow: /client_error/\nDisallow: /direct/\nDisallow: /logging/\nDisallow: /publicapi/\nDisallow: /qp/batch_fetch_web/\nDisallow: /query/\nAllow: /places/c/\n\nUser-agent: teoma\nDisallow: /accounts/login/*?next=\nDisallow: /ajax/\nDisallow: /client_error/\nDisallow: /direct/\nDisallow: /logging/\nDisallow: /publicapi/\nDisallow: /qp/batch_fetch_web/\nDisallow: /query/\nAllow: /places/c/\n\nUser-agent: TelegramBot\nDisallow: /accounts/login/*?next=\nDisallow: /ajax/\nDisallow: /client_error/\nDisallow: /direct/\nDisallow: /logging/\nDisallow: /publicapi/\nDisallow: /qp/batch_fetch_web/\nDisallow: /query/\nAllow: /places/c/\n\nUser-agent: Twitterbot\nDisallow: /accounts/login/*?next=\nDisallow: /ajax/\nDisallow: /client_error/\nDisallow: /direct/\nDisallow: /logging/\nDisallow: /publicapi/\nDisallow: /qp/batch_fetch_web/\nDisallow: /query/\nAllow: /places/c/\n\nUser-agent: Amazonbot\nDisallow: /accounts/login/*?next=\nDisallow: /ajax/\nDisallow: /client_error/\nDisallow: /direct/\nDisallow: /logging/\nDisallow: /publicapi/\nDisallow: /qp/batch_fetch_web/\nDisallow: /query/\nAllow: /places/c/\n\nUser-agent: Applebot-Extended\nDisallow: /accounts/login/*?next=\nDisallow: /ajax/\nDisallow: /client_error/\nDisallow: /direct/\nDisallow: /logging/\nDisallow: /publicapi/\nDisallow: /qp/batch_fetch_web/\nDisallow: /query/\nAllow: /places/c/\n\nUser-agent: Brightbot\nDisallow: /accounts/login/*?next=\nDisallow: /ajax/\nDisallow: /client_error/\nDisallow: /direct/\nDisallow: /logging/\nDisallow: /publicapi/\nDisallow: /qp/batch_fetch_web/\nDisallow: /query/\nAllow: /places/c/\n\nUser-agent: ClaudeBot\nDisallow: /accounts/login/*?next=\nDisallow: /ajax/\nDisallow: /client_error/\nDisallow: /direct/\nDisallow: /logging/\nDisallow: /publicapi/\nDisallow: /qp/batch_fetch_web/\nDisallow: /query/\nAllow: /places/c/\n\nUser-agent: GPTBot\nDisallow: /accounts/login/*?next=\nDisallow: /ajax/\nDisallow: /client_error/\nDisallow: /direct/\nDisallow: /logging/\nDisallow: /publicapi/\nDisallow: /qp/batch_fetch_web/\nDisallow: /query/\nAllow: /places/c/\n\nUser-agent: PerplexityBot\nDisallow: /accounts/login/*?next=\nDisallow: /ajax/\nDisallow: /client_error/\nDisallow: /direct/\nDisallow: /logging/\nDisallow: /publicapi/\nDisallow: /qp/batch_fetch_web/\nDisallow: /query/\nAllow: /places/c/\n\nUser-agent: *\nDisallow: /\n\nSitemap: https://www.instagram.com/sitemap/ig_places_sitemap.xml.gz\nSitemap: https://www.instagram.com/sitemap/ig_seo_profile_sitemap.xml.gz\nSitemap: https://www.instagram.com/sitemap/ig_seo_profile_sitemap_non_media_eligible.xml.gz\nSitemap: https://www.instagram.com/sitemap/dim_ig_logged_out_web_keyword_ent.xml.gz\n\n","agents":{"GPTBot":true,"OAI-SearchBot":false,"ChatGPT-User":false,"ClaudeBot":true,"Claude-SearchBot":false,"PerplexityBot":true,"Google-Extended":false,"Applebot-Extended":true,"CCBot":false,"Bytespider":false,"Meta-ExternalAgent":false,"Amazonbot":true}},"sitemap":{"status":403,"content_type":"text/html","bytes":0,"sha256":"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855","final_host":"www.instagram.com","ms":381,"url":"https://www.instagram.com/sitemap/ig_places_sitemap.xml.gz","declared":true,"xml":false},"files":{"llms_txt":{"status":429,"content_type":"text/html","bytes":0,"sha256":"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855","final_host":"www.instagram.com","ms":594,"path":"/llms.txt","html":true,"json":null,"present":false},"agents_md":{"status":429,"content_type":"text/html","bytes":0,"sha256":"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855","final_host":"www.instagram.com","ms":404,"path":"/agents.md","html":true,"json":null,"present":false},"agent_skills":{"status":404,"content_type":"text/html","bytes":585319,"sha256":"e8cc07d33d62ba836d7ea95836eca1e07a6fa48620d07389131559504a02209e","final_host":"www.instagram.com","ms":732,"path":"/.well-known/agent-skills/index.json","html":true,"json":null,"present":false},"mcp_card":{"status":404,"content_type":"text/html","bytes":585296,"sha256":"fec19069fa22e9e123a95b8048601617bfdac89a20654e2a136d3441ca42260f","final_host":"www.instagram.com","ms":483,"path":"/.well-known/mcp/server-card.json","html":true,"json":null,"present":false},"api_catalog":{"status":429,"content_type":"text/html","bytes":0,"sha256":"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855","final_host":"www.instagram.com","ms":466,"path":"/.well-known/api-catalog","html":true,"json":null,"present":false},"a2a_card":{"status":429,"content_type":"text/html","bytes":0,"sha256":"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855","final_host":"www.instagram.com","ms":584,"path":"/.well-known/agent-card.json","html":true,"json":null,"present":false}},"rings":{"v":1,"candidates":[],"mcp":[],"a2a":null,"webmcp":{"model_context":false,"bridge":false,"mcp_json":false,"link_rel":false,"any":false},"handshakes":0,"get_path":{"status":429,"content_type":"text/html","ms":404,"error":null},"get_host":{"status":530,"content_type":"text/plain","ms":43,"error":null}},"derived":{"reachable":true,"robots_served":true,"robots_sitemap":true,"content_signal":false,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":["OAI-SearchBot","ChatGPT-User","Claude-SearchBot","Google-Extended","CCBot","Bytespider","Meta-ExternalAgent"],"blocks_any_ai":true,"files":{"llms_txt":false,"agents_md":false,"agent_skills":false,"mcp_card":false,"api_catalog":false,"a2a_card":false},"any_capability":false,"ai_refused":[],"ai_thin":[],"rings":{"mcp_declared":false,"mcp_live":false,"mcp_auth":false,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":2,"fh":"www.instagram.com","fp_other_host":[]},"derive_v":3,"frame":"top1k","shard":2},{"v":1,"kind":"crawlcheck-census-record","run":14,"domain":"mail.ru","rank":12,"measured_at":"2026-10-15T05:29:03.596Z","home":{"browser":{"status":200,"content_type":"text/html","bytes":423988,"sha256":"fe2ad5ade1fcbabb6ca3897d1987f69eed9a01c3d5290361347827d47c9d5065","final_host":"mail.ru","ms":486,"identity":"Mobile browser","html":true,"fp":"9089414431008018"},"gptbot":{"status":200,"content_type":"text/html","bytes":936,"sha256":"dddb4d15a162533836fb9b6133c166d10c31b676bee8f1e98609916725d345c8","final_host":"account.mail.ru","ms":1369,"identity":"GPTBot","html":true},"claudebot":{"status":200,"content_type":"text/html","bytes":936,"sha256":"c49b3288fe6484115382aa737bf2bbe476a697908681adad7c39247acf95fca3","final_host":"account.mail.ru","ms":1391,"identity":"ClaudeBot","html":true},"oaisearch":{"status":200,"content_type":"text/html","bytes":936,"sha256":"11f5677b6ae929a9c5061db64c8080806d25db2e191568ee4c52d0ebd4a2f8b8","final_host":"account.mail.ru","ms":1360,"identity":"OAI-SearchBot","html":true},"perplexity":{"status":200,"content_type":"text/html","bytes":936,"sha256":"135bb5cd65579d5f7fd772f4b5f5ffb996e71fa0b1ab61f864b745575f7f5a40","final_host":"account.mail.ru","ms":1368,"identity":"PerplexityBot","html":true}},"robots":{"status":200,"content_type":"text/plain","bytes":433,"sha256":"d3fd6768e203ab4008918459ce8d1ccc9e31d703cc01c9765902bcfc894822bf","final_host":"mail.ru","ms":384,"html":false,"text":"User-agent: *\nAllow: /*.css\nAllow: /*.jpg\nAllow: /*.gif\nAllow: /*.png\nAllow: /v/j/*.js\nAllow: /$\nDisallow: /\n\nUser-agent: Googlebot\nAllow: /*.css\nAllow: /*.jpg\nAllow: /*.gif\nAllow: /*.png\nAllow: /v/j/*.js\nDisallow: *?\nDisallow: *search\nDisallow: *auth\nDisallow: *api\n\nUser-agent: Yandex\nAllow: /*.css\nAllow: /*.jpg\nAllow: /*.gif\nAllow: /*.png\nAllow: /v/j/*.js\nAllow: /$\nDisallow: /\n\nUser-agent: Twitterbot\nDisallow: /\nAllow: /?logo=\n","agents":{"GPTBot":true,"OAI-SearchBot":true,"ChatGPT-User":true,"ClaudeBot":true,"Claude-SearchBot":true,"PerplexityBot":true,"Google-Extended":true,"Applebot-Extended":true,"CCBot":true,"Bytespider":true,"Meta-ExternalAgent":true,"Amazonbot":true}},"sitemap":{"status":429,"content_type":"application/octet-stream","bytes":325,"sha256":"712cd10fff3534d66625b8da937b6517b4362ef332244693a9b6ec20fec2d5a2","final_host":"mail.ru","ms":95,"url":"https://mail.ru/sitemap.xml","declared":false,"xml":false},"files":{"llms_txt":{"status":200,"content_type":"text/html","bytes":936,"sha256":"eabbf19ece2d74eca9067faf69af161aebb6b303ec0a3885044d6a56994b3b13","final_host":"account.mail.ru","ms":309,"path":"/llms.txt","html":true,"json":null,"present":false},"agents_md":{"status":200,"content_type":"text/html","bytes":936,"sha256":"85d5612c2530baf2940af41c9bc573e98be1cdafa6962f766242d43353139b72","final_host":"account.mail.ru","ms":280,"path":"/agents.md","html":true,"json":null,"present":false},"agent_skills":{"status":200,"content_type":"text/html","bytes":936,"sha256":"b4ce3c065b8e9d77dea64afc3d1cb8d78adf5a98f99fb4d52130e51cc2a8e55b","final_host":"account.mail.ru","ms":417,"path":"/.well-known/agent-skills/index.json","html":true,"json":null,"present":false},"mcp_card":{"status":200,"content_type":"text/html","bytes":936,"sha256":"577d65d17b64dea6a92678456ba4fd7dcede1be2eff817cb34a949fab2d1d450","final_host":"account.mail.ru","ms":320,"path":"/.well-known/mcp/server-card.json","html":true,"json":null,"present":false},"api_catalog":{"status":200,"content_type":"text/html","bytes":936,"sha256":"9487c12026c4eb55ef1da2cb9d51b07105f80058a94687be8e40d97945b8f69d","final_host":"account.mail.ru","ms":311,"path":"/.well-known/api-catalog","html":true,"json":null,"present":false},"a2a_card":{"status":429,"content_type":"application/octet-stream","bytes":325,"sha256":"1981762000cfc1085a87214bcdad72f34250cb0e62aa48c74487af38dda65c26","final_host":"mail.ru","ms":47,"path":"/.well-known/agent-card.json","html":false,"json":null,"present":false}},"rings":{"v":1,"candidates":[],"mcp":[],"a2a":null,"webmcp":{"model_context":false,"bridge":false,"mcp_json":false,"link_rel":false,"any":false},"handshakes":0,"get_path":{"status":429,"content_type":"application/octet-stream","ms":50,"error":null},"get_host":{"status":429,"content_type":"application/octet-stream","ms":543,"error":null}},"derived":{"reachable":true,"robots_served":true,"robots_sitemap":false,"content_signal":false,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":[],"blocks_any_ai":false,"files":{"llms_txt":false,"agents_md":false,"agent_skills":false,"mcp_card":false,"api_catalog":false,"a2a_card":false},"any_capability":false,"ai_refused":[],"ai_thin":["gptbot","claudebot","oaisearch","perplexity"],"rings":{"mcp_declared":false,"mcp_live":false,"mcp_auth":false,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":2,"fh":"mail.ru","fp_other_host":["gptbot","claudebot","oaisearch","perplexity"],"fp":"9089414431008018","fp_bits":{}},"derive_v":3,"frame":"top1k","shard":3},{"v":1,"kind":"crawlcheck-census-record","run":14,"domain":"fbcdn.net","rank":13,"measured_at":"2026-10-15T05:30:03.842Z","home":{"browser":{"status":400,"content_type":"text/html","bytes":3676,"sha256":"b52591d3faa4ab5919a45e986e9e4ff28d73e1d7a86a3f1d51a0cf79ff9fb2e2","final_host":"m.facebook.com","ms":825,"identity":"Mobile browser","html":true},"gptbot":{"status":200,"content_type":"text/html","bytes":420358,"sha256":"451b24a9936dbffc933ab9dc29bf733470569dd36f49e33321fd25ea2ec819e7","final_host":"www.facebook.com","ms":804,"identity":"GPTBot","html":true,"fp":"1e33949b66889fb0"},"claudebot":{"status":200,"content_type":"text/html","bytes":420287,"sha256":"ad8a70710dfecea7f161cfaadd0c20f7543b8231441e87a6b92d1dc4a9c16f92","final_host":"www.facebook.com","ms":643,"identity":"ClaudeBot","html":true,"fp":"1e33949b66889fb0"},"oaisearch":{"status":200,"content_type":"text/html","bytes":420815,"sha256":"346ffc9c55e09cadf28dae93836aca8d911d287ca752f4b65177e8c7263192cd","final_host":"www.facebook.com","ms":805,"identity":"OAI-SearchBot","html":true,"fp":"1e33949b66889fb0"},"perplexity":{"status":200,"content_type":"text/html","bytes":375903,"sha256":"a2187cfa3aae743a3756b295f7340522a04aecf5e2d45c08c0ccd243a67d3796","final_host":"www.facebook.com","ms":987,"identity":"PerplexityBot","html":true}},"robots":{"status":200,"content_type":"text/plain","bytes":3355,"sha256":"8155e929427b274a20a74eca43333ed5d5fe3ba4eea4035aea704f57a72f0542","final_host":"fbcdn.net","ms":414,"html":false,"text":"# Notice: Collection of data on Facebook through automated means is\n# prohibited unless you have express written permission from Facebook\n# and may only be conducted for the limited purpose contained in said\n# permission.\n# All authorized user-agents listed on this page must comply with Meta’s\n# Automated Data Collection Terms available at:\n# https://www.facebook.com/legal/automated_data_collection_terms\nUser-agent: Amazonbot\nDisallow: /\n\nUser-agent: Applebot-Extended\nDisallow: /\n\nUser-agent: Brightbot\nDisallow: /\n\nUser-agent: ClaudeBot\nDisallow: /\n\nUser-agent: Google-Extended\nDisallow: /\n\nUser-agent: GPTBot\nDisallow: /\n\nUser-agent: PerplexityBot\nDisallow: /\n\nUser-agent: PetalBot\nDisallow: /\n\nUser-agent: Scrapy\nDisallow: /\n\nUser-agent: uptimerobot\nDisallow: /\n\nUser-agent: viberbot\nDisallow: /\n\nUser-agent: YaK\nDisallow: /\n\nUser-agent: Yandex\nDisallow: /\n\nUser-agent: Yeti\nDisallow: /\n\nUser-agent: Applebot\nDisallow: /\nAllow: /map_tile.php\nAllow: /security/hsts-pixel.gif\nAllow: /static_map.php\n\nUser-agent: Bingbot\nDisallow: /\nAllow: /map_tile.php\nAllow: /security/hsts-pixel.gif\nAllow: /static_map.php\n\nUser-agent: deepcrawl\nDisallow: /\nAllow: /map_tile.php\nAllow: /security/hsts-pixel.gif\nAllow: /static_map.php\n\nUser-agent: Discordbot\nDisallow: /\nAllow: /map_tile.php\nAllow: /security/hsts-pixel.gif\nAllow: /static_map.php\n\nUser-agent: DuckDuckBot\nDisallow: /\nAllow: /map_tile.php\nAllow: /security/hsts-pixel.gif\nAllow: /static_map.php\n\nUser-agent: facebookexternalhit\nDisallow: /\nAllow: /map_tile.php\nAllow: /security/hsts-pixel.gif\nAllow: /static_map.php\n\nUser-agent: Googlebot\nDisallow: /\nAllow: /map_tile.php\nAllow: /security/hsts-pixel.gif\nAllow: /static_map.php\n\nUser-agent: Googlebot-Image\nDisallow: /\nDisallow: /map_tile.php\nDisallow: /static_map.php\nAllow: /map_tile.php\nAllow: /security/hsts-pixel.gif\nAllow: /static_map.php\n\nUser-agent: Googlebot-Video\nDisallow: /\nAllow: /map_tile.php\nAllow: /security/hsts-pixel.gif\nAllow: /static_map.php\n\nUser-agent: Googlebot-News\nDisallow: /\nAllow: /map_tile.php\nAllow: /security/hsts-pixel.gif\nAllow: /static_map.php\n\nUser-agent: Google-InspectionTool\nDisallow: /\nAllow: /map_tile.php\nAllow: /security/hsts-pixel.gif\nAllow: /static_map.php\n\nUser-agent: lumar\nDisallow: /\nAllow: /map_tile.php\nAllow: /security/hsts-pixel.gif\nAllow: /static_map.php\n\nUser-agent: LinkedInBot\nDisallow: /\nAllow: /map_tile.php\nAllow: /security/hsts-pixel.gif\nAllow: /static_map.php\n\nUser-agent: msnbot\nDisallow: /\nAllow: /map_tile.php\nAllow: /security/hsts-pixel.gif\nAllow: /static_map.php\n\nUser-agent: Pinterestbot\nDisallow: /\nAllow: /map_tile.php\nAllow: /security/hsts-pixel.gif\nAllow: /static_map.php\n\nUser-agent: Screaming Frog SEO Spider\nDisallow: /\nAllow: /map_tile.php\nAllow: /security/hsts-pixel.gif\nAllow: /static_map.php\n\nUser-agent: seznambot\nDisallow: /\nAllow: /map_tile.php\nAllow: /security/hsts-pixel.gif\nAllow: /static_map.php\n\nUser-agent: Slurp\nDisallow: /\nAllow: /map_tile.php\nAllow: /security/hsts-pixel.gif\nAllow: /static_map.php\n\nUser-agent: teoma\nDisallow: /\nAllow: /map_tile.php\nAllow: /security/hsts-pixel.gif\nAllow: /static_map.php\n\nUser-agent: TelegramBot\nDisallow: /\nAllow: /map_tile.php\nAllow: /security/hsts-pixel.gif\nAllow: /static_map.php\n\nUser-agent: Twitterbot\nDisallow: /\nAllow: /map_tile.php\nAllow: /security/hsts-pixel.gif\nAllow: /static_map.php\n\nUser-agent: *\nDisallow: /\n\n\n","agents":{"GPTBot":false,"OAI-SearchBot":false,"ChatGPT-User":false,"ClaudeBot":false,"Claude-SearchBot":false,"PerplexityBot":false,"Google-Extended":false,"Applebot-Extended":false,"CCBot":false,"Bytespider":false,"Meta-ExternalAgent":false,"Amazonbot":false}},"sitemap":null,"files":null,"derived":{"reachable":false,"robots_served":false,"robots_sitemap":false,"content_signal":false,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":[],"blocks_any_ai":false,"files":{"llms_txt":false,"agents_md":false,"agent_skills":false,"mcp_card":false,"api_catalog":false,"a2a_card":false},"any_capability":false,"ai_refused":[],"ai_thin":[],"rings":{"mcp_declared":false,"mcp_live":false,"mcp_auth":false,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":2,"fh":"m.facebook.com","fp_other_host":[]},"derive_v":3,"frame":"top1k","shard":4},{"v":1,"kind":"crawlcheck-census-record","run":14,"domain":"ezviz7.com","rank":14,"measured_at":"2026-10-15T05:31:08.958Z","home":{"browser":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"ezviz7.com","ms":555,"identity":"Mobile browser","html":false},"gptbot":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"ezviz7.com","ms":527,"identity":"GPTBot","html":false},"claudebot":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"ezviz7.com","ms":524,"identity":"ClaudeBot","html":false},"oaisearch":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"ezviz7.com","ms":526,"identity":"OAI-SearchBot","html":false},"perplexity":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"ezviz7.com","ms":524,"identity":"PerplexityBot","html":false}},"robots":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"ezviz7.com","ms":526,"html":false,"text":null,"agents":null},"sitemap":null,"files":null,"derived":{"reachable":false,"robots_served":false,"robots_sitemap":false,"content_signal":false,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":[],"blocks_any_ai":false,"files":{"llms_txt":false,"agents_md":false,"agent_skills":false,"mcp_card":false,"api_catalog":false,"a2a_card":false},"any_capability":false,"ai_refused":[],"ai_thin":[],"rings":{"mcp_declared":false,"mcp_live":false,"mcp_auth":false,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":2,"fh":"ezviz7.com","fp_other_host":[]},"derive_v":3,"frame":"top1k","shard":5},{"v":1,"kind":"crawlcheck-census-record","run":14,"domain":"twitter.com","rank":15,"measured_at":"2026-10-15T05:33:03.639Z","home":{"browser":{"status":200,"content_type":"text/html","bytes":42199,"sha256":"0a07791d0f335064c6dbd5082282a38067bc35908e3412219051ed9e241a03e3","final_host":"x.com","ms":1225,"identity":"Mobile browser","html":true,"fp":"8b4892564ad225ec"},"gptbot":{"status":402,"content_type":"application/json","bytes":55,"sha256":"35e642f0e1bd426b9d8b0ee0284aefe801873dc13c18819fbeeaf2bf47136664","final_host":"x.com","ms":914,"identity":"GPTBot","html":false},"claudebot":{"status":403,"content_type":null,"bytes":0,"sha256":"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855","final_host":"x.com","ms":1112,"identity":"ClaudeBot","html":null},"oaisearch":{"status":402,"content_type":"application/json","bytes":55,"sha256":"35e642f0e1bd426b9d8b0ee0284aefe801873dc13c18819fbeeaf2bf47136664","final_host":"x.com","ms":204,"identity":"OAI-SearchBot","html":false},"perplexity":{"status":402,"content_type":"application/json","bytes":55,"sha256":"35e642f0e1bd426b9d8b0ee0284aefe801873dc13c18819fbeeaf2bf47136664","final_host":"x.com","ms":614,"identity":"PerplexityBot","html":false}},"robots":{"status":200,"content_type":"text/plain","bytes":2678,"sha256":"14a72e33b3e9af07e1e8f5f645bfa00ed3fe9f9c87249d861d369d33d3f3ddc6","final_host":"twitter.com","ms":197,"html":false,"text":"# Google / Bing Search Engine Robots\n# ==================================\n# Shared group so Googlebot and Bingbot always get the same rules.\nUser-agent: Googlebot\nUser-agent: Bingbot\n\nAllow: /*?s=\nAllow: /*?t=\nAllow: /*?ref_src=\nAllow: /hashtag/*?src=\nAllow: /search?q=%23\nAllow: /i/api/\n# Same length as the /*/likes and /*/media rules; RFC 9309 breaks the tie in\n# favor of Allow, so /i/api/ fetches stay crawlable.\nAllow: /i/api/*\nDisallow: /*?lang=en-ss\nAllow: /*?lang=\nDisallow: /search/realtime\nDisallow: /search/users\nDisallow: /search/*/grid\n\nDisallow: /*/analytics\nDisallow: /*/followers\nDisallow: /*/following\nDisallow: /*/verified_followers\n\nDisallow: /account/deactivated\nDisallow: /settings/deactivated\n\n# Only `*` and `$` are metacharacters here (RFC 9309); regex classes match\n# literally. /photo is anchored so /status/*/photo/1 media pages stay open.\nDisallow: /*/likes\nDisallow: /*/likes?\nDisallow: /*/retweets\nDisallow: /*/retweets?\nDisallow: /*/media\nDisallow: /*/media?\nDisallow: /*/photo$\nDisallow: /*/photo?\nAllow: /*?\n\nUser-agent: facebookexternalhit\n\nAllow: /*?lang=\nAllow: /*?s=\nAllow: /*?t=\nAllow: /*?ref_src=\nAllow: /hashtag/*?src=\nAllow: /search?*cashtagRestId=\nAllow: /i/api/\nAllow: /i/api/*\nDisallow: /search?q=\nDisallow: /search/realtime\nDisallow: /search/users\nDisallow: /search/*/grid\n\nDisallow: /*?\nDisallow: /*/followers\nDisallow: /*/following\nDisallow: /*/verified_followers\n\nDisallow: /account/deactivated\nDisallow: /settings/deactivated\n\nDisallow: /*/likes\nDisallow: /*/likes?\nDisallow: /*/retweets\nDisallow: /*/retweets?\nDisallow: /*/media\nDisallow: /*/media?\nDisallow: /*/photo$\nDisallow: /*/photo?\n\nUser-Agent: Google-Extended\nDisallow: *\n\nUser-Agent: FacebookBot\nDisallow: *\n\nUser-agent: Discordbot\nDisallow: *\n\n# Meta AI / product crawlers (product tokens; versions like /1.1 are ignored\n# under RFC 9309 token matching).\nUser-agent: meta-webindexer\nDisallow: *\n\nUser-agent: meta-externalagent\nDisallow: *\n\nUser-agent: meta-externalads\nDisallow: *\n\nUser-agent: meta-externalfetcher\nDisallow: *\n\n# Every bot that might possibly read and respect this file\n# ========================================================\nUser-agent: *\nDisallow: /\n\n\n# WHAT-4882 - Keep notification-email links (/i/u) out of search results.\n# Named crawlers stay un-blocked on purpose: they must crawl /i/u to see its\n# X-Robots-Tag noindex (the robots.txt Noindex directive died in 2019).\nDisallow: /i/u\n\n# Wait 1 second between successive requests. See ONBOARD-2698 for details.\nCrawl-delay: 1\n\n# Independent of user agent. Links in the sitemap are full URLs using https://\n# and need to match the protocol of the sitemap.\nSitemap: https://x.com/sitemap.xml\n","agents":{"GPTBot":false,"OAI-SearchBot":false,"ChatGPT-User":false,"ClaudeBot":false,"Claude-SearchBot":false,"PerplexityBot":false,"Google-Extended":false,"Applebot-Extended":false,"CCBot":false,"Bytespider":false,"Meta-ExternalAgent":false,"Amazonbot":false}},"sitemap":{"status":404,"content_type":"text/html","bytes":36629,"sha256":"7e66d604e2e9c7c2251d17e0d664692d2c0e6aa158aafb09e16f86c607ed0fda","final_host":"x.com","ms":196,"url":"https://x.com/sitemap.xml","declared":true,"xml":false},"files":{"llms_txt":{"status":404,"content_type":"text/html","bytes":36612,"sha256":"1befdcc3a82638c43021c7dd3921237bc8d0a9baeae73cfb54d14ba89edea804","final_host":"x.com","ms":432,"path":"/llms.txt","html":true,"json":null,"present":false},"agents_md":{"status":404,"content_type":"text/html","bytes":36618,"sha256":"be83fb79a67f1af20b718c037edab11f1f246820c4750e0b73786c3fb16fb601","final_host":"x.com","ms":335,"path":"/agents.md","html":true,"json":null,"present":false},"agent_skills":{"status":404,"content_type":"application/json","bytes":68,"sha256":"643770b397d393e8b83954adc74ed48dd2e27ab3abab48f2b79472f07808eeb0","final_host":"x.com","ms":323,"path":"/.well-known/agent-skills/index.json","html":false,"json":null,"present":false},"mcp_card":{"status":404,"content_type":"application/json","bytes":68,"sha256":"643770b397d393e8b83954adc74ed48dd2e27ab3abab48f2b79472f07808eeb0","final_host":"x.com","ms":2195,"path":"/.well-known/mcp/server-card.json","html":false,"json":null,"present":false},"api_catalog":{"status":404,"content_type":"text/html","bytes":3656,"sha256":"b4056ca07cbe8a3c71d16206c45c9d5515c74912647eb67100390f95a8a88600","final_host":"x.com","ms":563,"path":"/.well-known/api-catalog","html":true,"json":null,"present":false},"a2a_card":{"status":404,"content_type":"application/json","bytes":68,"sha256":"643770b397d393e8b83954adc74ed48dd2e27ab3abab48f2b79472f07808eeb0","final_host":"x.com","ms":2226,"path":"/.well-known/agent-card.json","html":false,"json":null,"present":false}},"rings":{"v":1,"candidates":[],"mcp":[],"a2a":null,"webmcp":{"model_context":false,"bridge":false,"mcp_json":false,"link_rel":false,"any":false},"handshakes":0,"get_path":{"status":200,"content_type":"text/html","ms":851,"error":null},"get_host":{"status":530,"content_type":"text/plain","ms":48,"error":null}},"derived":{"reachable":true,"robots_served":true,"robots_sitemap":true,"content_signal":false,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":["GPTBot","OAI-SearchBot","ChatGPT-User","ClaudeBot","Claude-SearchBot","PerplexityBot","Google-Extended","Applebot-Extended","CCBot","Bytespider","Meta-ExternalAgent","Amazonbot"],"blocks_any_ai":true,"files":{"llms_txt":false,"agents_md":false,"agent_skills":false,"mcp_card":false,"api_catalog":false,"a2a_card":false},"any_capability":false,"ai_refused":["gptbot","claudebot","oaisearch","perplexity"],"ai_thin":[],"rings":{"mcp_declared":false,"mcp_live":false,"mcp_auth":false,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":2,"fh":"x.com","fp_other_host":[],"fp":"8b4892564ad225ec","fp_bits":{}},"derive_v":3,"frame":"top1k","shard":6},{"v":1,"kind":"crawlcheck-census-record","run":14,"domain":"domaincontrol.com","rank":16,"measured_at":"2026-10-15T05:34:03.814Z","home":{"browser":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"domaincontrol.com","ms":26,"identity":"Mobile browser","html":false},"gptbot":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"domaincontrol.com","ms":4,"identity":"GPTBot","html":false},"claudebot":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"domaincontrol.com","ms":2,"identity":"ClaudeBot","html":false},"oaisearch":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"domaincontrol.com","ms":2,"identity":"OAI-SearchBot","html":false},"perplexity":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"domaincontrol.com","ms":2,"identity":"PerplexityBot","html":false}},"robots":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"domaincontrol.com","ms":4,"html":false,"text":null,"agents":null},"sitemap":null,"files":null,"derived":{"reachable":false,"robots_served":false,"robots_sitemap":false,"content_signal":false,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":[],"blocks_any_ai":false,"files":{"llms_txt":false,"agents_md":false,"agent_skills":false,"mcp_card":false,"api_catalog":false,"a2a_card":false},"any_capability":false,"ai_refused":[],"ai_thin":[],"rings":{"mcp_declared":false,"mcp_live":false,"mcp_auth":false,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":2,"fh":"domaincontrol.com","fp_other_host":[]},"derive_v":3,"frame":"top1k","shard":7},{"v":1,"kind":"crawlcheck-census-record","run":14,"domain":"dzen.ru","rank":17,"measured_at":"2026-10-15T05:26:03.595Z","home":{"browser":{"status":200,"content_type":"text/html","bytes":3046,"sha256":"81cb1012bc446898f136dfe4b2e8a93d40275aae22c78f37f098bd7da032139a","final_host":"sso.passport.yandex.ru","ms":764,"identity":"Mobile browser","html":true},"gptbot":{"status":200,"content_type":"text/html","bytes":2886,"sha256":"4b7d09f255dcdbb905e571ff3e45c987f80fd50ac0a497d4c61eca438b285802","final_host":"sso.passport.yandex.ru","ms":763,"identity":"GPTBot","html":true},"claudebot":{"status":200,"content_type":"text/html","bytes":2886,"sha256":"46a7b971aefa176962cf0e84cb18a5c6738d0412da3f35e0eabd32c8da46f911","final_host":"sso.passport.yandex.ru","ms":688,"identity":"ClaudeBot","html":true},"oaisearch":{"status":200,"content_type":"text/html","bytes":2886,"sha256":"fe87fc198a5f5aa63fc1b0f4328682fc04e3738b76a0c22520c220a6adaf29ed","final_host":"sso.passport.yandex.ru","ms":699,"identity":"OAI-SearchBot","html":true},"perplexity":{"status":200,"content_type":"text/html","bytes":2883,"sha256":"6b5ad0d3839c4a4c6d59915e6958594f43c77d40e44e9f01f73d155fbb864add","final_host":"sso.passport.yandex.ru","ms":2703,"identity":"PerplexityBot","html":true}},"robots":{"status":200,"content_type":"text/plain","bytes":9794,"sha256":"a6e7d16a66dcbbfe8d64ec966726922ac7736fd28f3a4b5bede7897992c92086","final_host":"dzen.ru","ms":482,"html":false,"text":"User-agent: Yandex\nAllow: /\nAllow: /.well-known/assetlinks.json\nAllow: /*?utm_sourse=55\n\nDisallow: /about*?*\nDisallow: /away\nDisallow: /top$\nDisallow: /search\nDisallow: /money\nDisallow: /onboarding\nDisallow: /news/top\nDisallow: /news/smi\nDisallow: /news/search\nDisallow: /t/\nDisallow: /user/\nDisallow: /embed/\nDisallow: /*/url\nDisallow: /video/games/tags/\nDisallow: /*sso_failed=\nDisallow: /*erid=\nDisallow: /*?utm*\nDisallow: /games\nDisallow: /subscriptions-manager?channel=\n\nDisallow: /articles?\nDisallow: /video?\nDisallow: /discussions\n\nClean-param: &_csrf\nClean-param: &action\nClean-param: &amp\nClean-param: &ampplace\nClean-param: &amupr_id\nClean-param: &appsearch_header\nClean-param: &autoplay\nClean-param: &backoffice\nClean-param: &branding\nClean-param: &briefEditorPublicationId\nClean-param: &bx_sender_conversion_id\nClean-param: &clid\nClean-param: &collection\nClean-param: &comment-request\nClean-param: &comments_data\nClean-param: &content\nClean-param: &contour\nClean-param: &country_code\nClean-param: &device_id\nClean-param: &encoded_pulse_user_id\nClean-param: &encoded_pulse_user_info\nClean-param: &epik\nClean-param: &etext=\nClean-param: &experiment\nClean-param: &f\nClean-param: &family\nClean-param: &fan\nClean-param: &favid\nClean-param: &feed_exp\nClean-param: &feed_filter_source\nClean-param: &firstchars\nClean-param: &flight_id\nClean-param: &from\nClean-param: &from_block\nClean-param: &from_cloud\nClean-param: &from_floors_promo\nClean-param: &from_page\nClean-param: &from_parent_id\nClean-param: &from_parent_type\nClean-param: &from_site\nClean-param: &ga_client_id\nClean-param: &hide_interest_header\nClean-param: &id\nClean-param: &integration\nClean-param: &interview_id\nClean-param: &invite\nClean-param: &item_id\nClean-param: &lang\nClean-param: &link\nClean-param: &loop\nClean-param: &lr\nClean-param: &modal\nClean-param: &mt_click_id\nClean-param: &mt_link_id\nClean-param: &mute\nClean-param: &no_ad\nClean-param: &noredir\nClean-param: &nw\nClean-param: &p\nClean-param: &page\nClean-param: &parent_rid\nClean-param: &persistent_id\nClean-param: &place\nClean-param: &play_on_visible\nClean-param: &pulse_user_id\nClean-param: &query\nClean-param: &ref\nClean-param: &referrer\nClean-param: &referrer_clid\nClean-param: &referrer_place\nClean-param: &repostId\nClean-param: &rid\nClean-param: &rubric\nClean-param: &secdata\nClean-param: &service\nClean-param: &share_to\nClean-param: &sid\nClean-param: &sortby\nClean-param: &source\nClean-param: &sso_failed\nClean-param: &stid\nClean-param: &story\nClean-param: &t\nClean-param: &text\nClean-param: &to\nClean-param: &token\nClean-param: &train\nClean-param: &tst\nClean-param: &tt\nClean-param: &tv\nClean-param: &utr\nClean-param: &uuid\nClean-param: &vertical\nClean-param: &via\nClean-param: &vid\nClean-param: &video_id\nClean-param: &wan\nClean-param: &web_uid\nClean-param: &web_yandexuid\nClean-param: &wizard\nClean-param: &yandexClientId\n\nUser-agent: Googlebot\nAllow: /\nAllow: /.well-known/assetlinks.json\nAllow: /list/*\n\nAllow: /a/*_csrf\nAllow: /a/*action\nAllow: /a/*amp\nAllow: /a/*ampplace\nAllow: /a/*amupr_id\nAllow: /a/*appsearch_header=\nAllow: /a/*autoplay=\nAllow: /a/*backoffice=\nAllow: /a/*branding=\nAllow: /a/*briefEditorPublicationId=\nAllow: /a/*bx_sender_conversion_id=\nAllow: /a/*cachebust=\nAllow: /a/*clid=\nAllow: /a/*collection=\nAllow: /a/*country_code=\nAllow: /a/*erid=\nAllow: /a/*experiment=\nAllow: /a/*feed_exp=\nAllow: /a/*from_cloud=\nAllow: /a/*from_page=\nAllow: /a/*invite=\nAllow: /a/*loop=\nAllow: /a/*pulse_user_id=\nAllow: /a/*sso_failed=\nAllow: /a/*t=\nAllow: /a/*to=\nAllow: /a/*token=\nAllow: /a/*utm_referrer=\nAllow: /a/*utm_source=\nAllow: /a/*?etext=\n\nAllow: /b/*_csrf\nAllow: /b/*action\nAllow: /b/*amp\nAllow: /b/*ampplace\nAllow: /b/*amupr_id\nAllow: /b/*appsearch_header=\nAllow: /b/*autoplay=\nAllow: /b/*backoffice=\nAllow: /b/*branding=\nAllow: /b/*briefEditorPublicationId=\nAllow: /b/*bx_sender_conversion_id=\nAllow: /b/*cachebust=\nAllow: /b/*clid=\nAllow: /b/*collection=\nAllow: /b/*country_code=\nAllow: /b/*erid=\nAllow: /b/*experiment=\nAllow: /b/*feed_exp=\nAllow: /b/*from_cloud=\nAllow: /b/*from_page=\nAllow: /b/*invite=\nAllow: /b/*loop=\nAllow: /b/*pulse_user_id=\nAllow: /b/*sso_failed=\nAllow: /b/*t=\nAllow: /b/*to=\nAllow: /b/*token=\nAllow: /b/*utm_referrer=\nAllow: /b/*utm_source=\nAllow: /b/*?etext=\n\nAllow: /list/*_csrf\nAllow: /list/*action\nAllow: /list/*amp\nAllow: /list/*ampplace\nAllow: /list/*amupr_id\nAllow: /list/*appsearch_header=\nAllow: /list/*autoplay=\nAllow: /list/*backoffice=\nAllow: /list/*branding=\nAllow: /list/*briefEditorPublicationId=\nAllow: /list/*bx_sender_conversion_id=\nAllow: /list/*cachebust=\nAllow: /list/*clid=\nAllow: /list/*collection=\nAllow: /list/*country_code=\nAllow: /list/*erid=\nAllow: /list/*experiment=\nAllow: /list/*feed_exp=\nAllow: /list/*from_cloud=\nAllow: /list/*from_page=\nAllow: /list/*invite=\nAllow: /list/*loop=\nAllow: /list/*pulse_user_id=\nAllow: /list/*sso_failed=\nAllow: /list/*t=\nAllow: /list/*to=\nAllow: /list/*token=\nAllow: /list/*utm_referrer=\nAllow: /list/*utm_source=\nAllow: /list/*?etext=\n\nAllow: /video/watch/*_csrf\nAllow: /video/watch/*action\nAllow: /video/watch/*amp\nAllow: /video/watch/*ampplace\nAllow: /video/watch/*amupr_id\nAllow: /video/watch/*appsearch_header=\nAllow: /video/watch/*autoplay=\nAllow: /video/watch/*backoffice=\nAllow: /video/watch/*branding=\nAllow: /video/watch/*briefEditorPublicationId=\nAllow: /video/watch/*bx_sender_conversion_id=\nAllow: /video/watch/*cachebust=\nAllow: /video/watch/*clid=\nAllow: /video/watch/*collection=\nAllow: /video/watch/*country_code=\nAllow: /video/watch/*erid=\nAllow: /video/watch/*experiment=\nAllow: /video/watch/*feed_exp=\nAllow: /video/watch/*from_cloud=\nAllow: /video/watch/*from_page=\nAllow: /video/watch/*invite=\nAllow: /video/watch/*loop=\nAllow: /video/watch/*pulse_user_id=\nAllow: /video/watch/*sso_failed=\nAllow: /video/watch/*t=\nAllow: /video/watch/*to=\nAllow: /video/watch/*token=\nAllow: /video/watch/*utm_referrer=\nAllow: /video/watch/*utm_source=\nAllow: /video/watch/*?etext=\n\nAllow: /shorts/*_csrf\nAllow: /shorts/*action\nAllow: /shorts/*amp\nAllow: /shorts/*ampplace\nAllow: /shorts/*amupr_id\nAllow: /shorts/*appsearch_header=\nAllow: /shorts/*autoplay=\nAllow: /shorts/*backoffice=\nAllow: /shorts/*branding=\nAllow: /shorts/*briefEditorPublicationId=\nAllow: /shorts/*bx_sender_conversion_id=\nAllow: /shorts/*cachebust=\nAllow: /shorts/*clid=\nAllow: /shorts/*collection=\nAllow: /shorts/*country_code=\nAllow: /shorts/*erid=\nAllow: /shorts/*experiment=\nAllow: /shorts/*feed_exp=\nAllow: /shorts/*from_cloud=\nAllow: /shorts/*from_page=\nAllow: /shorts/*invite=\nAllow: /shorts/*loop=\nAllow: /shorts/*pulse_user_id=\nAllow: /shorts/*sso_failed=\nAllow: /shorts/*t=\nAllow: /shorts/*to=\nAllow: /shorts/*token=\nAllow: /shorts/*utm_referrer=\nAllow: /shorts/*utm_source=\nAllow: /shorts/*?etext=\n\nDisallow: /about*?*\nDisallow: /away\nDisallow: /top$\nDisallow: /search\nDisallow: /money\nDisallow: /onboarding\nDisallow: /news/top\nDisallow: /news/smi\nDisallow: /news/search\nDisallow: /t/\nDisallow: /user/\nDisallow: /embed/\nDisallow: /*/url\nDisallow: /video/games/tags/\nDisallow: /*_csrf\nDisallow: /*action\nDisallow: /*amp\nDisallow: /*ampplace\nDisallow: /*amupr_id\nDisallow: /*appsearch_header=\nDisallow: /*autoplay=\nDisallow: /*backoffice=\nDisallow: /*branding=\nDisallow: /*briefEditorPublicationId=\nDisallow: /*bx_sender_conversion_id=\nDisallow: /*cachebust=\nDisallow: /*clid=\nDisallow: /*collection=\nDisallow: /*country_code=\nDisallow: /*erid=\nDisallow: /*experiment=\nDisallow: /*feed_exp=\nDisallow: /*from_cloud=\nDisallow: /*from_page=\nDisallow: /*invite=\nDisallow: /*loop=\nDisallow: /*pulse_user_id=\nDisallow: /*sso_failed=\nDisallow: /*t=\nDisallow: /*to=\nDisallow: /*token=\nDisallow: /*utm_referrer=\nDisallow: /*utm_source=\nDisallow: /games\nDisallow: /*?etext=\nDisallow: /subscriptions-manager?channel=\n\nDisallow: /articles?\nDisallow: /video?\n\nUser-Agent: *\nAllow: /\nAllow: /.well-known/assetlinks.json\nDisallow: /about*?*\nDisallow: /away\nDisallow: /embed/\nDisallow: /games\nDisallow: /money\nDisallow: /news/search\nDisallow: /news/smi\nDisallow: /news/top\nDisallow: /onboarding\nDisallow: /search\nDisallow: /subscriptions-manager?channel=\nDisallow: /t/\nDisallow: /top$\nDisallow: /user/\nDisallow: /*/url\nDisallow: /video/games/tags/\nDisallow: /*erid=\nDisallow: /*?etext=\nDisallow: /*sso_failed=\nDisallow: /*?sso_failed=\nDisallow: /*to=\n\nSitemap: https://dzen.ru/sitemaps/brief_v2/sitemap.xml\nSitemap: https://dzen.ru/sitemaps/channels_v2/sitemap.xml\nSitemap: https://dzen.ru/sitemaps/gif_long_v2/sitemap.xml\nSitemap: https://dzen.ru/sitemaps/gif_short_v2/sitemap.xml\nSitemap: https://dzen.ru/sitemaps/image_v2/sitemap.xml\nSitemap: https://dzen.ru/sitemaps/native_v2/sitemap.xml\nSitemap: https://dzen.ru/sitemaps/slug-factory/sitemap.xml\nSitemap: https://dzen.ru/sitemaps/suite_v2/sitemap.xml\nSitemap: https://dzen.ru/sitemaps/tv/sitemap.xml\nSitemap: https://dzen.ru/sitemaps/news-archive/sitemap.xml\nSitemap: https://dzen.ru/sitemaps/news-region/sitemap.xml\nSitemap: https://dzen.ru/sitemaps/news-rubric/sitemap.xml\nSitemap: https://dzen.ru/sitemaps/meta-channel_v2/sitemap.xml\n\nSitemap: https://dzen.ru/sitemaps/brief_v2/sitemap.xml.gz\nSitemap: https://dzen.ru/sitemaps/channels_v2/sitemap.xml.gz\nSitemap: https://dzen.ru/sitemaps/gif_long_v2/sitemap.xml.gz\nSitemap: https://dzen.ru/sitemaps/gif_short_v2/sitemap.xml.gz\nSitemap: https://dzen.ru/sitemaps/image_v2/sitemap.xml.gz\nSitemap: https://dzen.ru/sitemaps/native_v2/sitemap.xml.gz\nSitemap: https://dzen.ru/sitemaps/slug-factory/sitemap.xml.gz\nSitemap: https://dzen.ru/sitemaps/suite_v2/sitemap.xml.gz\nSitemap: https://dzen.ru/sitemaps/tv/sitemap.xml.gz\nSitemap: https://dzen.ru/sitemaps/news-archive/sitemap.xml.gz\nSitemap: https://dzen.ru/sitemaps/news-region/sitemap.xml.gz\nSitemap: https://dzen.ru/sitemaps/news-rubric/sitemap.xml.gz\nSitemap: https://dzen.ru/sitemaps/meta-channel_v2/sitemap.xml.gz","agents":{"GPTBot":true,"OAI-SearchBot":true,"ChatGPT-User":true,"ClaudeBot":true,"Claude-SearchBot":true,"PerplexityBot":true,"Google-Extended":true,"Applebot-Extended":true,"CCBot":true,"Bytespider":true,"Meta-ExternalAgent":true,"Amazonbot":true}},"sitemap":{"status":200,"content_type":"application/xml","bytes":32913,"sha256":"ba78e221b8096d3e6d097486d19f15eb47d6e9cfbb9a05d618abd1e5fee42810","final_host":"dzen.ru","ms":66,"url":"https://dzen.ru/sitemaps/brief_v2/sitemap.xml","declared":true,"xml":true},"files":{"llms_txt":{"status":200,"content_type":"text/html","bytes":2894,"sha256":"e9f5b0ddbeb21b1992cc52bf52a2498256ad1bbc0cea54248ac7c09bb1731c47","final_host":"sso.passport.yandex.ru","ms":119,"path":"/llms.txt","html":true,"json":null,"present":false},"agents_md":{"status":200,"content_type":"text/html","bytes":2895,"sha256":"bdaee6b5a2af70d4a83ae5eaf6aaef9afe280b8636e787bcc4737356f8f18613","final_host":"sso.passport.yandex.ru","ms":119,"path":"/agents.md","html":true,"json":null,"present":false},"agent_skills":{"status":200,"content_type":"text/html","bytes":2926,"sha256":"ab248f56af005aa59678edd8aed404139af094721df4c50369e62a2a10ee402d","final_host":"sso.passport.yandex.ru","ms":114,"path":"/.well-known/agent-skills/index.json","html":true,"json":null,"present":false},"mcp_card":{"status":200,"content_type":"text/html","bytes":2923,"sha256":"7a7c66b149840e7e12e3de2ae2e7af8f4a0dc07160d5af54e8b6f0872b797f48","final_host":"sso.passport.yandex.ru","ms":120,"path":"/.well-known/mcp/server-card.json","html":true,"json":null,"present":false},"api_catalog":{"status":200,"content_type":"text/html","bytes":2914,"sha256":"79eebc5a4ebf300e0a3421a3892d27731fce62d8fa13be0be7c434e1d63ba427","final_host":"sso.passport.yandex.ru","ms":182,"path":"/.well-known/api-catalog","html":true,"json":null,"present":false},"a2a_card":{"status":200,"content_type":"text/html","bytes":2913,"sha256":"5c4d853fd095facbf30662277e7e2fc901c272947f4b64e1003f51497d85841e","final_host":"sso.passport.yandex.ru","ms":175,"path":"/.well-known/agent-card.json","html":true,"json":null,"present":false}},"rings":{"v":1,"candidates":[],"mcp":[],"a2a":null,"webmcp":{"model_context":false,"bridge":false,"mcp_json":false,"link_rel":false,"any":false},"handshakes":0,"get_path":{"status":200,"content_type":"text/html","ms":130,"error":null},"get_host":{"status":530,"content_type":"text/plain","ms":201,"error":null}},"derived":{"reachable":true,"robots_served":true,"robots_sitemap":true,"content_signal":false,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":[],"blocks_any_ai":false,"files":{"llms_txt":false,"agents_md":false,"agent_skills":false,"mcp_card":false,"api_catalog":false,"a2a_card":false},"any_capability":false,"ai_refused":[],"ai_thin":[],"rings":{"mcp_declared":false,"mcp_live":false,"mcp_auth":false,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":2,"fh":"sso.passport.yandex.ru","fp_other_host":[]},"derive_v":3,"frame":"top1k","shard":0},{"v":1,"kind":"crawlcheck-census-record","run":14,"domain":"linkedin.com","rank":18,"measured_at":"2026-10-15T05:27:03.729Z","home":{"browser":{"status":200,"content_type":"text/html","bytes":139618,"sha256":"8649f6d9fa922474ab91edb38e08d64c5098100dea9265096067626bf0034cbe","final_host":"www.linkedin.com","ms":468,"identity":"Mobile browser","html":true,"fp":"e0f2000d7ba10acd"},"gptbot":{"status":999,"content_type":"text/html","bytes":2109,"sha256":"388be16d000f70286349134384cbd80f7b8eeac84e265029d6a2ab0ab4128ad9","final_host":"linkedin.com","ms":119,"identity":"GPTBot","html":true},"claudebot":{"status":200,"content_type":"text/html","bytes":141222,"sha256":"6558058a439b744731afd1fb6d3720a6c364b2c652f6f7f6aec428b822311d61","final_host":"www.linkedin.com","ms":541,"identity":"ClaudeBot","html":true,"fp":"e0f2000d7da10acd"},"oaisearch":{"status":200,"content_type":"text/html","bytes":141222,"sha256":"f6b0233525ca847e5292a33dab2375bd7643c7536401ae60976a8ca9b667ed47","final_host":"www.linkedin.com","ms":420,"identity":"OAI-SearchBot","html":true,"fp":"e0f2000d7da10acd"},"perplexity":{"status":200,"content_type":"text/html","bytes":141222,"sha256":"d165e2642802c0c0a8e9e3de5f7d7e85e9ee0b763721730c21fa6908ec2111e1","final_host":"www.linkedin.com","ms":372,"identity":"PerplexityBot","html":true,"fp":"e0f2000d7da10acd"}},"robots":{"status":200,"content_type":"text/html","bytes":21249,"sha256":"59722e62bdead19195d897e91156f495cf05c3fe08a9e2d96b67118a92a98f50","final_host":"linkedin.com","ms":121,"html":true,"text":null,"agents":null},"sitemap":{"status":200,"content_type":"text/html","bytes":21254,"sha256":"30269f99e609d55e816cfe777b450db991fd850ee4f02482a2184f96d566cd01","final_host":"linkedin.com","ms":89,"url":"https://linkedin.com/sitemap.xml","declared":false,"xml":false},"files":{"llms_txt":{"status":200,"content_type":"text/html","bytes":21247,"sha256":"ceaff32bedae92931adc4fb45ea981ff2a56996d7552424cc1b0921a32d2c93c","final_host":"linkedin.com","ms":90,"path":"/llms.txt","html":true,"json":null,"present":false},"agents_md":{"status":200,"content_type":"text/html","bytes":21249,"sha256":"6c17342e347e734894df18ffdadb3cf0c4868965f1b14014a416d0bb76da1be3","final_host":"linkedin.com","ms":88,"path":"/agents.md","html":true,"json":null,"present":false},"agent_skills":{"status":200,"content_type":"text/html","bytes":21319,"sha256":"c4d065ff4fc100ad7e88818ec75672dd1adcb51bb2949f56795c9066aa593518","final_host":"linkedin.com","ms":39,"path":"/.well-known/agent-skills/index.json","html":true,"json":null,"present":false},"mcp_card":{"status":200,"content_type":"text/html","bytes":21311,"sha256":"faff89266f99292e8962d0fa7410f7019b8bfa5c3fbe456a8bd3ab36d9c25766","final_host":"linkedin.com","ms":88,"path":"/.well-known/mcp/server-card.json","html":true,"json":null,"present":false},"api_catalog":{"status":200,"content_type":"text/html","bytes":21287,"sha256":"044147323cc0bc99001a04a396c194da51b2cecb0d582624617cd3ce8122d03e","final_host":"linkedin.com","ms":89,"path":"/.well-known/api-catalog","html":true,"json":null,"present":false},"a2a_card":{"status":200,"content_type":"text/html","bytes":21296,"sha256":"e0a04145fb26c1c0862f1d9560dbf2d8d2002931d27eff18d7d2c47e29fa8e2c","final_host":"linkedin.com","ms":85,"path":"/.well-known/agent-card.json","html":true,"json":null,"present":false}},"rings":{"v":1,"candidates":[],"mcp":[],"a2a":null,"webmcp":{"model_context":false,"bridge":false,"mcp_json":false,"link_rel":false,"any":false},"handshakes":0,"get_path":{"status":200,"content_type":"text/html","ms":39,"error":null},"get_host":{"status":404,"content_type":"text/html","ms":481,"error":null}},"derived":{"reachable":true,"robots_served":false,"robots_sitemap":false,"content_signal":false,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":[],"blocks_any_ai":false,"files":{"llms_txt":false,"agents_md":false,"agent_skills":false,"mcp_card":false,"api_catalog":false,"a2a_card":false},"any_capability":false,"ai_refused":["gptbot"],"ai_thin":[],"rings":{"mcp_declared":false,"mcp_live":false,"mcp_auth":false,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":2,"fh":"www.linkedin.com","fp_other_host":[],"fp":"e0f2000d7ba10acd","fp_bits":{"claudebot":2,"oaisearch":2,"perplexity":2}},"derive_v":3,"frame":"top1k","shard":1},{"v":1,"kind":"crawlcheck-census-record","run":14,"domain":"googlevideo.com","rank":19,"measured_at":"2026-10-15T05:28:03.923Z","home":{"browser":{"status":526,"content_type":"text/plain","bytes":16,"sha256":"4c543fe8a4d739c6f622b04303ac9e9d9fafbf00da950fc916ad826da2c0cdac","final_host":"googlevideo.com","ms":130,"identity":"Mobile browser","html":false},"gptbot":{"status":526,"content_type":"text/plain","bytes":16,"sha256":"4c543fe8a4d739c6f622b04303ac9e9d9fafbf00da950fc916ad826da2c0cdac","final_host":"googlevideo.com","ms":52,"identity":"GPTBot","html":false},"claudebot":{"status":526,"content_type":"text/plain","bytes":16,"sha256":"4c543fe8a4d739c6f622b04303ac9e9d9fafbf00da950fc916ad826da2c0cdac","final_host":"googlevideo.com","ms":47,"identity":"ClaudeBot","html":false},"oaisearch":{"status":526,"content_type":"text/plain","bytes":16,"sha256":"4c543fe8a4d739c6f622b04303ac9e9d9fafbf00da950fc916ad826da2c0cdac","final_host":"googlevideo.com","ms":178,"identity":"OAI-SearchBot","html":false},"perplexity":{"status":526,"content_type":"text/plain","bytes":16,"sha256":"4c543fe8a4d739c6f622b04303ac9e9d9fafbf00da950fc916ad826da2c0cdac","final_host":"googlevideo.com","ms":47,"identity":"PerplexityBot","html":false}},"robots":{"status":526,"content_type":"text/plain","bytes":16,"sha256":"4c543fe8a4d739c6f622b04303ac9e9d9fafbf00da950fc916ad826da2c0cdac","final_host":"googlevideo.com","ms":99,"html":false,"text":null,"agents":null},"sitemap":null,"files":null,"derived":{"reachable":false,"robots_served":false,"robots_sitemap":false,"content_signal":false,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":[],"blocks_any_ai":false,"files":{"llms_txt":false,"agents_md":false,"agent_skills":false,"mcp_card":false,"api_catalog":false,"a2a_card":false},"any_capability":false,"ai_refused":[],"ai_thin":[],"rings":{"mcp_declared":false,"mcp_live":false,"mcp_auth":false,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":2,"fh":"googlevideo.com","fp_other_host":[]},"derive_v":3,"frame":"top1k","shard":2},{"v":1,"kind":"crawlcheck-census-record","run":14,"domain":"akamaiedge.net","rank":20,"measured_at":"2026-10-15T05:29:03.596Z","home":{"browser":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"akamaiedge.net","ms":7,"identity":"Mobile browser","html":false},"gptbot":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"akamaiedge.net","ms":1,"identity":"GPTBot","html":false},"claudebot":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"akamaiedge.net","ms":2,"identity":"ClaudeBot","html":false},"oaisearch":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"akamaiedge.net","ms":1,"identity":"OAI-SearchBot","html":false},"perplexity":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"akamaiedge.net","ms":1,"identity":"PerplexityBot","html":false}},"robots":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"akamaiedge.net","ms":1,"html":false,"text":null,"agents":null},"sitemap":null,"files":null,"derived":{"reachable":false,"robots_served":false,"robots_sitemap":false,"content_signal":false,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":[],"blocks_any_ai":false,"files":{"llms_txt":false,"agents_md":false,"agent_skills":false,"mcp_card":false,"api_catalog":false,"a2a_card":false},"any_capability":false,"ai_refused":[],"ai_thin":[],"rings":{"mcp_declared":false,"mcp_live":false,"mcp_auth":false,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":2,"fh":"akamaiedge.net","fp_other_host":[]},"derive_v":3,"frame":"top1k","shard":3},{"v":1,"kind":"crawlcheck-census-record","run":14,"domain":"googletagmanager.com","rank":21,"measured_at":"2026-10-15T05:30:03.842Z","home":{"browser":{"status":404,"content_type":"text/html","bytes":1561,"sha256":"23926e9185d8d43c02807a838ffb373cc1977726094a4e46807c66ada9dd7660","final_host":"googletagmanager.com","ms":139,"identity":"Mobile browser","html":true},"gptbot":{"status":404,"content_type":"text/html","bytes":1561,"sha256":"23926e9185d8d43c02807a838ffb373cc1977726094a4e46807c66ada9dd7660","final_host":"googletagmanager.com","ms":51,"identity":"GPTBot","html":true},"claudebot":{"status":404,"content_type":"text/html","bytes":1561,"sha256":"23926e9185d8d43c02807a838ffb373cc1977726094a4e46807c66ada9dd7660","final_host":"googletagmanager.com","ms":27,"identity":"ClaudeBot","html":true},"oaisearch":{"status":404,"content_type":"text/html","bytes":1561,"sha256":"23926e9185d8d43c02807a838ffb373cc1977726094a4e46807c66ada9dd7660","final_host":"googletagmanager.com","ms":27,"identity":"OAI-SearchBot","html":true},"perplexity":{"status":404,"content_type":"text/html","bytes":1561,"sha256":"23926e9185d8d43c02807a838ffb373cc1977726094a4e46807c66ada9dd7660","final_host":"googletagmanager.com","ms":28,"identity":"PerplexityBot","html":true}},"robots":{"status":404,"content_type":"text/html","bytes":1571,"sha256":"6adee83a3eba87d30fdca3997d27097697f2767805f222a0f78c3f04392004d2","final_host":"googletagmanager.com","ms":108,"html":true,"text":null,"agents":null},"sitemap":null,"files":null,"derived":{"reachable":false,"robots_served":false,"robots_sitemap":false,"content_signal":false,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":[],"blocks_any_ai":false,"files":{"llms_txt":false,"agents_md":false,"agent_skills":false,"mcp_card":false,"api_catalog":false,"a2a_card":false},"any_capability":false,"ai_refused":[],"ai_thin":[],"rings":{"mcp_declared":false,"mcp_live":false,"mcp_auth":false,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":2,"fh":"googletagmanager.com","fp_other_host":[]},"derive_v":3,"frame":"top1k","shard":4},{"v":1,"kind":"crawlcheck-census-record","run":14,"domain":"office.com","rank":22,"measured_at":"2026-10-15T05:31:08.958Z","home":{"browser":{"status":200,"content_type":"text/html","bytes":255333,"sha256":"e6bf81264465555325b19e52db0cc2ae3686cf96beae161dd2e1dd66b47f9b26","final_host":"m365.cloud.microsoft","ms":551,"identity":"Mobile browser","html":true,"fp":"df6beec904d1e959"},"gptbot":{"status":200,"content_type":"text/html","bytes":347320,"sha256":"a1c1fdeeddd6eb8d65adb592d52d7fd58527a7cabb1e61f559b145911d1ead72","final_host":"m365.cloud.microsoft","ms":502,"identity":"GPTBot","html":true,"fp":"9bdfec8980d0e979"},"claudebot":{"status":200,"content_type":"text/html","bytes":347320,"sha256":"a1c1fdeeddd6eb8d65adb592d52d7fd58527a7cabb1e61f559b145911d1ead72","final_host":"m365.cloud.microsoft","ms":523,"identity":"ClaudeBot","html":true,"fp":"9bdfec8980d0e979"},"oaisearch":{"status":200,"content_type":"text/html","bytes":347320,"sha256":"a1c1fdeeddd6eb8d65adb592d52d7fd58527a7cabb1e61f559b145911d1ead72","final_host":"m365.cloud.microsoft","ms":500,"identity":"OAI-SearchBot","html":true,"fp":"9bdfec8980d0e979"},"perplexity":{"status":200,"content_type":"text/html","bytes":347320,"sha256":"a1c1fdeeddd6eb8d65adb592d52d7fd58527a7cabb1e61f559b145911d1ead72","final_host":"m365.cloud.microsoft","ms":523,"identity":"PerplexityBot","html":true,"fp":"9bdfec8980d0e979"}},"robots":{"status":200,"content_type":"text/html","bytes":60396,"sha256":"069ae797a1f984ea77e5fd87a41e452d867cd9a8418c8aa740023d8cb9e7886d","final_host":"login.microsoftonline.com","ms":636,"html":true,"text":null,"agents":null},"sitemap":{"status":200,"content_type":"text/html","bytes":60423,"sha256":"9f024e60e397f97f347029ea8c6a94e5434ee476f15144c30b2559f137657338","final_host":"login.microsoftonline.com","ms":154,"url":"https://office.com/sitemap.xml","declared":false,"xml":false},"files":{"llms_txt":{"status":200,"content_type":"text/html","bytes":60457,"sha256":"dd1fb154387e0be222a4122f68e20de1a5423b0fd602b63b2d67b827d1f16ba4","final_host":"login.microsoftonline.com","ms":281,"path":"/llms.txt","html":true,"json":null,"present":false},"agents_md":{"status":200,"content_type":"text/html","bytes":60393,"sha256":"356bf14d383cdf4ef0d50d275348d0d6fbae9b956285cbe6ecde0ed3228b88fe","final_host":"login.microsoftonline.com","ms":326,"path":"/agents.md","html":true,"json":null,"present":false},"agent_skills":{"status":200,"content_type":"text/html","bytes":60644,"sha256":"836861a68ac740731b465fd8d5ad1d4f9417c1660b77eb86fc6a05b5c134f9ef","final_host":"login.microsoftonline.com","ms":308,"path":"/.well-known/agent-skills/index.json","html":true,"json":null,"present":false},"mcp_card":{"status":200,"content_type":"text/html","bytes":60717,"sha256":"eb6b5f9a7a0e947be8e768b601933f224cc9c71af115648259d25240d1cacf33","final_host":"login.microsoftonline.com","ms":310,"path":"/.well-known/mcp/server-card.json","html":true,"json":null,"present":false},"api_catalog":{"status":200,"content_type":"text/html","bytes":60821,"sha256":"15cf65bb8e114ce580247e67314e2325bce1a3e16e6843195aeb216ea2e2c867","final_host":"login.microsoftonline.com","ms":310,"path":"/.well-known/api-catalog","html":true,"json":null,"present":false},"a2a_card":{"status":200,"content_type":"text/html","bytes":60739,"sha256":"24f85ff24acf2ee22afd8a6899d716f909acaaf10e7c6d55d1a987adb099f7cb","final_host":"login.microsoftonline.com","ms":153,"path":"/.well-known/agent-card.json","html":true,"json":null,"present":false}},"rings":{"v":1,"candidates":[],"mcp":[],"a2a":null,"webmcp":{"model_context":false,"bridge":false,"mcp_json":false,"link_rel":false,"any":false},"handshakes":0,"get_path":{"status":200,"content_type":"text/html","ms":173,"error":null},"get_host":{"status":530,"content_type":"text/plain","ms":20,"error":null}},"derived":{"reachable":true,"robots_served":false,"robots_sitemap":false,"content_signal":false,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":[],"blocks_any_ai":false,"files":{"llms_txt":false,"agents_md":false,"agent_skills":false,"mcp_card":false,"api_catalog":false,"a2a_card":false},"any_capability":false,"ai_refused":[],"ai_thin":[],"rings":{"mcp_declared":false,"mcp_live":false,"mcp_auth":false,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":2,"fh":"m365.cloud.microsoft","fp_other_host":[],"fp":"df6beec904d1e959","fp_bits":{"gptbot":12,"claudebot":12,"oaisearch":12,"perplexity":12}},"derive_v":3,"frame":"top1k","shard":5},{"v":1,"kind":"crawlcheck-census-record","run":14,"domain":"live.com","rank":23,"measured_at":"2026-10-15T05:33:03.639Z","home":{"browser":{"status":200,"content_type":"text/html","bytes":13819,"sha256":"308038837a08c0457aa9f086a7b67b1fe12c7670d55bc493694699d73e2e99f1","final_host":"outlook.live.com","ms":176,"identity":"Mobile browser","html":true},"gptbot":{"status":200,"content_type":"text/html","bytes":14277,"sha256":"4d7e5c591478d0d167735f10dfa60174e71c7d71f7d000312943c6385e3d65c7","final_host":"outlook.live.com","ms":168,"identity":"GPTBot","html":true},"claudebot":{"status":200,"content_type":"text/html","bytes":14277,"sha256":"4f622890dffaf014dd87dbcdd8a5ff9e55b8f3ffdcc928547b5b91b24b45035b","final_host":"outlook.live.com","ms":37,"identity":"ClaudeBot","html":true},"oaisearch":{"status":200,"content_type":"text/html","bytes":14277,"sha256":"f3bbd5cfb6fce5534df22edb88984cf6895b3487032eedbed5617b02e26a2544","final_host":"outlook.live.com","ms":32,"identity":"OAI-SearchBot","html":true},"perplexity":{"status":200,"content_type":"text/html","bytes":14277,"sha256":"6ada800cfebbf5d38aef06522fdfdd4db310f75c6bee72d254a56e65c9b1b59b","final_host":"outlook.live.com","ms":33,"identity":"PerplexityBot","html":true}},"robots":{"status":200,"content_type":"text/html","bytes":14277,"sha256":"24630e837e9358b8599ff5ca82ddda0d2aea8246390c7b21526970679dc84f7a","final_host":"outlook.live.com","ms":88,"html":true,"text":null,"agents":null},"sitemap":{"status":200,"content_type":"text/html","bytes":14277,"sha256":"04ca3c3292b52ddc30601e344dd5c1f996e2a74bf57aec4852c4089a7e0fb548","final_host":"outlook.live.com","ms":33,"url":"https://live.com/sitemap.xml","declared":false,"xml":false},"files":{"llms_txt":{"status":200,"content_type":"text/html","bytes":14277,"sha256":"d703cc3d1208ae5aaf1b92b470882a2966d7e629b9b92955bd196ea3339cd58c","final_host":"outlook.live.com","ms":29,"path":"/llms.txt","html":true,"json":null,"present":false},"agents_md":{"status":200,"content_type":"text/html","bytes":14277,"sha256":"5e4f9eff6adfdc90508509362e38b78256803c37eb77634a19715ba731c2fede","final_host":"outlook.live.com","ms":27,"path":"/agents.md","html":true,"json":null,"present":false},"agent_skills":{"status":200,"content_type":"text/html","bytes":14277,"sha256":"dd512fda41424d8828129e07fa66fa3927db93968a1deb0840dd78b1ff82a40e","final_host":"outlook.live.com","ms":35,"path":"/.well-known/agent-skills/index.json","html":true,"json":null,"present":false},"mcp_card":{"status":200,"content_type":"text/html","bytes":14277,"sha256":"c1d646088b6a588ecefafa29c8f8bd32cec852ccbb7329b1fbda06d95edfc5ba","final_host":"outlook.live.com","ms":31,"path":"/.well-known/mcp/server-card.json","html":true,"json":null,"present":false},"api_catalog":{"status":200,"content_type":"text/html","bytes":14277,"sha256":"79330a88bde8f31172adea370e5c697d6282d33bd2ccef592ccdcd7b59bade6f","final_host":"outlook.live.com","ms":46,"path":"/.well-known/api-catalog","html":true,"json":null,"present":false},"a2a_card":{"status":200,"content_type":"text/html","bytes":14277,"sha256":"3a0e045b246a21605c0fa149eb9ca17275a3c084fbef68347f3de8a400d3d0b6","final_host":"outlook.live.com","ms":31,"path":"/.well-known/agent-card.json","html":true,"json":null,"present":false}},"rings":{"v":1,"candidates":[],"mcp":[],"a2a":null,"webmcp":{"model_context":false,"bridge":false,"mcp_json":false,"link_rel":false,"any":false},"handshakes":0,"get_path":{"status":200,"content_type":"text/html","ms":30,"error":null},"get_host":{"status":200,"content_type":"text/html","ms":192,"error":null}},"derived":{"reachable":true,"robots_served":false,"robots_sitemap":false,"content_signal":false,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":[],"blocks_any_ai":false,"files":{"llms_txt":false,"agents_md":false,"agent_skills":false,"mcp_card":false,"api_catalog":false,"a2a_card":false},"any_capability":false,"ai_refused":[],"ai_thin":[],"rings":{"mcp_declared":false,"mcp_live":false,"mcp_auth":false,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":2,"fh":"outlook.live.com","fp_other_host":[]},"derive_v":3,"frame":"top1k","shard":6},{"v":1,"kind":"crawlcheck-census-record","run":14,"domain":"akadns.net","rank":24,"measured_at":"2026-10-15T05:34:03.814Z","home":{"browser":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"akadns.net","ms":8,"identity":"Mobile browser","html":false},"gptbot":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"akadns.net","ms":2,"identity":"GPTBot","html":false},"claudebot":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"akadns.net","ms":2,"identity":"ClaudeBot","html":false},"oaisearch":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"akadns.net","ms":2,"identity":"OAI-SearchBot","html":false},"perplexity":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"akadns.net","ms":1,"identity":"PerplexityBot","html":false}},"robots":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"akadns.net","ms":2,"html":false,"text":null,"agents":null},"sitemap":null,"files":null,"derived":{"reachable":false,"robots_served":false,"robots_sitemap":false,"content_signal":false,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":[],"blocks_any_ai":false,"files":{"llms_txt":false,"agents_md":false,"agent_skills":false,"mcp_card":false,"api_catalog":false,"a2a_card":false},"any_capability":false,"ai_refused":[],"ai_thin":[],"rings":{"mcp_declared":false,"mcp_live":false,"mcp_auth":false,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":2,"fh":"akadns.net","fp_other_host":[]},"derive_v":3,"frame":"top1k","shard":7},{"v":1,"kind":"crawlcheck-census-record","run":14,"domain":"hicloudcam.com","rank":25,"measured_at":"2026-10-15T05:26:03.595Z","home":{"browser":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"hicloudcam.com","ms":532,"identity":"Mobile browser","html":false},"gptbot":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"hicloudcam.com","ms":419,"identity":"GPTBot","html":false},"claudebot":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"hicloudcam.com","ms":413,"identity":"ClaudeBot","html":false},"oaisearch":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"hicloudcam.com","ms":413,"identity":"OAI-SearchBot","html":false},"perplexity":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"hicloudcam.com","ms":406,"identity":"PerplexityBot","html":false}},"robots":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"hicloudcam.com","ms":2,"html":false,"text":null,"agents":null},"sitemap":null,"files":null,"derived":{"reachable":false,"robots_served":false,"robots_sitemap":false,"content_signal":false,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":[],"blocks_any_ai":false,"files":{"llms_txt":false,"agents_md":false,"agent_skills":false,"mcp_card":false,"api_catalog":false,"a2a_card":false},"any_capability":false,"ai_refused":[],"ai_thin":[],"rings":{"mcp_declared":false,"mcp_live":false,"mcp_auth":false,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":2,"fh":"hicloudcam.com","fp_other_host":[]},"derive_v":3,"frame":"top1k","shard":0},{"v":1,"kind":"crawlcheck-census-record","run":14,"domain":"amazon.com","rank":26,"measured_at":"2026-10-15T05:27:03.729Z","home":{"browser":{"status":202,"content_type":"text/html","bytes":2007,"sha256":"5d2706148b261393db09dcf5fbe426b66e44fa8a629eae4afcef484edfa7276d","final_host":"www.amazon.com","ms":354,"identity":"Mobile browser","html":true},"gptbot":{"status":202,"content_type":"text/html","bytes":2007,"sha256":"7636f8e8c0c7b3d4b4df83b62b8b62b63313090de964115c32ab7b8327885b01","final_host":"www.amazon.com","ms":370,"identity":"GPTBot","html":true},"claudebot":{"status":202,"content_type":"text/html","bytes":2007,"sha256":"fcb4fd1161b4044f1e383ef7d7ec8c5fa3550fa7318aaca2a7484199e451ffb1","final_host":"www.amazon.com","ms":337,"identity":"ClaudeBot","html":true},"oaisearch":{"status":202,"content_type":"text/html","bytes":2007,"sha256":"819a97ab7a0925809b4170552b850fbde387a329e93baf08e9f491e926bbdaaf","final_host":"www.amazon.com","ms":340,"identity":"OAI-SearchBot","html":true},"perplexity":{"status":202,"content_type":"text/html","bytes":2007,"sha256":"30a0fbc768e27c6eb2b61a6e690b6fe41c03a92c546271e543643f868e2610f6","final_host":"www.amazon.com","ms":344,"identity":"PerplexityBot","html":true}},"robots":{"status":200,"content_type":"text/plain","bytes":7947,"sha256":"74fa27e28cd9d80bdd155ffecf3a033ea2e195a80c83d6fa46acb20665c3cb4d","final_host":"www.amazon.com","ms":684,"html":false,"text":"User-agent: *\nDisallow: /exec/obidos/account-access-login\nDisallow: /exec/obidos/change-style\nDisallow: /exec/obidos/flex-sign-in\nDisallow: /exec/obidos/handle-buy-box\nDisallow: /exec/obidos/tg/cm/member/\nDisallow: /gp/aw/help/id=sss\nDisallow: /gp/cart\nDisallow: /gp/flex\nDisallow: /gp/product/e-mail-friend\nDisallow: /gp/product/product-availability\nDisallow: /gp/product/rate-this-item\nDisallow: /gp/sign-in\nDisallow: /gp/reader\nDisallow: /gp/sitbv3/reader\nDisallow: /gp/richpub/syltguides/create\nDisallow: /gp/gfix\nDisallow: /gp/associations/wizard.html\nDisallow: /gp/dmusic/order\nDisallow: /gp/legacy-handle-buy-box.html\nDisallow: /gp/aws/ssop\nDisallow: /gp/yourstore\nDisallow: /gp/gift-central/organizer/add-wishlist\nDisallow: /gp/vote\nDisallow: /gp/voting/\nDisallow: /gp/music/wma-pop-up\nDisallow: /gp/customer-images\nDisallow: /gp/richpub/listmania/createpipeline\nDisallow: /gp/content-form\nDisallow: /gp/pdp/invitation/invite\nDisallow: /gp/customer-reviews/common/du\nDisallow: /gp/customer-reviews/write-a-review.html\nDisallow: /gp/associations/wizard.html\nDisallow: /gp/music/clipserve\nDisallow: /gp/customer-media/upload\nDisallow: /gp/history\nDisallow: /gp/item-dispatch\nDisallow: /gp/dmusic/order/handle-buy-box.html\nDisallow: /gp/recsradio\nDisallow: /gp/slredirect\nDisallow: /dp/shipping/\nDisallow: /dp/twister-update/\nDisallow: /dp/manual-submit/\nDisallow: /dp/e-mail-friend/\nDisallow: /dp/product-availability/\nDisallow: /dp/rate-this-item/\nDisallow: /gp/registry/wishlist/*/reserve\nDisallow: /gp/structured-ratings/actions/get-experience.html\nDisallow: /gp/twitter/\nDisallow: /ap/signin\nDisallow: /gp/registry/wishlist/\nDisallow: /wishlist/\nAllow: /wishlist/universal*\nAllow: /wishlist/vendor-button*\nAllow: /wishlist/get-button*\nDisallow: /gp/wishlist/\nAllow: /gp/wishlist/universal*\nAllow: /gp/wishlist/vendor-button*\nAllow: /gp/wishlist/ipad-install*\nDisallow: /registry/wishlist/\nDisallow: /review/common/du\nDisallow: /gp/registry/search.html\nDisallow: /product-reviews/B0069IY63Y\nDisallow: /gp/orc/rml/\nDisallow: */gcrnsts\nDisallow: /gp/gc/widget\nDisallow: /gp/dmusic/mp3/player\nDisallow: /gp/entity-alert/external\nDisallow: /gp/customer-reviews/dynamic/sims-box\nDisallow: /review/dynamic/sims-box\nDisallow: /gp/redirect.html\nDisallow: /gp/twister/ajaxv2\nDisallow: /ss/twister/ajax\nDisallow: /b?*node=7454917011\nDisallow: /b?*node=7454927011\nDisallow: /b?*node=7454939011\nDisallow: /b?*node=7454898011\nDisallow: /gp/customer-media/actions/delete/\nDisallow: /gp/customer-media/actions/edit-caption/\nDisallow: /gp/dmusic/\nAllow: /gp/dmusic/promotions/PrimeMusic\nAllow: /gp/dmusic/promotions/AmazonMusicUnlimited\nDisallow: /gp/offer-listing/\nDisallow: /b?*node=9052533011\nDisallow: /lm/R1XIHQVKXSKBNJ\nDisallow: /lm/R3HQ5WJSZK6QSO\nDisallow: /surprise/\nDisallow: /local/ajax/\nDisallow: /gp/socialmedia/giveaways\nDisallow: /gp/b2b-rd\nDisallow: /gp/aw/so.html\nDisallow: /reviews/iframe\nDisallow: /gp/switch-language\nDisallow: /ga/p/\nDisallow: /gp/profile/\nDisallow: /giveaway/host/setup/\nDisallow: /ss/customer-reviews/lighthouse/\nDisallow: /ospublishing/story/*\nDisallow: /gp/aw/ol/\nDisallow: /gp/promotion/\nDisallow: /hz/leaderboard/top-reviewers/\nDisallow: /creatorhub\nDisallow: /creatorhub/*\nDisallow: /slp/s$\nDisallow: /-/\nAllow: /-/es/\nAllow: /-/en$\nAllow: /-/zh_TW/\nAllow: /-/zh_TW$\nAllow: /-/he/\nAllow: /-/he$\nDisallow: /hz/help/contact/*/message/$\nDisallow: /gp/aw/shoppingAids/\nDisallow: /rss/people/*/reviews\nDisallow: /gp/pdp/rss/*/reviews\nDisallow: /gp/cdp/member-reviews/\nDisallow: /gp/aw/cr/\nAllow: /gp/offer-listing/B000\nAllow: /gp/offer-listing/9000\nDisallow: /gp/aag\nAllow: /gp/aag/main?*seller=ABVFEJU8LS620\nDisallow: /gp/pdp/profile/\nDisallow: /gp/help/customer/express/c2c/\nDisallow: /slp/*/b$\nDisallow: /hz/contact-us/ajax/initiate-trusted-contact/\nDisallow: /hp/video/\nDisallow: /gp/video/api\nDisallow: /gp/video/settings\nDisallow: /gp/video/library\nDisallow: /gp/video/watchlist\nDisallow: /gp/video/mystuff\nDisallow: /gp/video/profiles\nDisallow: /gp/video/search\nDisallow: /gp/video/auth\nDisallow: /gp/video/salp/i\nDisallow: /icdp/\nDisallow: /shoptheshow/*?cat=\nDisallow: /shoptheshow/*&cat=\n\nUser-agent: EtaoSpider\nDisallow: /\n\nUser-agent: GPTBot\nDisallow: /\n\n\nUser-agent: CCBot\nDisallow: /\n\nUser-Agent: PerplexityBot\nDisallow: /\n\nUser-agent: Google-Extended\nDisallow: /\n\nUser-agent: GoogleAgent-Mariner\nDisallow: /\n\nUser-agent: GoogleAgent-Shopping\nDisallow: /\n\nUser-agent: ClaudeBot\nDisallow: /\n\nUser-agent: meta-externalagent\nDisallow: /\n\nUser-agent: Bytespider\nDisallow: /\n\nUser-agent: Scrapy\nDisallow: /\n\nUser-agent: PetalBot\nDisallow: /\n\nUser-agent: Devin\nDisallow: /\n\nUser-agent: omgili\nDisallow: /\n\nUser-agent: AI2Bot\nDisallow: /\n\nUser-agent: Gemini-Deep-Research\nDisallow: /\n\nUser-agent: PanguBot\nDisallow: /\n\nUser-agent: MistralAI-User\nDisallow: /\n\nUser-agent: Diffbot\nDisallow: /\n\nUser-agent: Sidetrade indexer bot\nDisallow: /\n\nUser-agent: Google-NotebookLM\nDisallow: /\n\nUser-agent: DuckAssistBot\nDisallow: /\n\nUser-agent: Copilot\nDisallow: /\n\nUser-agent: Ai2Bot-Dolma\nDisallow: /\n\nUser-agent: Google-CloudVertexBot\nDisallow: /\n\nUser-agent: meta-externalfetcher\nDisallow: /\n\nUser-agent: iaskspider\nDisallow: /\n\nUser-agent: wpbot\nDisallow: /\n\nUser-agent: aiHitBot\nDisallow: /\n\nUser-agent: cohere-ai\nDisallow: /\n\nUser-agent: FriendlyCrawler\nDisallow: /\n\nUser-agent: img2dataset\nDisallow: /\n\nUser-agent: VelenPublicWebCrawler\nDisallow: /\n\nUser-agent: YouBot\nDisallow: /\n\nUser-agent: Brightbot\nDisallow: /\n\nUser-agent: Claude-User\nDisallow: /\n\nUser-agent: ClaudeBot\nDisallow: /\n\nUser-agent: Claude-SearchBot\nDisallow: /\n\nUser-agent: Perplexity-User\nDisallow: /\n\nUser-agent: ISSCyberRiskCrawler\nDisallow: /\n\nUser-agent: CopilotNative\nDisallow: /\n\nUser-agent: CopilotSapphire\nDisallow: /\n\nUser-agent: imgproxy\nDisallow: /\n\nUser-agent: SemrushBot-SWA\nDisallow: /\n\nUser-agent: ChatGPT-User\nDisallow: /\n\nUser-agent: OAI-SearchBot\nDisallow: /\n\nUser-agent: Timpibot\nDisallow: /\n\nUser-agent: Thinkbot\nDisallow: /\n\nUser-agent: AddSearchBot\nDisallow: /\n\nUser-agent: AI2Bot-DeepResearchEval\nDisallow: /\n\nUser-agent: Andibot\nDisallow: /\n\nUser-agent: Anomura\nDisallow: /\n\nUser-agent: atlassian-bot\nDisallow: /\n\nUser-agent: BuddyBot\nDisallow: /\n\nUser-agent: Channel3Bot\nDisallow: /\n\nUser-agent: ChatGLM-Spider\nDisallow: /\n\nUser-agent: Claude-Web\nDisallow: /\n\nUser-agent: Cloudflare-AutoRAG\nDisallow: /\n\nUser-agent: CloudflareBrowserRenderingCrawler\nDisallow: /\n\nUser-agent: cohere-training-data-crawler\nDisallow: /\n\nUser-agent: Cotoyogi\nDisallow: /\n\nUser-agent: Crawl4AI\nDisallow: /\n\nUser-agent: Datenbank Crawler\nDisallow: /\n\nUser-agent: DeepSeekBot\nDisallow: /\n\nUser-agent: Echobot Bot\nDisallow: /\n\nUser-agent: EchoboxBot\nDisallow: /\n\nUser-agent: factset_spyderbot\nDisallow: /\n\nUser-agent: Google-Firebase\nDisallow: /\n\nUser-agent: GoogleOther\nDisallow: /\n\nUser-agent: Grok-DeepSearch\nDisallow: /\n\nUser-agent: GrokBot\nDisallow: /\n\nUser-agent: iAskBot\nDisallow: /\n\nUser-agent: IbouBot\nDisallow: /\n\nUser-agent: ICC-Crawler\nDisallow: /\n\nUser-agent: ImagesiftBot\nDisallow: /\n\nUser-agent: imageSpider\nDisallow: /\n\nUser-agent: Kangaroo Bot\nDisallow: /\n\nUser-agent: KlaviyoAIBot\nDisallow: /\n\nUser-agent: KunatoCrawler\nDisallow: /\n\nUser-agent: laion-huggingface-processor\nDisallow: /\n\nUser-agent: Linguee Bot\nDisallow: /\n\nUser-agent: LinkupBot\nDisallow: /\n\nUser-agent: Manus-User\nDisallow: /\n\nUser-agent: meta-webindexer\nDisallow: /\n\nUser-agent: netEstate Imprint Crawler\nDisallow: /\n\nUser-agent: Panscient\nDisallow: /\n\nUser-agent: panscient.com\nDisallow: /\n\nUser-agent: PhindBot\nDisallow: /\n\nUser-agent: Poggio-Citations\nDisallow: /\n\nUser-agent: QualifiedBot\nDisallow: /\n\nUser-agent: SBIntuitionsBot\nDisallow: /\n\nUser-agent: ShapBot\nDisallow: /\n\nUser-agent: TavilyBot\nDisallow: /\n\nUser-agent: TerraCotta\nDisallow: /\n\nUser-agent: TwinAgent\nDisallow: /\n\nUser-agent: webzio-extended\nDisallow: /\n\nUser-agent: WRTNBot\nDisallow: /\n\nUser-agent: xAI-Grok\nDisallow: /\n\nUser-agent: YaK\nDisallow: /\n\nUser-agent: ZanistaBot\nDisallow: /","agents":{"GPTBot":false,"OAI-SearchBot":false,"ChatGPT-User":false,"ClaudeBot":false,"Claude-SearchBot":false,"PerplexityBot":false,"Google-Extended":false,"Applebot-Extended":true,"CCBot":false,"Bytespider":false,"Meta-ExternalAgent":false,"Amazonbot":true}},"sitemap":{"status":500,"content_type":"text/html","bytes":2672,"sha256":"c669d528a3876410eebef4c55e504ec016707d321f090eae9f3b09b8eaedc67f","final_host":"www.amazon.com","ms":1569,"url":"https://amazon.com/sitemap.xml","declared":false,"xml":false},"files":{"llms_txt":{"status":404,"content_type":"text/html","bytes":2296,"sha256":"34c92fb4c6351b94e5cc16f68bb483b0fcda57ee7c43887214005b87d1888d3a","final_host":"www.amazon.com","ms":525,"path":"/llms.txt","html":true,"json":null,"present":false},"agents_md":{"status":404,"content_type":"text/html","bytes":2296,"sha256":"34c92fb4c6351b94e5cc16f68bb483b0fcda57ee7c43887214005b87d1888d3a","final_host":"www.amazon.com","ms":279,"path":"/agents.md","html":true,"json":null,"present":false},"agent_skills":{"status":404,"content_type":"text/html","bytes":2296,"sha256":"34c92fb4c6351b94e5cc16f68bb483b0fcda57ee7c43887214005b87d1888d3a","final_host":"www.amazon.com","ms":320,"path":"/.well-known/agent-skills/index.json","html":true,"json":null,"present":false},"mcp_card":{"status":404,"content_type":"text/html","bytes":2296,"sha256":"34c92fb4c6351b94e5cc16f68bb483b0fcda57ee7c43887214005b87d1888d3a","final_host":"www.amazon.com","ms":316,"path":"/.well-known/mcp/server-card.json","html":true,"json":null,"present":false},"api_catalog":{"status":404,"content_type":"text/html","bytes":2296,"sha256":"34c92fb4c6351b94e5cc16f68bb483b0fcda57ee7c43887214005b87d1888d3a","final_host":"www.amazon.com","ms":550,"path":"/.well-known/api-catalog","html":true,"json":null,"present":false},"a2a_card":{"status":404,"content_type":"text/html","bytes":2296,"sha256":"34c92fb4c6351b94e5cc16f68bb483b0fcda57ee7c43887214005b87d1888d3a","final_host":"www.amazon.com","ms":355,"path":"/.well-known/agent-card.json","html":true,"json":null,"present":false}},"rings":{"v":1,"candidates":[],"mcp":[],"a2a":null,"webmcp":{"model_context":false,"bridge":false,"mcp_json":false,"link_rel":false,"any":false},"handshakes":0,"get_path":{"status":200,"content_type":null,"ms":295,"error":null},"get_host":{"status":530,"content_type":"text/plain","ms":5,"error":null}},"derived":{"reachable":false,"robots_served":false,"robots_sitemap":false,"content_signal":false,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":[],"blocks_any_ai":false,"files":{"llms_txt":false,"agents_md":false,"agent_skills":false,"mcp_card":false,"api_catalog":false,"a2a_card":false},"any_capability":false,"ai_refused":[],"ai_thin":[],"rings":{"mcp_declared":false,"mcp_live":false,"mcp_auth":false,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":2,"fh":"www.amazon.com","fp_other_host":[]},"derive_v":3,"frame":"top1k","shard":1},{"v":1,"kind":"crawlcheck-census-record","run":14,"domain":"gtld-servers.net","rank":27,"measured_at":"2026-10-15T05:28:03.923Z","home":{"browser":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"gtld-servers.net","ms":146,"identity":"Mobile browser","html":false},"gptbot":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"gtld-servers.net","ms":124,"identity":"GPTBot","html":false},"claudebot":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"gtld-servers.net","ms":110,"identity":"ClaudeBot","html":false},"oaisearch":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"gtld-servers.net","ms":6,"identity":"OAI-SearchBot","html":false},"perplexity":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"gtld-servers.net","ms":2,"identity":"PerplexityBot","html":false}},"robots":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"gtld-servers.net","ms":3,"html":false,"text":null,"agents":null},"sitemap":null,"files":null,"derived":{"reachable":false,"robots_served":false,"robots_sitemap":false,"content_signal":false,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":[],"blocks_any_ai":false,"files":{"llms_txt":false,"agents_md":false,"agent_skills":false,"mcp_card":false,"api_catalog":false,"a2a_card":false},"any_capability":false,"ai_refused":[],"ai_thin":[],"rings":{"mcp_declared":false,"mcp_live":false,"mcp_auth":false,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":2,"fh":"gtld-servers.net","fp_other_host":[]},"derive_v":3,"frame":"top1k","shard":2},{"v":1,"kind":"crawlcheck-census-record","run":14,"domain":"azure.com","rank":28,"measured_at":"2026-10-15T05:29:03.596Z","home":{"browser":{"status":200,"content_type":"text/html","bytes":464207,"sha256":"f373004384fd7c99df320ec0344599221b336cd9113b649a10fcf9aaff7d951d","final_host":"azure.microsoft.com","ms":678,"identity":"Mobile browser","html":true,"fp":"ea5feb592c880626"},"gptbot":{"status":200,"content_type":"text/html","bytes":464207,"sha256":"18f87d2bf175f5ec3b876063cf2a483c3bcf87cb36bbf60fec4515e0d1b680ef","final_host":"azure.microsoft.com","ms":6236,"identity":"GPTBot","html":true,"fp":"ea5feb592c880e26"},"claudebot":{"status":200,"content_type":"text/html","bytes":464206,"sha256":"49b3ade03105279113aa68ffa9a5721e18feee50ee2d8e55c35961daeb1d6953","final_host":"azure.microsoft.com","ms":6142,"identity":"ClaudeBot","html":true,"fp":"ea5feb592cc80e26"},"oaisearch":{"status":200,"content_type":"text/html","bytes":464207,"sha256":"322ec411a3ecd657b23688143cb00bc1b6d6329e331947118c4c47a2eaca0914","final_host":"azure.microsoft.com","ms":7142,"identity":"OAI-SearchBot","html":true,"fp":"ea5feb592c880e26"},"perplexity":{"status":200,"content_type":"text/html","bytes":464206,"sha256":"3046db7415a5981b3843ce6cc762242449651b61a1e0d9f44006f21e7972f92a","final_host":"azure.microsoft.com","ms":5499,"identity":"PerplexityBot","html":true,"fp":"ea5feb592cc80e26"}},"robots":{"status":200,"content_type":"text/html","bytes":464206,"sha256":"1757a69fa07dee650448d7e7a6eb717713cce4a5e44a0aa618a67b114b032b19","final_host":"azure.microsoft.com","ms":611,"html":true,"text":null,"agents":null},"sitemap":{"status":200,"content_type":"text/html","bytes":464207,"sha256":"2824751af554499e3609179f9e063069252dd21e62a8b8a7a7618b4f0b2e6834","final_host":"azure.microsoft.com","ms":212,"url":"https://azure.com/sitemap.xml","declared":false,"xml":false},"files":{"llms_txt":{"status":404,"content_type":"text/html","bytes":94794,"sha256":"184751bec282e7be57d76d203c33c1c44d65055d2cb2c69e64667dab855991fe","final_host":"azure.microsoft.com","ms":737,"path":"/llms.txt","html":true,"json":null,"present":false},"agents_md":{"status":200,"content_type":"text/html","bytes":464206,"sha256":"9021e1ddb774c9c644522d56bdc4adae85a4f83108597b397fb99a60fb854278","final_host":"azure.microsoft.com","ms":259,"path":"/agents.md","html":true,"json":null,"present":false},"agent_skills":{"status":200,"content_type":"text/html","bytes":464206,"sha256":"0c6da1708106b7bde160f179121aeadfa79268195557ab8630b48dbb033b3e92","final_host":"azure.microsoft.com","ms":601,"path":"/.well-known/agent-skills/index.json","html":true,"json":null,"present":false},"mcp_card":{"status":200,"content_type":"text/html","bytes":464207,"sha256":"8bd8edc7277a46c7e05bd286fcd3bc76495ad7fe4d9188051da8d782b895031e","final_host":"azure.microsoft.com","ms":389,"path":"/.well-known/mcp/server-card.json","html":true,"json":null,"present":false},"api_catalog":{"status":200,"content_type":"text/html","bytes":464206,"sha256":"4c826c824f488b99d79f1668e84421648d67a4b2efb2177d8169fec9965c8156","final_host":"azure.microsoft.com","ms":301,"path":"/.well-known/api-catalog","html":true,"json":null,"present":false},"a2a_card":{"status":200,"content_type":"text/html","bytes":464206,"sha256":"994bfafb1d62c211bc53a2908b8abc2b565c8894f807c2659e1aa737f967155c","final_host":"azure.microsoft.com","ms":239,"path":"/.well-known/agent-card.json","html":true,"json":null,"present":false}},"rings":{"v":1,"candidates":[],"mcp":[],"a2a":null,"webmcp":{"model_context":false,"bridge":false,"mcp_json":false,"link_rel":false,"any":false},"handshakes":0,"get_path":{"status":200,"content_type":"text/html","ms":343,"error":null},"get_host":{"status":530,"content_type":"text/plain","ms":74,"error":null}},"derived":{"reachable":true,"robots_served":false,"robots_sitemap":false,"content_signal":false,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":[],"blocks_any_ai":false,"files":{"llms_txt":false,"agents_md":false,"agent_skills":false,"mcp_card":false,"api_catalog":false,"a2a_card":false},"any_capability":false,"ai_refused":[],"ai_thin":[],"rings":{"mcp_declared":false,"mcp_live":false,"mcp_auth":false,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":2,"fh":"azure.microsoft.com","fp_other_host":[],"fp":"ea5feb592c880626","fp_bits":{"gptbot":1,"claudebot":2,"oaisearch":1,"perplexity":2}},"derive_v":3,"frame":"top1k","shard":3},{"v":1,"kind":"crawlcheck-census-record","run":14,"domain":"github.com","rank":29,"measured_at":"2026-10-15T05:30:03.842Z","home":{"browser":{"status":200,"content_type":"text/html","bytes":572372,"sha256":"82b3030be15f9d74593ec35a7b8a1cfff3266c2ac27513b9c80517d7eb5fb830","final_host":"github.com","ms":306,"identity":"Mobile browser","html":true,"fp":"6456a3418aa14c8f"},"gptbot":{"status":200,"content_type":"text/html","bytes":572373,"sha256":"22065fc73c36ea4e6c6ebb5ebc62ed54f307cd317e71f1c8a09f82f55bf35f47","final_host":"github.com","ms":164,"identity":"GPTBot","html":true,"fp":"6456a3418aa14c8f"},"claudebot":{"status":200,"content_type":"text/html","bytes":572373,"sha256":"3b3e180a8abc13b74c9a6f7b3bdaa6df9060317112582bff9b1484bd6284f983","final_host":"github.com","ms":125,"identity":"ClaudeBot","html":true,"fp":"6456a3418aa14c8f"},"oaisearch":{"status":200,"content_type":"text/html","bytes":572373,"sha256":"9ce9c1eeaa1b3748b686100d82a14b4547b1e34b51ec6a6905e2e7b5bfc75214","final_host":"github.com","ms":108,"identity":"OAI-SearchBot","html":true,"fp":"6456a3418aa14c8f"},"perplexity":{"status":200,"content_type":"text/html","bytes":572373,"sha256":"1b9d18e11e034fcb2c4e14a7ba7ef6308a01a3dee53f1b8369c4b0a30e9cd937","final_host":"github.com","ms":64,"identity":"PerplexityBot","html":true,"fp":"6456a3418aa14c8f"}},"robots":{"status":200,"content_type":"text/plain","bytes":16739,"sha256":"12bc55582b1ddb8eb3300d9cb26b0421c9fc80193fc77e5f04862e934f8097db","final_host":"github.com","ms":210,"html":false,"text":"# If you would like to crawl GitHub contact us via https://support.github.com?tags=dotcom-robots\n# We also provide an extensive API: https://docs.github.com\nUser-agent: GPTBot\nUser-agent: OAI-SearchBot\nUser-agent: ClaudeBot\nUser-agent: anthropic-ai\nUser-agent: PerplexityBot\nCrawl-delay: 1\nAllow: /$\nAllow: /about\nAllow: /customer-stories\nAllow: /enterprise\nAllow: /features\nAllow: /marketplace\nAllow: /mcp\nAllow: /newsroom\nAllow: /pricing\nAllow: /resources\nAllow: /security\nAllow: /solutions\nAllow: /topics/download*\nAllow: /topics/comments*\nAllow: /marketplace/actions/download*\nDisallow: /*/*/pulse\nDisallow: /*/*/projects\nDisallow: /*/*/forks\nDisallow: /*/*/issues/new\nDisallow: /*/*/milestones/new\nDisallow: /*/*/issues/search\nDisallow: /*/*/commits/\nDisallow: /*/*/branches\nDisallow: /*/*/contributors\nDisallow: /*/*/tags\nDisallow: /*/*/stargazers\nDisallow: /*/*/watchers\nDisallow: /*/*/network\nDisallow: /*/*/graphs\nDisallow: /*/*/compare\nDisallow: /*/tree/\nDisallow: /gist/\nDisallow: /*/download\nDisallow: /*/revisions\nDisallow: /*/commits/*?author\nDisallow: /*/commits/*?path\nDisallow: /*/comments\nDisallow: /*/archive/\nDisallow: /*/blame/\nDisallow: /*/raw/\nDisallow: /*/cache/\nDisallow: /.git/\nDisallow: */.git/\nDisallow: /*.git$\nDisallow: /search/advanced\nDisallow: /search$\nDisallow: /*.atom$\nDisallow: /ekansa/Open-Context-Data\nDisallow: /ekansa/opencontext-*\nDisallow: */tarball/\nDisallow: */zipball/\nDisallow: /*commits?author=*\nDisallow: /*report-abuse?report=*\nDisallow: /account-login\nDisallow: /Explodingstuff/\nDisallow: /copilot/\nDisallow: /copilot/c/\nDisallow: /marketplace/*/*?tab=readme$\nDisallow: /marketplace/*?*?tab=readme$\nDisallow: /marketplace/actions?tab=readme$\nDisallow: /marketplace/free-trials?tab=readme$\nDisallow: /marketplace/manage?tab=readme$\nDisallow: /marketplace/models?tab=readme$\nDisallow: /marketplace/new?tab=readme$\nDisallow: /marketplace/publishers?tab=readme$\nDisallow: /marketplace/recently-added?tab=readme$\nDisallow: /marketplace/*/*?tab=transparency$\nDisallow: /marketplace/*?*?tab=transparency$\nDisallow: /marketplace/actions?tab=transparency$\nDisallow: /marketplace/free-trials?tab=transparency$\nDisallow: /marketplace/manage?tab=transparency$\nDisallow: /marketplace/models?tab=transparency$\nDisallow: /marketplace/new?tab=transparency$\nDisallow: /marketplace/publishers?tab=transparency$\nDisallow: /marketplace/recently-added?tab=transparency$\nDisallow: /*?source=*\nDisallow: /*&source=*\nDisallow: /*?ref_cta=*\nDisallow: /*&ref_cta=*\nDisallow: /*?plan=*\nDisallow: /*&plan=*\nDisallow: /*?return_to=*\nDisallow: /*&return_to=*\nDisallow: /*?ref_loc=*\nDisallow: /*&ref_loc=*\nDisallow: /*?setup_organization=*\nDisallow: /*&setup_organization=*\nDisallow: /*?source_repo=*\nDisallow: /*&source_repo=*\nDisallow: /*?ref_page=*\nDisallow: /*&ref_page=*\nDisallow: /*?referrer=*\nDisallow: /*&referrer=*\nDisallow: /*?report=*\nDisallow: /*&report=*\nDisallow: /*?author=*\nDisallow: /*&author=*\nDisallow: /*?since=*\nDisallow: /*&since=*\nDisallow: /*?until=*\nDisallow: /*&until=*\nDisallow: /*?tab=*\nDisallow: /*&tab=*\nDisallow: /*?q=*\nDisallow: /*&q=*\nDisallow: /topics/download*?source=*\nDisallow: /topics/download*&source=*\nDisallow: /topics/download*?ref_cta=*\nDisallow: /topics/download*&ref_cta=*\nDisallow: /topics/download*?plan=*\nDisallow: /topics/download*&plan=*\nDisallow: /topics/download*?return_to=*\nDisallow: /topics/download*&return_to=*\nDisallow: /topics/download*?ref_loc=*\nDisallow: /topics/download*&ref_loc=*\nDisallow: /topics/download*?setup_organization=*\nDisallow: /topics/download*&setup_organization=*\nDisallow: /topics/download*?source_repo=*\nDisallow: /topics/download*&source_repo=*\nDisallow: /topics/download*?ref_page=*\nDisallow: /topics/download*&ref_page=*\nDisallow: /topics/download*?referrer=*\nDisallow: /topics/download*&referrer=*\nDisallow: /topics/download*?report=*\nDisallow: /topics/download*&report=*\nDisallow: /topics/download*?author=*\nDisallow: /topics/download*&author=*\nDisallow: /topics/download*?since=*\nDisallow: /topics/download*&since=*\nDisallow: /topics/download*?until=*\nDisallow: /topics/download*&until=*\nDisallow: /topics/download*?tab=*\nDisallow: /topics/download*&tab=*\nDisallow: /topics/download*?q=*\nDisallow: /topics/download*&q=*\nDisallow: /topics/comments*?source=*\nDisallow: /topics/comments*&source=*\nDisallow: /topics/comments*?ref_cta=*\nDisallow: /topics/comments*&ref_cta=*\nDisallow: /topics/comments*?plan=*\nDisallow: /topics/comments*&plan=*\nDisallow: /topics/comments*?return_to=*\nDisallow: /topics/comments*&return_to=*\nDisallow: /topics/comments*?ref_loc=*\nDisallow: /topics/comments*&ref_loc=*\nDisallow: /topics/comments*?setup_organization=*\nDisallow: /topics/comments*&setup_organization=*\nDisallow: /topics/comments*?source_repo=*\nDisallow: /topics/comments*&source_repo=*\nDisallow: /topics/comments*?ref_page=*\nDisallow: /topics/comments*&ref_page=*\nDisallow: /topics/comments*?referrer=*\nDisallow: /topics/comments*&referrer=*\nDisallow: /topics/comments*?report=*\nDisallow: /topics/comments*&report=*\nDisallow: /topics/comments*?author=*\nDisallow: /topics/comments*&author=*\nDisallow: /topics/comments*?since=*\nDisallow: /topics/comments*&since=*\nDisallow: /topics/comments*?until=*\nDisallow: /topics/comments*&until=*\nDisallow: /topics/comments*?tab=*\nDisallow: /topics/comments*&tab=*\nDisallow: /topics/comments*?q=*\nDisallow: /topics/comments*&q=*\nDisallow: /marketplace/actions/download*?source=*\nDisallow: /marketplace/actions/download*&source=*\nDisallow: /marketplace/actions/download*?ref_cta=*\nDisallow: /marketplace/actions/download*&ref_cta=*\nDisallow: /marketplace/actions/download*?plan=*\nDisallow: /marketplace/actions/download*&plan=*\nDisallow: /marketplace/actions/download*?return_to=*\nDisallow: /marketplace/actions/download*&return_to=*\nDisallow: /marketplace/actions/download*?ref_loc=*\nDisallow: /marketplace/actions/download*&ref_loc=*\nDisallow: /marketplace/actions/download*?setup_organization=*\nDisallow: /marketplace/actions/download*&setup_organization=*\nDisallow: /marketplace/actions/download*?source_repo=*\nDisallow: /marketplace/actions/download*&source_repo=*\nDisallow: /marketplace/actions/download*?ref_page=*\nDisallow: /marketplace/actions/download*&ref_page=*\nDisallow: /marketplace/actions/download*?referrer=*\nDisallow: /marketplace/actions/download*&referrer=*\nDisallow: /marketplace/actions/download*?report=*\nDisallow: /marketplace/actions/download*&report=*\nDisallow: /marketplace/actions/download*?author=*\nDisallow: /marketplace/actions/download*&author=*\nDisallow: /marketplace/actions/download*?since=*\nDisallow: /marketplace/actions/download*&since=*\nDisallow: /marketplace/actions/download*?until=*\nDisallow: /marketplace/actions/download*&until=*\nDisallow: /marketplace/actions/download*?tab=*\nDisallow: /marketplace/actions/download*&tab=*\nDisallow: /marketplace/actions/download*?q=*\nDisallow: /marketplace/actions/download*&q=*\n\nUser-agent: Bytespider\nDisallow: /\n\nUser-agent: AdsBot-Google\nUser-agent: AdsBot-Google-Mobile\nDisallow: /*/*/pulse\nDisallow: /*/*/projects\nDisallow: /*/*/forks\nDisallow: /*/*/issues/new\nDisallow: /*/*/milestones/new\nDisallow: /*/*/issues/search\nDisallow: /*/*/commits/\nDisallow: /*/*/branches\nDisallow: /*/*/contributors\nDisallow: /*/*/tags\nDisallow: /*/*/stargazers\nDisallow: /*/*/watchers\nDisallow: /*/*/network\nDisallow: /*/*/graphs\nDisallow: /*/*/compare\nDisallow: /*/tree/\nDisallow: /gist/\nDisallow: /*/download\nDisallow: /*/revisions\nDisallow: /*/commits/*?author\nDisallow: /*/commits/*?path\nDisallow: /*/comments\nDisallow: /*/archive/\nDisallow: /*/blame/\nDisallow: /*/raw/\nDisallow: /*/cache/\nDisallow: /.git/\nDisallow: */.git/\nDisallow: /*.git$\nDisallow: /search/advanced\nDisallow: /search$\nDisallow: /*.atom$\nDisallow: /ekansa/Open-Context-Data\nDisallow: /ekansa/opencontext-*\nDisallow: */tarball/\nDisallow: */zipball/\nDisallow: /*commits?author=*\nDisallow: /*report-abuse?report=*\nDisallow: /account-login\nDisallow: /Explodingstuff/\nDisallow: /copilot/\nDisallow: /copilot/c/\nDisallow: /marketplace/*/*?tab=readme$\nDisallow: /marketplace/*?*?tab=readme$\nDisallow: /marketplace/actions?tab=readme$\nDisallow: /marketplace/free-trials?tab=readme$\nDisallow: /marketplace/manage?tab=readme$\nDisallow: /marketplace/models?tab=readme$\nDisallow: /marketplace/new?tab=readme$\nDisallow: /marketplace/publishers?tab=readme$\nDisallow: /marketplace/recently-added?tab=readme$\nDisallow: /marketplace/*/*?tab=transparency$\nDisallow: /marketplace/*?*?tab=transparency$\nDisallow: /marketplace/actions?tab=transparency$\nDisallow: /marketplace/free-trials?tab=transparency$\nDisallow: /marketplace/manage?tab=transparency$\nDisallow: /marketplace/models?tab=transparency$\nDisallow: /marketplace/new?tab=transparency$\nDisallow: /marketplace/publishers?tab=transparency$\nDisallow: /marketplace/recently-added?tab=transparency$\nDisallow: /*?source=*\nDisallow: /*&source=*\nDisallow: /*?ref_cta=*\nDisallow: /*&ref_cta=*\nDisallow: /*?plan=*\nDisallow: /*&plan=*\nDisallow: /*?return_to=*\nDisallow: /*&return_to=*\nDisallow: /*?ref_loc=*\nDisallow: /*&ref_loc=*\nDisallow: /*?setup_organization=*\nDisallow: /*&setup_organization=*\nDisallow: /*?source_repo=*\nDisallow: /*&source_repo=*\nDisallow: /*?ref_page=*\nDisallow: /*&ref_page=*\nDisallow: /*?referrer=*\nDisallow: /*&referrer=*\nDisallow: /*?report=*\nDisallow: /*&report=*\nDisallow: /*?author=*\nDisallow: /*&author=*\nDisallow: /*?since=*\nDisallow: /*&since=*\nDisallow: /*?until=*\nDisallow: /*&until=*\nDisallow: /*?tab=*\nDisallow: /*&tab=*\nDisallow: /*?q=*\nDisallow: /*&q=*\n\nUser-agent: bingbot\nDisallow: /ekansa/Open-Context-Data\nDisallow: /ekansa/opencontext-*\nDisallow: /account-login\nDisallow: */tarball/\nDisallow: */zipball/\nDisallow: /Explodingstuff/\nDisallow: /copilot/\nDisallow: /copilot/c/\n\nUser-agent: adidxbot\nDisallow: /ekansa/Open-Context-Data\nDisallow: /ekansa/opencontext-*\nDisallow: /account-login\nDisallow: */tarball/\nDisallow: */zipball/\nDisallow: /Explodingstuff/\nDisallow: /copilot/\nDisallow: /copilot/c/\n\nUser-agent: BingPreview\nDisallow: /ekansa/Open-Context-Data\nDisallow: /ekansa/opencontext-*\nDisallow: /account-login\nDisallow: */tarball/\nDisallow: */zipball/\nDisallow: /Explodingstuff/\nDisallow: /copilot/\nDisallow: /copilot/c/\n\nUser-agent: baidu\ncrawl-delay: 1\n\n\nUser-agent: *\nAllow: /*?tab=achievements&achievement=*\nAllow: /marketplace/*?tab=readme$\nAllow: /marketplace/*?tab=transparency$\nAllow: /topics/download*\nAllow: /topics/comments*\nAllow: /marketplace/actions/download*\nDisallow: /*/*/pulse\nDisallow: /*/*/projects\nDisallow: /*/*/forks\nDisallow: /*/*/issues/new\nDisallow: /*/*/milestones/new\nDisallow: /*/*/issues/search\nDisallow: /*/*/commits/\nDisallow: /*/*/branches\nDisallow: /*/*/contributors\nDisallow: /*/*/tags\nDisallow: /*/*/stargazers\nDisallow: /*/*/watchers\nDisallow: /*/*/network\nDisallow: /*/*/graphs\nDisallow: /*/*/compare\nDisallow: /*/tree/\nDisallow: /gist/\nDisallow: /*/download\nDisallow: /*/revisions\nDisallow: /*/commits/*?author\nDisallow: /*/commits/*?path\nDisallow: /*/comments\nDisallow: /*/archive/\nDisallow: /*/blame/\nDisallow: /*/raw/\nDisallow: /*/cache/\nDisallow: /.git/\nDisallow: */.git/\nDisallow: /*.git$\nDisallow: /search/advanced\nDisallow: /search$\nDisallow: /*.atom$\nDisallow: /ekansa/Open-Context-Data\nDisallow: /ekansa/opencontext-*\nDisallow: */tarball/\nDisallow: */zipball/\nDisallow: /*commits?author=*\nDisallow: /*report-abuse?report=*\nDisallow: /account-login\nDisallow: /Explodingstuff/\nDisallow: /copilot/\nDisallow: /copilot/c/\nDisallow: /marketplace/*/*?tab=readme$\nDisallow: /marketplace/*?*?tab=readme$\nDisallow: /marketplace/actions?tab=readme$\nDisallow: /marketplace/free-trials?tab=readme$\nDisallow: /marketplace/manage?tab=readme$\nDisallow: /marketplace/models?tab=readme$\nDisallow: /marketplace/new?tab=readme$\nDisallow: /marketplace/publishers?tab=readme$\nDisallow: /marketplace/recently-added?tab=readme$\nDisallow: /marketplace/*/*?tab=transparency$\nDisallow: /marketplace/*?*?tab=transparency$\nDisallow: /marketplace/actions?tab=transparency$\nDisallow: /marketplace/free-trials?tab=transparency$\nDisallow: /marketplace/manage?tab=transparency$\nDisallow: /marketplace/models?tab=transparency$\nDisallow: /marketplace/new?tab=transparency$\nDisallow: /marketplace/publishers?tab=transparency$\nDisallow: /marketplace/recently-added?tab=transparency$\nDisallow: /*?source=*\nDisallow: /*&source=*\nDisallow: /*?ref_cta=*\nDisallow: /*&ref_cta=*\nDisallow: /*?plan=*\nDisallow: /*&plan=*\nDisallow: /*?return_to=*\nDisallow: /*&return_to=*\nDisallow: /*?ref_loc=*\nDisallow: /*&ref_loc=*\nDisallow: /*?setup_organization=*\nDisallow: /*&setup_organization=*\nDisallow: /*?source_repo=*\nDisallow: /*&source_repo=*\nDisallow: /*?ref_page=*\nDisallow: /*&ref_page=*\nDisallow: /*?referrer=*\nDisallow: /*&referrer=*\nDisallow: /*?report=*\nDisallow: /*&report=*\nDisallow: /*?author=*\nDisallow: /*&author=*\nDisallow: /*?since=*\nDisallow: /*&since=*\nDisallow: /*?until=*\nDisallow: /*&until=*\nDisallow: /*?tab=*\nDisallow: /*&tab=*\nDisallow: /*?q=*\nDisallow: /*&q=*\nDisallow: /topics/download*?source=*\nDisallow: /topics/download*&source=*\nDisallow: /topics/download*?ref_cta=*\nDisallow: /topics/download*&ref_cta=*\nDisallow: /topics/download*?plan=*\nDisallow: /topics/download*&plan=*\nDisallow: /topics/download*?return_to=*\nDisallow: /topics/download*&return_to=*\nDisallow: /topics/download*?ref_loc=*\nDisallow: /topics/download*&ref_loc=*\nDisallow: /topics/download*?setup_organization=*\nDisallow: /topics/download*&setup_organization=*\nDisallow: /topics/download*?source_repo=*\nDisallow: /topics/download*&source_repo=*\nDisallow: /topics/download*?ref_page=*\nDisallow: /topics/download*&ref_page=*\nDisallow: /topics/download*?referrer=*\nDisallow: /topics/download*&referrer=*\nDisallow: /topics/download*?report=*\nDisallow: /topics/download*&report=*\nDisallow: /topics/download*?author=*\nDisallow: /topics/download*&author=*\nDisallow: /topics/download*?since=*\nDisallow: /topics/download*&since=*\nDisallow: /topics/download*?until=*\nDisallow: /topics/download*&until=*\nDisallow: /topics/download*?tab=*\nDisallow: /topics/download*&tab=*\nDisallow: /topics/download*?q=*\nDisallow: /topics/download*&q=*\nDisallow: /topics/comments*?source=*\nDisallow: /topics/comments*&source=*\nDisallow: /topics/comments*?ref_cta=*\nDisallow: /topics/comments*&ref_cta=*\nDisallow: /topics/comments*?plan=*\nDisallow: /topics/comments*&plan=*\nDisallow: /topics/comments*?return_to=*\nDisallow: /topics/comments*&return_to=*\nDisallow: /topics/comments*?ref_loc=*\nDisallow: /topics/comments*&ref_loc=*\nDisallow: /topics/comments*?setup_organization=*\nDisallow: /topics/comments*&setup_organization=*\nDisallow: /topics/comments*?source_repo=*\nDisallow: /topics/comments*&source_repo=*\nDisallow: /topics/comments*?ref_page=*\nDisallow: /topics/comments*&ref_page=*\nDisallow: /topics/comments*?referrer=*\nDisallow: /topics/comments*&referrer=*\nDisallow: /topics/comments*?report=*\nDisallow: /topics/comments*&report=*\nDisallow: /topics/comments*?author=*\nDisallow: /topics/comments*&author=*\nDisallow: /topics/comments*?since=*\nDisallow: /topics/comments*&since=*\nDisallow: /topics/comments*?until=*\nDisallow: /topics/comments*&until=*\nDisallow: /topics/comments*?tab=*\nDisallow: /topics/comments*&tab=*\nDisallow: /topics/comments*?q=*\nDisallow: /topics/comments*&q=*\nDisallow: /marketplace/actions/download*?source=*\nDisallow: /marketplace/actions/download*&source=*\nDisallow: /marketplace/actions/download*?ref_cta=*\nDisallow: /marketplace/actions/download*&ref_cta=*\nDisallow: /marketplace/actions/download*?plan=*\nDisallow: /marketplace/actions/download*&plan=*\nDisallow: /marketplace/actions/download*?return_to=*\nDisallow: /marketplace/actions/download*&return_to=*\nDisallow: /marketplace/actions/download*?ref_loc=*\nDisallow: /marketplace/actions/download*&ref_loc=*\nDisallow: /marketplace/actions/download*?setup_organization=*\nDisallow: /marketplace/actions/download*&setup_organization=*\nDisallow: /marketplace/actions/download*?source_repo=*\nDisallow: /marketplace/actions/download*&source_repo=*\nDisallow: /marketplace/actions/download*?ref_page=*\nDisallow: /marketplace/actions/download*&ref_page=*\nDisallow: /marketplace/actions/download*?referrer=*\nDisallow: /marketplace/actions/download*&referrer=*\nDisallow: /marketplace/actions/download*?report=*\nDisallow: /marketplace/actions/download*&report=*\nDisallow: /marketplace/actions/download*?author=*\nDisallow: /marketplace/actions/download*&author=*\nDisallow: /marketplace/actions/download*?since=*\nDisallow: /marketplace/actions/download*&since=*\nDisallow: /marketplace/actions/download*?until=*\nDisallow: /marketplace/actions/download*&until=*\nDisallow: /marketplace/actions/download*?tab=*\nDisallow: /marketplace/actions/download*&tab=*\nDisallow: /marketplace/actions/download*?q=*\nDisallow: /marketplace/actions/download*&q=*\n","agents":{"GPTBot":true,"OAI-SearchBot":true,"ChatGPT-User":true,"ClaudeBot":true,"Claude-SearchBot":true,"PerplexityBot":true,"Google-Extended":true,"Applebot-Extended":true,"CCBot":true,"Bytespider":false,"Meta-ExternalAgent":true,"Amazonbot":true}},"sitemap":{"status":406,"content_type":"application/xml","bytes":0,"sha256":"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855","final_host":"github.com","ms":215,"url":"https://github.com/sitemap.xml","declared":false,"xml":false},"files":{"llms_txt":{"status":200,"content_type":"text/plain","bytes":28658,"sha256":"71d217ec3186152052ca1050932c0e6e48e766d3bc5a1359b50aa140436d297f","final_host":"github.com","ms":208,"path":"/llms.txt","html":false,"json":null,"present":true},"agents_md":{"status":406,"content_type":"text/html","bytes":0,"sha256":"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855","final_host":"github.com","ms":218,"path":"/agents.md","html":true,"json":null,"present":false},"agent_skills":{"status":404,"content_type":"application/json","bytes":21,"sha256":"e28bf7d9fcc22cdfde0c7c8f31e3648a4847e7bda9cb69f309f24257eee3dd41","final_host":"github.com","ms":147,"path":"/.well-known/agent-skills/index.json","html":false,"json":null,"present":false},"mcp_card":{"status":404,"content_type":"application/json","bytes":21,"sha256":"e28bf7d9fcc22cdfde0c7c8f31e3648a4847e7bda9cb69f309f24257eee3dd41","final_host":"github.com","ms":141,"path":"/.well-known/mcp/server-card.json","html":false,"json":null,"present":false},"api_catalog":{"status":404,"content_type":"text/html","bytes":264600,"sha256":"423598ed22b0f2418c1a24b4ba43ddbcb641e6011570167a472ab8cbe238a3e0","final_host":"github.com","ms":265,"path":"/.well-known/api-catalog","html":true,"json":null,"present":false},"a2a_card":{"status":404,"content_type":"application/json","bytes":21,"sha256":"e28bf7d9fcc22cdfde0c7c8f31e3648a4847e7bda9cb69f309f24257eee3dd41","final_host":"github.com","ms":176,"path":"/.well-known/agent-card.json","html":false,"json":null,"present":false}},"rings":{"v":1,"candidates":[{"url":"https://github.com/mcp","source":"homepage_link"}],"mcp":[{"url":"https://github.com/mcp","source":"homepage_link","outcome":"http_error","status":422,"ms":125,"initialized":false,"protocol_version":null,"server":null,"tools":null,"tools_sha256":null,"auth_header":false}],"a2a":null,"webmcp":{"model_context":false,"bridge":false,"mcp_json":false,"link_rel":false,"any":false},"handshakes":1,"get_host":{"status":530,"content_type":"text/plain","ms":178,"error":null}},"derived":{"reachable":true,"robots_served":true,"robots_sitemap":false,"content_signal":false,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":["Bytespider"],"blocks_any_ai":true,"files":{"llms_txt":true,"agents_md":false,"agent_skills":false,"mcp_card":false,"api_catalog":false,"a2a_card":false},"any_capability":false,"ai_refused":[],"ai_thin":[],"rings":{"mcp_declared":true,"mcp_live":false,"mcp_auth":false,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":2,"fh":"github.com","fp_other_host":[],"fp":"6456a3418aa14c8f","fp_bits":{"gptbot":0,"claudebot":0,"oaisearch":0,"perplexity":0}},"derive_v":3,"frame":"top1k","shard":4},{"v":1,"kind":"crawlcheck-census-record","run":14,"domain":"wikipedia.org","rank":30,"measured_at":"2026-10-15T05:31:08.958Z","home":{"browser":{"status":200,"content_type":"text/html","bytes":122197,"sha256":"5c0de7d8a1af1dc8c06cc2a517265ff96db0ed11114a62be2625ed13c9cdb466","final_host":"www.wikipedia.org","ms":297,"identity":"Mobile browser","html":true,"fp":"9f62870a873f37ff"},"gptbot":{"status":200,"content_type":"text/html","bytes":122197,"sha256":"5c0de7d8a1af1dc8c06cc2a517265ff96db0ed11114a62be2625ed13c9cdb466","final_host":"www.wikipedia.org","ms":297,"identity":"GPTBot","html":true,"fp":"9f62870a873f37ff"},"claudebot":{"status":403,"content_type":"text/plain","bytes":82,"sha256":"41609a6edee1e154f2092d6b2c64e26c5aa5a21d445ae636fe5a1f24b1ebc905","final_host":"wikipedia.org","ms":111,"identity":"ClaudeBot","html":false},"oaisearch":{"status":200,"content_type":"text/html","bytes":122197,"sha256":"5c0de7d8a1af1dc8c06cc2a517265ff96db0ed11114a62be2625ed13c9cdb466","final_host":"www.wikipedia.org","ms":281,"identity":"OAI-SearchBot","html":true,"fp":"9f62870a873f37ff"},"perplexity":{"status":200,"content_type":"text/html","bytes":122197,"sha256":"5c0de7d8a1af1dc8c06cc2a517265ff96db0ed11114a62be2625ed13c9cdb466","final_host":"www.wikipedia.org","ms":245,"identity":"PerplexityBot","html":true,"fp":"9f62870a873f37ff"}},"robots":{"status":200,"content_type":"text/plain","bytes":28275,"sha256":"48b98d80c8444d37cdf50cc9a4d2257b66039477e581066d2356062a4cbd678f","final_host":"en.wikipedia.org","ms":542,"html":false,"text":"# robots.txt for http://www.wikipedia.org/ and friends\n#\n# Please note: There are a lot of pages on this site, and there are\n# some misbehaved spiders out there that go _way_ too fast. If you're\n# irresponsible, your access to the site may be blocked.\n#\n\n# Observed spamming large amounts of https://en.wikipedia.org/?curid=NNNNNN\n# and ignoring 429 ratelimit responses, claims to respect robots:\n# http://mj12bot.com/\nUser-agent: MJ12bot\nDisallow: /\n\n# advertising-related bots:\nUser-agent: Mediapartners-Google*\nDisallow: /\n\n# Wikipedia work bots:\nUser-agent: IsraBot\nDisallow:\n\nUser-agent: Orthogaffe\nDisallow:\n\n# Crawlers that are kind enough to obey, but which we'd rather not have\n# unless they're feeding search engines.\nUser-agent: UbiCrawler\nDisallow: /\n\nUser-agent: DOC\nDisallow: /\n\nUser-agent: Zao\nDisallow: /\n\n# Some bots are known to be trouble, particularly those designed to copy\n# entire sites. Please obey robots.txt.\nUser-agent: sitecheck.internetseer.com\nDisallow: /\n\nUser-agent: Zealbot\nDisallow: /\n\nUser-agent: MSIECrawler\nDisallow: /\n\nUser-agent: SiteSnagger\nDisallow: /\n\nUser-agent: WebStripper\nDisallow: /\n\nUser-agent: WebCopier\nDisallow: /\n\nUser-agent: Fetch\nDisallow: /\n\nUser-agent: Offline Explorer\nDisallow: /\n\nUser-agent: Teleport\nDisallow: /\n\nUser-agent: TeleportPro\nDisallow: /\n\nUser-agent: WebZIP\nDisallow: /\n\nUser-agent: linko\nDisallow: /\n\nUser-agent: HTTrack\nDisallow: /\n\nUser-agent: Microsoft.URL.Control\nDisallow: /\n\nUser-agent: Xenu\nDisallow: /\n\nUser-agent: larbin\nDisallow: /\n\nUser-agent: libwww\nDisallow: /\n\nUser-agent: ZyBORG\nDisallow: /\n\nUser-agent: Download Ninja\nDisallow: /\n\n# Misbehaving, requests much too fast\nUser-agent: fast\nDisallow: /\n\n# Sorry, wget in its recursive mode is a frequent problem.\n# Please read the man page and use it properly; there is a\n# --wait option you can use to set the delay between hits,\n# for instance.\nUser-agent: wget\nDisallow: /\n\n# The 'grub' distributed client has been *very* poorly behaved.\nUser-agent: grub-client\nDisallow: /\n\n# Doesn't follow robots.txt anyway, but...\nUser-agent: k2spider\nDisallow: /\n\n# Hits many times per second, not acceptable\n# http://www.nameprotect.com/botinfo.html\nUser-agent: NPBot\nDisallow: /\n\n# A capture bot, downloads gazillions of pages with no public benefit\n# http://www.webreaper.net/\nUser-agent: WebReaper\nDisallow: /\n\n# Per their statement, semrushbot respects crawl-delay directives\n# We want them to overall stay within reasonable request rates to\n# the backend (20 rps); keeping in mind that the crawl-delay will\n# be applied by site and not globally by the bot, 5 seconds seem\n# like a reasonable approximation\nUser-agent: SemrushBot\nCrawl-delay: 5\n\n#\n# Friendly, low-speed bots are welcome viewing article pages, but not\n# dynamically-generated pages please.\n#\n# Inktomi's \"Slurp\" can read a minimum delay between hits; if your\n# bot supports such a thing using the 'Crawl-delay' or another\n# instruction, please let us know.\n#\n# There is a special exception for API mobileview to allow dynamic\n# mobile web & app views to load section content.\n# These views aren't HTTP-cached but use parser cache aggressively\n# and don't expose Special-pages and other uncached content.\n#\n# Another exception is for REST API documentation, located at\n# /api/rest_v1/?doc.\n#\nUser-agent: *\nAllow: /w/api.php?action=mobileview&\nAllow: /w/load.php?\nAllow: /api/rest_v1/?doc\nAllow: /w/rest.php/site/v1/sitemap\nDisallow: /w/\nDisallow: /api/\nDisallow: /trap/\nDisallow: /wiki/Special:\nDisallow: /wiki/Spezial:\nDisallow: /wiki/Spesial:\nDisallow: /wiki/Special%3A\nDisallow: /wiki/Spezial%3A\nDisallow: /wiki/Spesial%3A\n#\n# ar:\nDisallow: /wiki/%D8%AE%D8%A7%D8%B5:Search\nDisallow: /wiki/%D8%AE%D8%A7%D8%B5%3ASearch\n#\n# dewiki:\n# T6937\n# sensible deletion and meta user discussion pages:\nDisallow: /wiki/Wikipedia:L%C3%B6schkandidaten/\nDisallow: /wiki/Wikipedia:Löschkandidaten/\nDisallow: /wiki/Wikipedia:Vandalensperrung/\nDisallow: /wiki/Wikipedia:Benutzersperrung/\nDisallow: /wiki/Wikipedia:Vermittlungsausschuss/\nDisallow: /wiki/Wikipedia:Administratoren/Probleme/\nDisallow: /wiki/Wikipedia:Adminkandidaturen/\nDisallow: /wiki/Wikipedia:Qualitätssicherung/\nDisallow: /wiki/Wikipedia:Qualit%C3%A4tssicherung/\n# 4937#5\nDisallow: /wiki/Wikipedia:Vandalismusmeldung/\nDisallow: /wiki/Wikipedia:Gesperrte_Lemmata/\nDisallow: /wiki/Wikipedia:Löschprüfung/\nDisallow: /wiki/Wikipedia:L%C3%B6schprüfung/\nDisallow: /wiki/Wikipedia:Administratoren/Notizen/\nDisallow: /wiki/Wikipedia:Schiedsgericht/Anfragen/\nDisallow: /wiki/Wikipedia:L%C3%B6schpr%C3%BCfung/\n# T14111\nDisallow: /wiki/Wikipedia:Checkuser/\nDisallow: /wiki/Wikipedia_Diskussion:Checkuser/\nDisallow: /wiki/Wikipedia_Diskussion:Adminkandidaturen/\n# T15961\nDisallow: /wiki/Wikipedia:Spam-Blacklist-Log\nDisallow: /wiki/Wikipedia%3ASpam-Blacklist-Log\nDisallow: /wiki/Wikipedia_Diskussion:Spam-Blacklist-Log\nDisallow: /wiki/Wikipedia_Diskussion%3ASpam-Blacklist-Log\n#\n# enwiki:\n# Folks get annoyed when VfD discussions end up the number 1 google hit for\n# their name. See T6776\nDisallow: /wiki/Wikipedia:Articles_for_deletion/\nDisallow: /wiki/Wikipedia%3AArticles_for_deletion/\nDisallow: /wiki/Wikipedia:Votes_for_deletion/\nDisallow: /wiki/Wikipedia%3AVotes_for_deletion/\nDisallow: /wiki/Wikipedia:Pages_for_deletion/\nDisallow: /wiki/Wikipedia%3APages_for_deletion/\nDisallow: /wiki/Wikipedia:Miscellany_for_deletion/\nDisallow: /wiki/Wikipedia%3AMiscellany_for_deletion/\nDisallow: /wiki/Wikipedia:Miscellaneous_deletion/\nDisallow: /wiki/Wikipedia%3AMiscellaneous_deletion/\nDisallow: /wiki/Wikipedia:Copyright_problems\nDisallow: /wiki/Wikipedia%3ACopyright_problems\nDisallow: /wiki/Wikipedia:Protected_titles/\nDisallow: /wiki/Wikipedia%3AProtected_titles/\n# T15398\nDisallow: /wiki/Wikipedia:WikiProject_Spam/\nDisallow: /wiki/Wikipedia%3AWikiProject_Spam/\n# T16075\nDisallow: /wiki/MediaWiki:Spam-blacklist\nDisallow: /wiki/MediaWiki%3ASpam-blacklist\nDisallow: /wiki/MediaWiki_talk:Spam-blacklist\nDisallow: /wiki/MediaWiki_talk%3ASpam-blacklist\n# T13261\nDisallow: /wiki/Wikipedia:Requests_for_arbitration/\nDisallow: /wiki/Wikipedia%3ARequests_for_arbitration/\nDisallow: /wiki/Wikipedia:Requests_for_comment/\nDisallow: /wiki/Wikipedia%3ARequests_for_comment/\nDisallow: /wiki/Wikipedia:Requests_for_adminship/\nDisallow: /wiki/Wikipedia%3ARequests_for_adminship/\n# T12288\nDisallow: /wiki/Wikipedia_talk:Articles_for_deletion/\nDisallow: /wiki/Wikipedia_talk%3AArticles_for_deletion/\nDisallow: /wiki/Wikipedia_talk:Votes_for_deletion/\nDisallow: /wiki/Wikipedia_talk%3AVotes_for_deletion/\nDisallow: /wiki/Wikipedia_talk:Pages_for_deletion/\nDisallow: /wiki/Wikipedia_talk%3APages_for_deletion/\nDisallow: /wiki/Wikipedia_talk:Miscellany_for_deletion/\nDisallow: /wiki/Wikipedia_talk%3AMiscellany_for_deletion/\nDisallow: /wiki/Wikipedia_talk:Miscellaneous_deletion/\nDisallow: /wiki/Wikipedia_talk%3AMiscellaneous_deletion/\n# T16793\nDisallow: /wiki/Wikipedia:Changing_username\nDisallow: /wiki/Wikipedia%3AChanging_username\nDisallow: /wiki/Wikipedia:Changing_username/\nDisallow: /wiki/Wikipedia%3AChanging_username/\nDisallow: /wiki/Wikipedia_talk:Changing_username\nDisallow: /wiki/Wikipedia_talk%3AChanging_username\nDisallow: /wiki/Wikipedia_talk:Changing_username/\nDisallow: /wiki/Wikipedia_talk%3AChanging_username/\n#\n# eswiki:\n# T8746\nDisallow: /wiki/Wikipedia:Consultas_de_borrado/\nDisallow: /wiki/Wikipedia%3AConsultas_de_borrado/\n#\n# fiwiki:\n# T10695\nDisallow: /wiki/Wikipedia:Poistettavat_sivut\nDisallow: /wiki/K%C3%A4ytt%C3%A4j%C3%A4:\nDisallow: /wiki/Käyttäjä:\nDisallow: /wiki/Keskustelu_k%C3%A4ytt%C3%A4j%C3%A4st%C3%A4:\nDisallow: /wiki/Keskustelu_käyttäjästä:\nDisallow: /wiki/Wikipedia:Yll%C3%A4pit%C3%A4j%C3%A4t/\nDisallow: /wiki/Wikipedia:Ylläpitäjät/\n#\n# hewiki:\nDisallow: /wiki/%D7%9E%D7%99%D7%95%D7%97%D7%93:Search\nDisallow: /wiki/%D7%9E%D7%99%D7%95%D7%97%D7%93%3ASearch\n#T11517\nDisallow: /wiki/ויקיפדיה:רשימת_מועמדים_למחיקה/\nDisallow: /wiki/ויקיפדיה%3Aרשימת_מועמדים_למחיקה/\nDisallow: /wiki/%D7%95%D7%99%D7%A7%D7%99%D7%A4%D7%93%D7%99%D7%94:%D7%A8%D7%A9%D7%99%D7%9E%D7%AA_%D7%9E%D7%95%D7%A2%D7%9E%D7%93%D7%99%D7%9D_%D7%9C%D7%9E%D7%97%D7%99%D7%A7%D7%94/\nDisallow: /wiki/%D7%95%D7%99%D7%A7%D7%99%D7%A4%D7%93%D7%99%D7%94%3A%D7%A8%D7%A9%D7%99%D7%9E%D7%AA_%D7%9E%D7%95%D7%A2%D7%9E%D7%93%D7%99%D7%9D_%D7%9C%D7%9E%D7%97%D7%99%D7%A7%D7%94/\nDisallow: /wiki/ויקיפדיה:ערכים_לא_קיימים_ומוגנים\nDisallow: /wiki/ויקיפדיה%3Aערכים_לא_קיימים_ומוגנים\nDisallow: /wiki/%D7%95%D7%99%D7%A7%D7%99%D7%A4%D7%93%D7%99%D7%94:%D7%A2%D7%A8%D7%9B%D7%99%D7%9D_%D7%9C%D7%90_%D7%A7%D7%99%D7%99%D7%9E%D7%99%D7%9D_%D7%95%D7%9E%D7%95%D7%92%D7%A0%D7%99%D7%9D\nDisallow: /wiki/%D7%95%D7%99%D7%A7%D7%99%D7%A4%D7%93%D7%99%D7%94%3A%D7%A2%D7%A8%D7%9B%D7%99%D7%9D_%D7%9C%D7%90_%D7%A7%D7%99%D7%99%D7%9E%D7%99%D7%9D_%D7%95%D7%9E%D7%95%D7%92%D7%A0%D7%99%D7%9D\nDisallow: /wiki/ויקיפדיה:דפים_לא_קיימים_ומוגנים\nDisallow: /wiki/ויקיפדיה%3Aדפים_לא_קיימים_ומוגנים\nDisallow: /wiki/%D7%95%D7%99%D7%A7%D7%99%D7%A4%D7%93%D7%99%D7%94:%D7%93%D7%A4%D7%99%D7%9D_%D7%9C%D7%90_%D7%A7%D7%99%D7%99%D7%9E%D7%99%D7%9D_%D7%95%D7%9E%D7%95%D7%92%D7%A0%D7%99%D7%9D\nDisallow: /wiki/%D7%95%D7%99%D7%A7%D7%99%D7%A4%D7%93%D7%99%D7%94%3A%D7%93%D7%A4%D7%99%D7%9D_%D7%9C%D7%90_%D7%A7%D7%99%D7%99%D7%9E%D7%99%D7%9D_%D7%95%D7%9E%D7%95%D7%92%D7%A0%D7%99%D7%9D\n#\n# huwiki:\nDisallow: /wiki/Speci%C3%A1lis:Search\nDisallow: /wiki/Speci%C3%A1lis%3ASearch\n#\n# itwiki:\n# T7545\nDisallow: /wiki/Wikipedia:Pagine_da_cancellare\nDisallow: /wiki/Wikipedia%3APagine_da_cancellare\nDisallow: /wiki/Wikipedia:Utenti_problematici\nDisallow: /wiki/Wikipedia%3AUtenti_problematici\nDisallow: /wiki/Wikipedia:Vandalismi_in_corso\nDisallow: /wiki/Wikipedia%3AVandalismi_in_corso\nDisallow: /wiki/Wikipedia:Amministratori\nDisallow: /wiki/Wikipedia%3AAmministratori\nDisallow: /wiki/Wikipedia:Proposte_di_cancellazione_semplificata\nDisallow: /wiki/Wikipedia%3AProposte_di_cancellazione_semplificata\nDisallow: /wiki/Categoria:Da_cancellare_subito\nDisallow: /wiki/Categoria%3ADa_cancellare_subito\nDisallow: /wiki/Wikipedia:Sospette_violazioni_di_copyright\nDisallow: /wiki/Wikipedia%3ASospette_violazioni_di_copyright\nDisallow: /wiki/Categoria:Da_controllare_per_copyright\nDisallow: /wiki/Categoria%3ADa_controllare_per_copyright\nDisallow: /wiki/Progetto:Rimozione_contributi_sospetti\nDisallow: /wiki/Progetto%3ARimozione_contributi_sospetti\nDisallow: /wiki/Categoria:Da_cancellare_subito_per_violazione_integrale_copyright\nDisallow: /wiki/Categoria%3ADa_cancellare_subito_per_violazione_integrale_copyright\nDisallow: /wiki/Progetto:Cococo\nDisallow: /wiki/Progetto%3ACococo\nDisallow: /wiki/Discussioni_progetto:Cococo\nDisallow: /wiki/Discussioni_progetto%3ACococo\n#\n# jawiki\nDisallow: /wiki/%E7%89%B9%E5%88%A5:Search\nDisallow: /wiki/%E7%89%B9%E5%88%A5%3ASearch\n# T7239\nDisallow: /wiki/Wikipedia:%E5%89%8A%E9%99%A4%E4%BE%9D%E9%A0%BC/\nDisallow: /wiki/Wikipedia%3A%E5%89%8A%E9%99%A4%E4%BE%9D%E9%A0%BC/\nDisallow: /wiki/Wikipedia:%E5%88%A9%E7%94%A8%E8%80%85%E3%83%9A%E3%83%BC%E3%82%B8%E3%81%AE%E5%89%8A%E9%99%A4%E4%BE%9D%E9%A0%BC\nDisallow: /wiki/Wikipedia%3A%E5%88%A9%E7%94%A8%E8%80%85%E3%83%9A%E3%83%BC%E3%82%B8%E3%81%AE%E5%89%8A%E9%99%A4%E4%BE%9D%E9%A0%BC\n# nowiki\n# T13432\nDisallow: /wiki/Bruker:\nDisallow: /wiki/Bruker%3A\nDisallow: /wiki/Brukerdiskusjon\nDisallow: /wiki/Wikipedia:Administratorer\nDisallow: /wiki/Wikipedia%3AAdministratorer\nDisallow: /wiki/Wikipedia-diskusjon:Administratorer\nDisallow: /wiki/Wikipedia-diskusjon%3AAdministratorer\nDisallow: /wiki/Wikipedia:Sletting\nDisallow: /wiki/Wikipedia%3ASletting\nDisallow: /wiki/Wikipedia-diskusjon:Sletting\nDisallow: /wiki/Wikipedia-diskusjon%3ASletting\n#\n# plwiki\n# T10067\nDisallow: /wiki/Wikipedia:Strony_do_usuni%C4%99cia\nDisallow: /wiki/Wikipedia%3AStrony_do_usuni%C4%99cia\nDisallow: /wiki/Wikipedia:Do_usuni%C4%99cia\nDisallow: /wiki/Wikipedia%3ADo_usuni%C4%99cia\nDisallow: /wiki/Wikipedia:SDU/\nDisallow: /wiki/Wikipedia%3ASDU/\nDisallow: /wiki/Wikipedia:Strony_podejrzane_o_naruszenie_praw_autorskich\nDisallow: /wiki/Wikipedia%3AStrony_podejrzane_o_naruszenie_praw_autorskich\n#\n# ptwiki:\n# T7394\nDisallow: /wiki/Wikipedia:Páginas_para_eliminar/\nDisallow: /wiki/Wikipedia:P%C3%A1ginas_para_eliminar/\nDisallow: /wiki/Wikipedia%3AP%C3%A1ginas_para_eliminar/\nDisallow: /wiki/Wikipedia_Discussão:Páginas_para_eliminar/\nDisallow: /wiki/Wikipedia_Discuss%C3%A3o:P%C3%A1ginas_para_eliminar/\nDisallow: /wiki/Wikipedia_Discuss%C3%A3o%3AP%C3%A1ginas_para_eliminar/\n#\n# rowiki:\n# T14546\nDisallow: /wiki/Wikipedia:Pagini_de_%C5%9Fters\nDisallow: /wiki/Wikipedia%3APagini_de_%C5%9Fters\nDisallow: /wiki/Discu%C5%A3ie_Wikipedia:Pagini_de_%C5%9Fters\nDisallow: /wiki/Discu%C5%A3ie_Wikipedia%3APagini_de_%C5%9Fters\n#\n# ruwiki:\nDisallow: /wiki/%D0%A1%D0%BF%D0%B5%D1%86%D0%B8%D0%B0%D0%BB%D1%8C%D0%BD%D1%8B%D0%B5:Search\nDisallow: /wiki/%D0%A1%D0%BF%D0%B5%D1%86%D0%B8%D0%B0%D0%BB%D1%8C%D0%BD%D1%8B%D0%B5%3ASearch\n#\n# svwiki:\n# T12229\nDisallow: /wiki/Wikipedia%3ASidor_f%C3%B6reslagna_f%C3%B6r_radering\nDisallow: /wiki/Wikipedia:Sidor_f%C3%B6reslagna_f%C3%B6r_radering\nDisallow: /wiki/Wikipedia:Sidor_föreslagna_för_radering\nDisallow: /wiki/Användare\nDisallow: /wiki/Anv%C3%A4ndare\nDisallow: /wiki/Användardiskussion\nDisallow: /wiki/Anv%C3%A4ndardiskussion\nDisallow: /wiki/Wikipedia:Skyddade_sidnamn\nDisallow: /wiki/Wikipedia%3ASkyddade_sidnamn\n# T13291\nDisallow: /wiki/Wikipedia:Sidor_som_bör_raderas\nDisallow: /wiki/Wikipedia:Sidor_som_b%C3%B6r_raderas\nDisallow: /wiki/Wikipedia%3ASidor_som_b%C3%B6r_raderas\n#\n# zhwiki:\n# T7104\nDisallow: /wiki/Wikipedia:删除投票/侵权\nDisallow: /wiki/Wikipedia:%E5%88%A0%E9%99%A4%E6%8A%95%E7%A5%A8/%E4%BE%B5%E6%9D%83\nDisallow: /wiki/Wikipedia:删除投票和请求\nDisallow: /wiki/Wikipedia:%E5%88%A0%E9%99%A4%E6%8A%95%E7%A5%A8%E5%92%8C%E8%AF%B7%E6%B1%82\nDisallow: /wiki/Category:快速删除候选\nDisallow: /wiki/Category:%E5%BF%AB%E9%80%9F%E5%88%A0%E9%99%A4%E5%80%99%E9%80%89\nDisallow: /wiki/Category:维基百科需要翻译的文章\nDisallow: /wiki/Category:%E7%BB%B4%E5%9F%BA%E7%99%BE%E7%A7%91%E9%9C%80%E8%A6%81%E7%BF%BB%E8%AF%91%E7%9A%84%E6%96%87%E7%AB%A0\n#\n# sister projects\n#\n# enwikinews:\n# T7340\nDisallow: /wiki/Portal:Prepared_stories/\nDisallow: /wiki/Portal%3APrepared_stories/\n#\n# itwikinews\n# T11138\nDisallow: /wiki/Wikinotizie:Richieste_di_cancellazione\nDisallow: /wiki/Wikinotizie:Sospette_violazioni_di_copyright\nDisallow: /wiki/Categoria:Da_cancellare_subito\nDisallow: /wiki/Categoria:Da_cancellare_subito_per_violazione_integrale_copyright\nDisallow: /wiki/Wikinotizie:Storie_in_preparazione\n#\n# enwikiquote:\n# T17095\nDisallow: /wiki/Wikiquote:Votes_for_deletion/\nDisallow: /wiki/Wikiquote%3AVotes_for_deletion/\nDisallow: /wiki/Wikiquote_talk:Votes_for_deletion/\nDisallow: /wiki/Wikiquote_talk%3AVotes_for_deletion/\nDisallow: /wiki/Wikiquote:Votes_for_deletion_archive/\nDisallow: /wiki/Wikiquote%3AVotes_for_deletion_archive/\nDisallow: /wiki/Wikiquote_talk:Votes_for_deletion_archive/\nDisallow: /wiki/Wikiquote_talk%3AVotes_for_deletion_archive/\n#\n# enwikibooks\nDisallow: /wiki/Wikibooks:Votes_for_deletion\n#\n\n\nSitemap: https://en.wikipedia.org/w/rest.php/site/v1/sitemap/0\n\n#\n#\n#----------------------------------------------------------#\n#\n#\n#\n#\n# Localisable part of robots.txt for en.wikipedia.org\n#\n# Edit at https://en.wikipedia.org/w/index.php?title=MediaWiki:Robots.txt&action=edit\n# Don't add newlines here. All rules set here are active for every user-agent.\n#\n# Please check any changes using a syntax validator\n# Enter https://en.wikipedia.org/robots.txt as the URL to check.\n#\n# https://phabricator.wikimedia.org/T16075\nDisallow: /wiki/MediaWiki:Spam-blacklist\nDisallow: /wiki/MediaWiki%3ASpam-blacklist\nDisallow: /wiki/MediaWiki_talk:Spam-blacklist\nDisallow: /wiki/MediaWiki_talk%3ASpam-blacklist\nDisallow: /wiki/Wikipedia:WikiProject_Spam\nDisallow: /wiki/Wikipedia_talk:WikiProject_Spam\n#\n# Folks get annoyed when XfD discussions end up the number 1 google hit for\n# their name. \n# https://phabricator.wikimedia.org/T16075\nDisallow: /wiki/Wikipedia:Articles_for_deletion\nDisallow: /wiki/Wikipedia%3AArticles_for_deletion\nDisallow: /wiki/Wikipedia:Votes_for_deletion\nDisallow: /wiki/Wikipedia%3AVotes_for_deletion\nDisallow: /wiki/Wikipedia:Pages_for_deletion\nDisallow: /wiki/Wikipedia%3APages_for_deletion\nDisallow: /wiki/Wikipedia:Miscellany_for_deletion\nDisallow: /wiki/Wikipedia%3AMiscellany_for_deletion\nDisallow: /wiki/Wikipedia:Miscellaneous_deletion\nDisallow: /wiki/Wikipedia%3AMiscellaneous_deletion\nDisallow: /wiki/Wikipedia:Categories_for_discussion\nDisallow: /wiki/Wikipedia%3ACategories_for_discussion\nDisallow: /wiki/Wikipedia:Templates_for_deletion\nDisallow: /wiki/Wikipedia%3ATemplates_for_deletion\nDisallow: /wiki/Wikipedia:Redirects_for_discussion\nDisallow: /wiki/Wikipedia%3ARedirects_for_discussion\nDisallow: /wiki/Wikipedia:Deletion_review\nDisallow: /wiki/Wikipedia%3ADeletion_review\nDisallow: /wiki/Wikipedia:WikiProject_Deletion_sorting\nDisallow: /wiki/Wikipedia%3AWikiProject_Deletion_sorting\nDisallow: /wiki/Wikipedia:Files_for_deletion\nDisallow: /wiki/Wikipedia%3AFiles_for_deletion\nDisallow: /wiki/Wikipedia:Files_for_discussion\nDisallow: /wiki/Wikipedia%3AFiles_for_discussion\nDisallow: /wiki/Wikipedia:Possibly_unfree_files\nDisallow: /wiki/Wikipedia%3APossibly_unfree_files\n#\n# https://phabricator.wikimedia.org/T12288\nDisallow: /wiki/Wikipedia_talk:Articles_for_deletion\nDisallow: /wiki/Wikipedia_talk%3AArticles_for_deletion\nDisallow: /wiki/Wikipedia_talk:Votes_for_deletion\nDisallow: /wiki/Wikipedia_talk%3AVotes_for_deletion\nDisallow: /wiki/Wikipedia_talk:Pages_for_deletion\nDisallow: /wiki/Wikipedia_talk%3APages_for_deletion\nDisallow: /wiki/Wikipedia_talk:Miscellany_for_deletion\nDisallow: /wiki/Wikipedia_talk%3AMiscellany_for_deletion\nDisallow: /wiki/Wikipedia_talk:Miscellaneous_deletion\nDisallow: /wiki/Wikipedia_talk%3AMiscellaneous_deletion\nDisallow: /wiki/Wikipedia_talk:Templates_for_deletion\nDisallow: /wiki/Wikipedia_talk%3ATemplates_for_deletion\nDisallow: /wiki/Wikipedia_talk:Categories_for_discussion\nDisallow: /wiki/Wikipedia_talk%3ACategories_for_discussion\nDisallow: /wiki/Wikipedia_talk:Deletion_review\nDisallow: /wiki/Wikipedia_talk%3ADeletion_review\nDisallow: /wiki/Wikipedia_talk:WikiProject_Deletion_sorting\nDisallow: /wiki/Wikipedia_talk%3AWikiProject_Deletion_sorting\nDisallow: /wiki/Wikipedia_talk:Files_for_deletion\nDisallow: /wiki/Wikipedia_talk%3AFiles_for_deletion\nDisallow: /wiki/Wikipedia_talk:Files_for_discussion\nDisallow: /wiki/Wikipedia_talk%3AFiles_for_discussion\nDisallow: /wiki/Wikipedia_talk:Possibly_unfree_files\nDisallow: /wiki/Wikipedia_talk%3APossibly_unfree_files\n#\nDisallow: /wiki/Wikipedia:Copyright_problems\nDisallow: /wiki/Wikipedia%3ACopyright_problems\nDisallow: /wiki/Wikipedia_talk:Copyright_problems\nDisallow: /wiki/Wikipedia_talk%3ACopyright_problems\nDisallow: /wiki/Wikipedia:Suspected_copyright_violations\nDisallow: /wiki/Wikipedia%3ASuspected_copyright_violations\nDisallow: /wiki/Wikipedia_talk:Suspected_copyright_violations\nDisallow: /wiki/Wikipedia_talk%3ASuspected_copyright_violations\nDisallow: /wiki/Wikipedia:Contributor_copyright_investigations\nDisallow: /wiki/Wikipedia%3AContributor_copyright_investigations\nDisallow: /wiki/Wikipedia:Contributor_copyright_investigations\nDisallow: /wiki/Wikipedia%3AContributor_copyright_investigations\nDisallow: /wiki/Wikipedia_talk:Contributor_copyright_investigations\nDisallow: /wiki/Wikipedia_talk%3AContributor_copyright_investigations\nDisallow: /wiki/Wikipedia_talk:Contributor_copyright_investigations\nDisallow: /wiki/Wikipedia_talk%3AContributor_copyright_investigations\nDisallow: /wiki/Wikipedia:Protected_titles\nDisallow: /wiki/Wikipedia%3AProtected_titles\nDisallow: /wiki/Wikipedia_talk:Protected_titles\nDisallow: /wiki/Wikipedia_talk%3AProtected_titles\nDisallow: /wiki/Wikipedia:Articles_for_creation\nDisallow: /wiki/Wikipedia%3AArticles_for_creation\nDisallow: /wiki/Wikipedia_talk:Articles_for_creation\nDisallow: /wiki/Wikipedia_talk%3AArticles_for_creation\nDisallow: /wiki/Wikipedia_talk:Article_wizard\nDisallow: /wiki/Wikipedia_talk%3AArticle_wizard\n#\n# https://phabricator.wikimedia.org/T13261\nDisallow: /wiki/Wikipedia:Requests_for_arbitration\nDisallow: /wiki/Wikipedia%3ARequests_for_arbitration\nDisallow: /wiki/Wikipedia_talk:Requests_for_arbitration\nDisallow: /wiki/Wikipedia_talk%3ARequests_for_arbitration\nDisallow: /wiki/Wikipedia:Requests_for_comment\nDisallow: /wiki/Wikipedia%3ARequests_for_comment\nDisallow: /wiki/Wikipedia_talk:Requests_for_comment\nDisallow: /wiki/Wikipedia_talk%3ARequests_for_comment\nDisallow: /wiki/Wikipedia:Requests_for_adminship\nDisallow: /wiki/Wikipedia%3ARequests_for_adminship\nDisallow: /wiki/Wikipedia_talk:Requests_for_adminship\nDisallow: /wiki/Wikipedia_talk%3ARequests_for_adminship\n#\n# https://phabricator.wikimedia.org/T14111\nDisallow: /wiki/Wikipedia:Requests_for_checkuser\nDisallow: /wiki/Wikipedia%3ARequests_for_checkuser\nDisallow: /wiki/Wikipedia_talk:Requests_for_checkuser\nDisallow: /wiki/Wikipedia_talk%3ARequests_for_checkuser\n#\n# https://phabricator.wikimedia.org/T15398\nDisallow: /wiki/Wikipedia:WikiProject_Spam\nDisallow: /wiki/Wikipedia%3AWikiProject_Spam\n#\n# https://phabricator.wikimedia.org/T16793\nDisallow: /wiki/Wikipedia:Changing_username\nDisallow: /wiki/Wikipedia%3AChanging_username\nDisallow: /wiki/Wikipedia:Changing_username\nDisallow: /wiki/Wikipedia%3AChanging_username\nDisallow: /wiki/Wikipedia_talk:Changing_username\nDisallow: /wiki/Wikipedia_talk%3AChanging_username\nDisallow: /wiki/Wikipedia_talk:Changing_username\nDisallow: /wiki/Wikipedia_talk%3AChanging_username\n#\nDisallow: /wiki/Wikipedia:Administrators%27_noticeboard\nDisallow: /wiki/Wikipedia%3AAdministrators%27_noticeboard\nDisallow: /wiki/Wikipedia_talk:Administrators%27_noticeboard\nDisallow: /wiki/Wikipedia_talk%3AAdministrators%27_noticeboard\nDisallow: /wiki/Wikipedia:Community_sanction_noticeboard\nDisallow: /wiki/Wikipedia%3ACommunity_sanction_noticeboard\nDisallow: /wiki/Wikipedia_talk:Community_sanction_noticeboard\nDisallow: /wiki/Wikipedia_talk%3ACommunity_sanction_noticeboard\nDisallow: /wiki/Wikipedia:Bureaucrats%27_noticeboard\nDisallow: /wiki/Wikipedia%3ABureaucrats%27_noticeboard\nDisallow: /wiki/Wikipedia_talk:Bureaucrats%27_noticeboard\nDisallow: /wiki/Wikipedia_talk%3ABureaucrats%27_noticeboard\n#\nDisallow: /wiki/Wikipedia:Sockpuppet_investigations\nDisallow: /wiki/Wikipedia%3ASockpuppet_investigations\nDisallow: /wiki/Wikipedia_talk:Sockpuppet_investigations\nDisallow: /wiki/Wikipedia_talk%3ASockpuppet_investigations\n#\nDisallow: /wiki/Wikipedia:Neutral_point_of_view/Noticeboard\nDisallow: /wiki/Wikipedia%3ANeutral_point_of_view/Noticeboard\nDisallow: /wiki/Wikipedia_talk:Neutral_point_of_view/Noticeboard\nDisallow: /wiki/Wikipedia_talk%3ANeutral_point_of_view/Noticeboard\n#\nDisallow: /wiki/Wikipedia:No_original_research/noticeboard\nDisallow: /wiki/Wikipedia%3ANo_original_research/noticeboard\nDisallow: /wiki/Wikipedia_talk:No_original_research/noticeboard\nDisallow: /wiki/Wikipedia_talk%3ANo_original_research/noticeboard\n#\nDisallow: /wiki/Wikipedia:Fringe_theories/Noticeboard\nDisallow: /wiki/Wikipedia%3AFringe_theories/Noticeboard\nDisallow: /wiki/Wikipedia_talk:Fringe_theories/Noticeboard\nDisallow: /wiki/Wikipedia_talk%3AFringe_theories/Noticeboard\n#\nDisallow: /wiki/Wikipedia:Conflict_of_interest/Noticeboard\nDisallow: /wiki/Wikipedia%3AConflict_of_interest/Noticeboard\nDisallow: /wiki/Wikipedia_talk:Conflict_of_interest/Noticeboard\nDisallow: /wiki/Wikipedia_talk%3AConflict_of_interest/Noticeboard\n#\nDisallow: /wiki/Wikipedia:Long-term_abuse\nDisallow: /wiki/Wikipedia%3ALong-term_abuse\nDisallow: /wiki/Wikipedia_talk:Long-term_abuse\nDisallow: /wiki/Wikipedia_talk%3ALong-term_abuse\nDisallow: /wiki/Wikipedia:Long_term_abuse\nDisallow: /wiki/Wikipedia%3ALong_term_abuse\nDisallow: /wiki/Wikipedia_talk:Long_term_abuse\nDisallow: /wiki/Wikipedia_talk%3ALong_term_abuse\n#\nDisallow: /wiki/Wikipedia:Wikiquette_assistance\nDisallow: /wiki/Wikipedia%3AWikiquette_assistance\n#\nDisallow: /wiki/Wikipedia:Abuse_reports\nDisallow: /wiki/Wikipedia%3AAbuse_reports\nDisallow: /wiki/Wikipedia_talk:Abuse_reports\nDisallow: /wiki/Wikipedia_talk%3AAbuse_reports\nDisallow: /wiki/Wikipedia:Abuse_response\nDisallow: /wiki/Wikipedia%3AAbuse_response\nDisallow: /wiki/Wikipedia_talk:Abuse_response\nDisallow: /wiki/Wikipedia_talk%3AAbuse_response\n#\nDisallow: /wiki/Wikipedia:Reliable_sources/Noticeboard\nDisallow: /wiki/Wikipedia%3AReliable_sources/Noticeboard\nDisallow: /wiki/Wikipedia_talk:Reliable_sources/Noticeboard\nDisallow: /wiki/Wikipedia_talk%3AReliable_sources/Noticeboard\n#\nDisallow: /wiki/Wikipedia:Suspected_sock_puppets\nDisallow: /wiki/Wikipedia%3ASuspected_sock_puppets\nDisallow: /wiki/Wikipedia_talk:Suspected_sock_puppets\nDisallow: /wiki/Wikipedia_talk%3ASuspected_sock_puppets\n#\nDisallow: /wiki/Wikipedia:Biographies_of_living_persons/Noticeboard\nDisallow: /wiki/Wikipedia%3ABiographies_of_living_persons/Noticeboard\nDisallow: /wiki/Wikipedia_talk:Biographies_of_living_persons/Noticeboard\nDisallow: /wiki/Wikipedia_talk%3ABiographies_of_living_persons/Noticeboard\nDisallow: /wiki/Wikipedia:Biographies_of_living_persons%2FNoticeboard\nDisallow: /wiki/Wikipedia%3ABiographies_of_living_persons%2FNoticeboard\nDisallow: /wiki/Wikipedia_talk:Biographies_of_living_persons%2FNoticeboard\nDisallow: /wiki/Wikipedia_talk%3ABiographies_of_living_persons%2FNoticeboard\n#\nDisallow: /wiki/Wikipedia:Content_noticeboard\nDisallow: /wiki/Wikipedia%3AContent_noticeboard\nDisallow: /wiki/Wikipedia_talk:Content_noticeboard\nDisallow: /wiki/Wikipedia_talk%3AContent_noticeboard\n#\nDisallow: /wiki/Template:Editnotices\nDisallow: /wiki/Template%3AEditnotices\n#\nDisallow: /wiki/Wikipedia:Arbitration\nDisallow: /wiki/Wikipedia%3AArbitration\nDisallow: /wiki/Wikipedia_talk:Arbitration\nDisallow: /wiki/Wikipedia_talk%3AArbitration\n#\nDisallow: /wiki/Wikipedia:Arbitration_Committee\nDisallow: /wiki/Wikipedia%3AArbitration_Committee\nDisallow: /wiki/Wikipedia_talk:Arbitration_Committee\nDisallow: /wiki/Wikipedia_talk%3AArbitration_Committee\n#\nDisallow: /wiki/Wikipedia:Arbitration_Committee_Elections\nDisallow: /wiki/Wikipedia%3AArbitration_Committee_Elections\nDisallow: /wiki/Wikipedia_talk:Arbitration_Committee_Elections\nDisallow: /wiki/Wikipedia_talk%3AArbitration_Committee_Elections\n#\nDisallow: /wiki/Wikipedia:Mediation_Committee\nDisallow: /wiki/Wikipedia%3AMediation_Committee\nDisallow: /wiki/Wikipedia_talk:Mediation_Committee\nDisallow: /wiki/Wikipedia_talk%3AMediation_Committee\n#\nDisallow: /wiki/Wikipedia:Mediation_Cabal/Cases\nDisallow: /wiki/Wikipedia%3AMediation_Cabal/Cases\n#\nDisallow: /wiki/Wikipedia:Requests_for_bureaucratship\nDisallow: /wiki/Wikipedia%3ARequests_for_bureaucratship\nDisallow: /wiki/Wikipedia_talk:Requests_for_bureaucratship\nDisallow: /wiki/Wikipedia_talk%3ARequests_for_bureaucratship\n#\nDisallow: /wiki/Wikipedia:Administrator_review\nDisallow: /wiki/Wikipedia%3AAdministrator_review\nDisallow: /wiki/Wikipedia_talk:Administrator_review\nDisallow: /wiki/Wikipedia_talk%3AAdministrator_review\n#\nDisallow: /wiki/Wikipedia:Editor_review\nDisallow: /wiki/Wikipedia%3AEditor_review\nDisallow: /wiki/Wikipedia_talk:Editor_review\nDisallow: /wiki/Wikipedia_talk%3AEditor_review\n#\nDisallow: /wiki/Wikipedia:Article_Incubator\nDisallow: /wiki/Wikipedia%3AArticle_Incubator\nDisallow: /wiki/Wikipedia_talk:Article_Incubator\nDisallow: /wiki/Wikipedia_talk%3AArticle_Incubator\n#\nDisallow: /wiki/Category:Noindexed_pages\nDisallow: /wiki/Category%3ANoindexed_pages\n#\n# User sandboxes for modules and Template Styles are placed in these subpages for testing\n#\nDisallow: /wiki/Module:Sandbox\nDisallow: /wiki/Module%3ASandbox\nDisallow: /wiki/Template:TemplateStyles_sandbox\nDisallow: /wiki/Template%3ATemplateStyles_sandbox\n# Adminship and arbitration discussions are blocked from search engines, so blocking newer adminship processes (as of 2024) as well\nDisallow: /wiki/Wikipedia:Administrator_recall\nDisallow: /wiki/Wikipedia%3AAdministrator_recall\nDisallow: /wiki/Wikipedia_talk:Administrator_recall\nDisallow: /wiki/Wikipedia_talk%3AAdministrator_recall\nDisallow: /wiki/Wikipedia:Administrator_elections\nDisallow: /wiki/Wikipedia%3AAdministrator_elections\nDisallow: /wiki/Wikipedia_talk:Administrator_elections\nDisallow: /wiki/Wikipedia_talk%3AAdministrator_elections","agents":{"GPTBot":true,"OAI-SearchBot":true,"ChatGPT-User":true,"ClaudeBot":true,"Claude-SearchBot":true,"PerplexityBot":true,"Google-Extended":true,"Applebot-Extended":true,"CCBot":true,"Bytespider":true,"Meta-ExternalAgent":true,"Amazonbot":true}},"sitemap":{"status":403,"content_type":"text/html","bytes":2104,"sha256":"1936f1d6a7302bc14ee213fd705b4f5e3e45af3fe772a901aecb323dc6a3a40c","final_host":"en.wikipedia.org","ms":27,"url":"https://en.wikipedia.org/w/rest.php/site/v1/sitemap/0","declared":true,"xml":false},"files":{"llms_txt":{"status":404,"content_type":"text/html","bytes":1868,"sha256":"76563b4eeb07e461f97c018d6f0744fdaf2b0af79ebd90410e0452733d164bdf","final_host":"en.wikipedia.org","ms":139,"path":"/llms.txt","html":true,"json":null,"present":false},"agents_md":{"status":404,"content_type":"text/html","bytes":1871,"sha256":"f258eadc2d42d5a154d77a0dbf077f2c20b3746429256f14411e7f96202b6cde","final_host":"en.wikipedia.org","ms":328,"path":"/agents.md","html":true,"json":null,"present":false},"agent_skills":{"status":404,"content_type":"text/html","bytes":1949,"sha256":"1ee573c039ee6c0ada299e677e2d54a9c806bbf443416f7e61afd1389c0bce39","final_host":"en.wikipedia.org","ms":347,"path":"/.well-known/agent-skills/index.json","html":true,"json":null,"present":false},"mcp_card":{"status":404,"content_type":"text/html","bytes":1940,"sha256":"1c4ba99d1ebef86ec79ac094e3c465236974c6844bab4955f3ed0949ffac9384","final_host":"en.wikipedia.org","ms":250,"path":"/.well-known/mcp/server-card.json","html":true,"json":null,"present":false},"api_catalog":{"status":404,"content_type":"text/html","bytes":1913,"sha256":"c0af58a441039877235c3c8470adc120371f4ba295261ceff33d4bbbdc43446f","final_host":"en.wikipedia.org","ms":344,"path":"/.well-known/api-catalog","html":true,"json":null,"present":false},"a2a_card":{"status":404,"content_type":"text/html","bytes":1925,"sha256":"d013b23e4a84bbbb9d35ce0c332ff1a101052a5965593f0786693a0e6c0b24d3","final_host":"en.wikipedia.org","ms":229,"path":"/.well-known/agent-card.json","html":true,"json":null,"present":false}},"rings":{"v":1,"candidates":[],"mcp":[],"a2a":null,"webmcp":{"model_context":false,"bridge":false,"mcp_json":false,"link_rel":false,"any":false},"handshakes":0,"get_path":{"status":404,"content_type":"text/html","ms":154,"error":null},"get_host":{"status":530,"content_type":"text/plain","ms":84,"error":null}},"derived":{"reachable":true,"robots_served":true,"robots_sitemap":true,"content_signal":false,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":[],"blocks_any_ai":false,"files":{"llms_txt":false,"agents_md":false,"agent_skills":false,"mcp_card":false,"api_catalog":false,"a2a_card":false},"any_capability":false,"ai_refused":["claudebot"],"ai_thin":[],"rings":{"mcp_declared":false,"mcp_live":false,"mcp_auth":false,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":2,"fh":"www.wikipedia.org","fp_other_host":[],"fp":"9f62870a873f37ff","fp_bits":{"gptbot":0,"oaisearch":0,"perplexity":0}},"derive_v":3,"frame":"top1k","shard":5},{"v":1,"kind":"crawlcheck-census-record","run":14,"domain":"appsflyersdk.com","rank":31,"measured_at":"2026-10-15T05:33:03.639Z","home":{"browser":{"status":403,"content_type":"text/xml","bytes":146,"sha256":"95769eb326e23b8cd1b7a9b82d884a713a9724f49b2794ac5b967d8cd706de83","final_host":"appsflyersdk.com","ms":129,"identity":"Mobile browser","html":false},"gptbot":{"status":403,"content_type":"text/xml","bytes":146,"sha256":"95769eb326e23b8cd1b7a9b82d884a713a9724f49b2794ac5b967d8cd706de83","final_host":"appsflyersdk.com","ms":71,"identity":"GPTBot","html":false},"claudebot":{"status":403,"content_type":"text/xml","bytes":146,"sha256":"95769eb326e23b8cd1b7a9b82d884a713a9724f49b2794ac5b967d8cd706de83","final_host":"appsflyersdk.com","ms":11,"identity":"ClaudeBot","html":false},"oaisearch":{"status":403,"content_type":"text/xml","bytes":146,"sha256":"95769eb326e23b8cd1b7a9b82d884a713a9724f49b2794ac5b967d8cd706de83","final_host":"appsflyersdk.com","ms":5,"identity":"OAI-SearchBot","html":false},"perplexity":{"status":403,"content_type":"text/xml","bytes":146,"sha256":"95769eb326e23b8cd1b7a9b82d884a713a9724f49b2794ac5b967d8cd706de83","final_host":"appsflyersdk.com","ms":6,"identity":"PerplexityBot","html":false}},"robots":{"status":403,"content_type":"text/xml","bytes":146,"sha256":"95769eb326e23b8cd1b7a9b82d884a713a9724f49b2794ac5b967d8cd706de83","final_host":"appsflyersdk.com","ms":72,"html":false,"text":null,"agents":null},"sitemap":null,"files":null,"derived":{"reachable":false,"robots_served":false,"robots_sitemap":false,"content_signal":false,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":[],"blocks_any_ai":false,"files":{"llms_txt":false,"agents_md":false,"agent_skills":false,"mcp_card":false,"api_catalog":false,"a2a_card":false},"any_capability":false,"ai_refused":[],"ai_thin":[],"rings":{"mcp_declared":false,"mcp_live":false,"mcp_auth":false,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":2,"fh":"appsflyersdk.com","fp_other_host":[]},"derive_v":3,"frame":"top1k","shard":6},{"v":1,"kind":"crawlcheck-census-record","run":14,"domain":"bing.com","rank":32,"measured_at":"2026-10-15T05:34:03.814Z","home":{"browser":{"status":200,"content_type":"text/html","bytes":51782,"sha256":"c38d0990b3bcb8d685ad2e99a7e80644db4192feef572508ef8d89b635125f16","final_host":"www.bing.com","ms":184,"identity":"Mobile browser","html":true},"gptbot":{"status":200,"content_type":"text/html","bytes":76989,"sha256":"ab911983789b3838f58f95c7286f6e45ca86aefebf96713283755b1322e60669","final_host":"www.bing.com","ms":134,"identity":"GPTBot","html":true},"claudebot":{"status":200,"content_type":"text/html","bytes":77304,"sha256":"f6de31a179043ff39c9e4f71bb28f4792cb0d6f3eeea70d06d9d7fae90fb72fc","final_host":"www.bing.com","ms":133,"identity":"ClaudeBot","html":true},"oaisearch":{"status":200,"content_type":"text/html","bytes":77590,"sha256":"86380964f271c3ea9c61d9b8f67cf79c353d34a59db0759f7f5894f1453106ca","final_host":"www.bing.com","ms":151,"identity":"OAI-SearchBot","html":true},"perplexity":{"status":200,"content_type":"text/html","bytes":77197,"sha256":"83334f0ee463c247f7e22192996a2daac9d683f0c04c7e153bd1828c13831b64","final_host":"www.bing.com","ms":155,"identity":"PerplexityBot","html":true}},"robots":{"status":200,"content_type":"text/plain","bytes":2676,"sha256":"40d2da70c3b443225d5b01cc7b05117944980347637cc95ab285c6e4f5548235","final_host":"www.bing.com","ms":236,"html":false,"text":"User-agent: msnbot-media\r\nDisallow: /\r\nAllow: /th?\r\n\r\nUser-agent: Twitterbot\r\nDisallow:\r\n\r\nUser-agent: *\r\nDisallow: /academic/profile\r\nDisallow: /academic/search\r\nDisallow: /account/\r\nDisallow: /aclick\r\nDisallow: /alink\r\nDisallow: /amp/\r\nAllow: /api/maps/\r\nDisallow: /api/\r\nDisallow: /bfp/search\r\nDisallow: /bing-site-safety\r\nDisallow: /blogs/search/\r\nDisallow: /ck/\r\nDisallow: /copilotsearch?q=\r\nDisallow: /cr$\r\nDisallow: /cr?\r\nDisallow: /entities/search\r\nDisallow: /entityexplore$\r\nDisallow: /entityexplore?\r\nDisallow: /fd/\r\nDisallow: /fun/api/\r\nDisallow: /fun/g/\r\nDisallow: /history\r\nDisallow: /hotel/reviews?\r\nDisallow: /hotels/search\r\nDisallow: /HpImageArchive.aspx\r\nDisallow: /hpm\r\nDisallow: /hpmob\r\nDisallow: /images/async/?\r\nDisallow: /images/async?\r\nDisallow: /images/hosted/?\r\nDisallow: /images/hosted/search?\r\nDisallow: /images/hosted?\r\nDisallow: /images/search%3F\r\nDisallow: /images/search%5C\r\nDisallow: /images/search/?\r\nDisallow: /images/search?\r\nDisallow: /images/searchbyimage\r\nDisallow: /images?\r\nDisallow: /inboxcommerce\r\nDisallow: /local\r\nDisallow: /maps/adsendpoint\r\nDisallow: /merchant/reviews?\r\nDisallow: /notifications/\r\nDisallow: /offers/proxy/dealsserver/api/log\r\nDisallow: /offers/proxy/dealsserver/buy\r\nDisallow: /ping\r\nDisallow: /product/reviews?\r\nDisallow: /profile/history?\r\nDisallow: /proFile/history?\r\nDisallow: /Proxy.ashx\r\nDisallow: /results\r\nDisallow: /rewardsapp/\r\nDisallow: /search\r\nDisallow: /Search\r\nDisallow: /settings\r\nDisallow: /shenghuo\r\nAllow: /shop$\r\nAllow: /shop/\r\nDisallow: /shop/collections\r\nDisallow: /shop/productpage\r\nDisallow: /shop/topics\r\nDisallow: /shop?*\r\nDisallow: /social/search?\r\nDisallow: /spbasic\r\nDisallow: /spresults\r\nDisallow: /static/\r\nDisallow: /th$\r\nDisallow: /th?\r\nDisallow: /translator/?ref=\r\nDisallow: /travel/api\r\nDisallow: /travel/css\r\nDisallow: /travel/flight/flightSearch\r\nDisallow: /travel/flight/flightSearchAction\r\nDisallow: /travel/flight/search/?\r\nDisallow: /travel/flight/search?\r\nDisallow: /travel/flights\r\nDisallow: /travel/flight-search\r\nDisallow: /travel/hotel/hotelMiniSearchRequest\r\nDisallow: /travel/hotel/hotelSearch\r\nDisallow: /travel/hotels/checkout\r\nDisallow: /travel/hotels/detail\r\nDisallow: /travel/hotels/search/?\r\nDisallow: /travel/hotels/search?\r\nDisallow: /travel/hotel-search\r\nDisallow: /travel/scripts\r\nDisallow: /travel/secure\r\nDisallow: /url\r\nDisallow: /videos/browsing\r\nDisallow: /videos/explore\r\nDisallow: /videos/favorites\r\nDisallow: /videos/feed\r\nDisallow: /videos/music\r\nDisallow: /videos/trending\r\nDisallow: /widget/cr\r\nDisallow: /widget/entity/search/?\r\nDisallow: /widget/render\r\nDisallow: /widget/snapshot\r\nDisallow: /work\r\n\r\nSitemap: https://www.bing.com/sitemap.xml","agents":{"GPTBot":true,"OAI-SearchBot":true,"ChatGPT-User":true,"ClaudeBot":true,"Claude-SearchBot":true,"PerplexityBot":true,"Google-Extended":true,"Applebot-Extended":true,"CCBot":true,"Bytespider":true,"Meta-ExternalAgent":true,"Amazonbot":true}},"sitemap":{"status":200,"content_type":"text/xml","bytes":1187,"sha256":"b01f48436b21eb3993fa19ff9244e963061c9be6b2fa159d18de0d8995334c13","final_host":"www.bing.com","ms":75,"url":"https://www.bing.com/sitemap.xml","declared":true,"xml":true},"files":{"llms_txt":{"status":404,"content_type":"text/html","bytes":0,"sha256":"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855","final_host":"www.bing.com","ms":128,"path":"/llms.txt","html":true,"json":null,"present":false},"agents_md":{"status":404,"content_type":"text/html","bytes":77005,"sha256":"f0c2131ac6f7958021d705130bd8e042281637bc6ea019156aff87bc60d6fc0a","final_host":"www.bing.com","ms":129,"path":"/agents.md","html":true,"json":null,"present":false},"agent_skills":{"status":404,"content_type":"text/html","bytes":0,"sha256":"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855","final_host":"www.bing.com","ms":121,"path":"/.well-known/agent-skills/index.json","html":true,"json":null,"present":false},"mcp_card":{"status":404,"content_type":"text/html","bytes":0,"sha256":"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855","final_host":"www.bing.com","ms":175,"path":"/.well-known/mcp/server-card.json","html":true,"json":null,"present":false},"api_catalog":{"status":404,"content_type":"text/html","bytes":76420,"sha256":"95ca646e8c9b20a5564e33cd652086ddf4486d0c4b118edf1b0ae0f175d6590a","final_host":"www.bing.com","ms":146,"path":"/.well-known/api-catalog","html":true,"json":null,"present":false},"a2a_card":{"status":404,"content_type":"text/html","bytes":0,"sha256":"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855","final_host":"www.bing.com","ms":164,"path":"/.well-known/agent-card.json","html":true,"json":null,"present":false}},"rings":{"v":1,"candidates":[],"mcp":[],"a2a":null,"webmcp":{"model_context":false,"bridge":false,"mcp_json":false,"link_rel":false,"any":false},"handshakes":0,"get_path":{"status":404,"content_type":"text/html","ms":143,"error":null},"get_host":{"status":200,"content_type":"text/html","ms":230,"error":null}},"derived":{"reachable":true,"robots_served":true,"robots_sitemap":true,"content_signal":false,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":[],"blocks_any_ai":false,"files":{"llms_txt":false,"agents_md":false,"agent_skills":false,"mcp_card":false,"api_catalog":false,"a2a_card":false},"any_capability":false,"ai_refused":[],"ai_thin":[],"rings":{"mcp_declared":false,"mcp_live":false,"mcp_auth":false,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":2,"fh":"www.bing.com","fp_other_host":[]},"derive_v":3,"frame":"top1k","shard":7},{"v":1,"kind":"crawlcheck-census-record","run":14,"domain":"whatsapp.net","rank":33,"measured_at":"2026-10-15T05:26:03.595Z","home":{"browser":{"status":400,"content_type":"text/html","bytes":6278,"sha256":"e2b36c87740e40f73bf81489877cac0e606c6cd83a251022c164c6d744080a26","final_host":"www.whatsapp.com","ms":280,"identity":"Mobile browser","html":true},"gptbot":{"status":200,"content_type":"text/html","bytes":274613,"sha256":"c790dc34b089db50be049f195086c473de9a2c2fadb6a21c7c340edb42341678","final_host":"www.whatsapp.com","ms":477,"identity":"GPTBot","html":true,"fp":"031ccbb7a3071b56"},"claudebot":{"status":200,"content_type":"text/html","bytes":274562,"sha256":"fdd24864f6f80c766b932d2bffce084d115e89d433c428f818f7de66389da6f6","final_host":"www.whatsapp.com","ms":561,"identity":"ClaudeBot","html":true,"fp":"031ccbb7a3071b56"},"oaisearch":{"status":200,"content_type":"text/html","bytes":274622,"sha256":"c34f9d5d86d574fe560a5fe16415b9c7efa9d7c0771182b957e97e1474ba78dc","final_host":"www.whatsapp.com","ms":478,"identity":"OAI-SearchBot","html":true,"fp":"031ccbb7a3071b56"},"perplexity":{"status":200,"content_type":"text/html","bytes":291851,"sha256":"73f3cc659f4006a47d4e332e772bd824d09545105cb3edb8dd2f47808d00c96e","final_host":"www.whatsapp.com","ms":1016,"identity":"PerplexityBot","html":true,"fp":"011ccab7a90f4116"}},"robots":{"status":200,"content_type":"text/plain","bytes":1180,"sha256":"b0dc29a03f811635b16c416f0c890b4acad4112533b78feed9bdb13de7156cf9","final_host":"www.whatsapp.com","ms":595,"html":false,"text":"# Notice: Collection of data on Facebook through automated means is\n# prohibited unless you have express written permission from Facebook\n# and may only be conducted for the limited purpose contained in said\n# permission.\n# See: http://www.facebook.com/apps/site_scraping_tos_terms.php\n\nUser-agent: Amazonbot\nDisallow: /\n\nUser-agent: Applebot-Extended\nDisallow: /\n\nUser-agent: Brightbot\nDisallow: /\n\nUser-agent: ClaudeBot\nDisallow: /\n\nUser-agent: Google-Extended\nDisallow: /\n\nUser-agent: GPTBot\nDisallow: /\n\nUser-agent: PerplexityBot\nDisallow: /\n\nUser-agent: PetalBot\nDisallow: /\n\nUser-agent: Scrapy\nDisallow: /\n\nUser-agent: uptimerobot\nDisallow: /\n\nUser-agent: viberbot\nDisallow: /\n\nUser-agent: YaK\nDisallow: /\n\nUser-agent: Yandex\nDisallow: /\n\nUser-agent: Yeti\nDisallow: /\n\nUser-agent: facebookexternalhit\nAllow: *\n\nUser-agent: meta-externalads\nAllow: *\n\nUser-agent: *\nDisallow: /*cursor=\nDisallow: /*fb_comment_id=\nDisallow: /ajax/\nDisallow: /tealium/\nDisallow: /intern/\nDisallow: /internal/\nDisallow: /login/\nDisallow: /oidc/callback/\nDisallow: /*.php\nDisallow: /android/third_party\nDisallow: /settings\n\nSitemap: https://www.whatsapp.com/sitemap/www_whatsapp_com_sitemap.xml.gz","agents":{"GPTBot":false,"OAI-SearchBot":true,"ChatGPT-User":true,"ClaudeBot":false,"Claude-SearchBot":true,"PerplexityBot":false,"Google-Extended":false,"Applebot-Extended":false,"CCBot":true,"Bytespider":true,"Meta-ExternalAgent":true,"Amazonbot":false}},"sitemap":null,"files":null,"derived":{"reachable":false,"robots_served":false,"robots_sitemap":false,"content_signal":false,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":[],"blocks_any_ai":false,"files":{"llms_txt":false,"agents_md":false,"agent_skills":false,"mcp_card":false,"api_catalog":false,"a2a_card":false},"any_capability":false,"ai_refused":[],"ai_thin":[],"rings":{"mcp_declared":false,"mcp_live":false,"mcp_auth":false,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":2,"fh":"www.whatsapp.com","fp_other_host":[]},"derive_v":3,"frame":"top1k","shard":0},{"v":1,"kind":"crawlcheck-census-record","run":14,"domain":"apple-dns.net","rank":34,"measured_at":"2026-10-15T05:27:03.729Z","home":{"browser":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"apple-dns.net","ms":77,"identity":"Mobile browser","html":false},"gptbot":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"apple-dns.net","ms":67,"identity":"GPTBot","html":false},"claudebot":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"apple-dns.net","ms":2,"identity":"ClaudeBot","html":false},"oaisearch":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"apple-dns.net","ms":2,"identity":"OAI-SearchBot","html":false},"perplexity":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"apple-dns.net","ms":1,"identity":"PerplexityBot","html":false}},"robots":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"apple-dns.net","ms":1,"html":false,"text":null,"agents":null},"sitemap":null,"files":null,"derived":{"reachable":false,"robots_served":false,"robots_sitemap":false,"content_signal":false,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":[],"blocks_any_ai":false,"files":{"llms_txt":false,"agents_md":false,"agent_skills":false,"mcp_card":false,"api_catalog":false,"a2a_card":false},"any_capability":false,"ai_refused":[],"ai_thin":[],"rings":{"mcp_declared":false,"mcp_live":false,"mcp_auth":false,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":2,"fh":"apple-dns.net","fp_other_host":[]},"derive_v":3,"frame":"top1k","shard":1},{"v":1,"kind":"crawlcheck-census-record","run":14,"domain":"aaplimg.com","rank":35,"measured_at":"2026-10-15T05:28:03.923Z","home":{"browser":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"aaplimg.com","ms":32,"identity":"Mobile browser","html":false},"gptbot":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"aaplimg.com","ms":30,"identity":"GPTBot","html":false},"claudebot":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"aaplimg.com","ms":28,"identity":"ClaudeBot","html":false},"oaisearch":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"aaplimg.com","ms":29,"identity":"OAI-SearchBot","html":false},"perplexity":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"aaplimg.com","ms":20,"identity":"PerplexityBot","html":false}},"robots":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"aaplimg.com","ms":3,"html":false,"text":null,"agents":null},"sitemap":null,"files":null,"derived":{"reachable":false,"robots_served":false,"robots_sitemap":false,"content_signal":false,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":[],"blocks_any_ai":false,"files":{"llms_txt":false,"agents_md":false,"agent_skills":false,"mcp_card":false,"api_catalog":false,"a2a_card":false},"any_capability":false,"ai_refused":[],"ai_thin":[],"rings":{"mcp_declared":false,"mcp_live":false,"mcp_auth":false,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":2,"fh":"aaplimg.com","fp_other_host":[]},"derive_v":3,"frame":"top1k","shard":2},{"v":1,"kind":"crawlcheck-census-record","run":14,"domain":"fastly.net","rank":36,"measured_at":"2026-10-15T05:29:03.596Z","home":{"browser":{"status":200,"content_type":"text/html","bytes":224032,"sha256":"cc2eae645362f230509335c87794d6173dbe1b252f052d7ade2b014998ee56e2","final_host":"www.fastly.com","ms":1629,"identity":"Mobile browser","html":true,"fp":"df694c8900b1be79"},"gptbot":{"status":200,"content_type":"text/html","bytes":224032,"sha256":"cc2eae645362f230509335c87794d6173dbe1b252f052d7ade2b014998ee56e2","final_host":"www.fastly.com","ms":1051,"identity":"GPTBot","html":true,"fp":"df694c8900b1be79"},"claudebot":{"status":200,"content_type":"text/html","bytes":224032,"sha256":"cc2eae645362f230509335c87794d6173dbe1b252f052d7ade2b014998ee56e2","final_host":"www.fastly.com","ms":129,"identity":"ClaudeBot","html":true,"fp":"df694c8900b1be79"},"oaisearch":{"status":200,"content_type":"text/html","bytes":224032,"sha256":"cc2eae645362f230509335c87794d6173dbe1b252f052d7ade2b014998ee56e2","final_host":"www.fastly.com","ms":97,"identity":"OAI-SearchBot","html":true,"fp":"df694c8900b1be79"},"perplexity":{"status":200,"content_type":"text/html","bytes":224032,"sha256":"cc2eae645362f230509335c87794d6173dbe1b252f052d7ade2b014998ee56e2","final_host":"www.fastly.com","ms":134,"identity":"PerplexityBot","html":true,"fp":"df694c8900b1be79"}},"robots":{"status":200,"content_type":"text/plain","bytes":398,"sha256":"23afd0d2868ab27736e15dd91bfa2b061fff83da62d60d131f335d21311dc6f3","final_host":"www.fastly.com","ms":164,"html":false,"text":"User-agent: *\nDisallow: /ja/*\nDisallow: /labs/*\nDisallow: /production-2019/*\nDisallow: /cassets/*\nDisallow: /resources?*\nDisallow: /altitude/2020\nDisallow: /astro-header/*\nDisallow: /*/astro-header/*\nDisallow: /feature-o11y/*\nDisallow: /*/feature-o11y/*\nDisallow: /learning/fastly-research-methodology-glossary\nSitemap: https://www.fastly.com/sitemap/sitemap-index.xml\nHost: https://www.fastly.com\n","agents":{"GPTBot":true,"OAI-SearchBot":true,"ChatGPT-User":true,"ClaudeBot":true,"Claude-SearchBot":true,"PerplexityBot":true,"Google-Extended":true,"Applebot-Extended":true,"CCBot":true,"Bytespider":true,"Meta-ExternalAgent":true,"Amazonbot":true}},"sitemap":{"status":200,"content_type":"application/xml","bytes":380,"sha256":"d4951d59091004f4b2646cfdce10fcca2264b3a7a4c3e6737af619655d916450","final_host":"www.fastly.com","ms":160,"url":"https://www.fastly.com/sitemap/sitemap-index.xml","declared":true,"xml":true},"files":{"llms_txt":{"status":200,"content_type":"text/plain","bytes":16618,"sha256":"7d44aa6ffeddff002cd037eae424224165bc5c5c669c58e9c2e7497d58e2e028","final_host":"www.fastly.com","ms":83,"path":"/llms.txt","html":false,"json":null,"present":true},"agents_md":{"status":404,"content_type":"text/html","bytes":108558,"sha256":"32a50dd1d502060a9d7764ca16ee265b2d6ba40e2e859109ba37c92edef20801","final_host":"www.fastly.com","ms":675,"path":"/agents.md","html":true,"json":null,"present":false},"agent_skills":{"status":404,"content_type":"text/html","bytes":108558,"sha256":"32a50dd1d502060a9d7764ca16ee265b2d6ba40e2e859109ba37c92edef20801","final_host":"www.fastly.com","ms":624,"path":"/.well-known/agent-skills/index.json","html":true,"json":null,"present":false},"mcp_card":{"status":404,"content_type":"text/html","bytes":108558,"sha256":"32a50dd1d502060a9d7764ca16ee265b2d6ba40e2e859109ba37c92edef20801","final_host":"www.fastly.com","ms":682,"path":"/.well-known/mcp/server-card.json","html":true,"json":null,"present":false},"api_catalog":{"status":404,"content_type":"text/html","bytes":108558,"sha256":"32a50dd1d502060a9d7764ca16ee265b2d6ba40e2e859109ba37c92edef20801","final_host":"www.fastly.com","ms":655,"path":"/.well-known/api-catalog","html":true,"json":null,"present":false},"a2a_card":{"status":404,"content_type":"text/html","bytes":108558,"sha256":"32a50dd1d502060a9d7764ca16ee265b2d6ba40e2e859109ba37c92edef20801","final_host":"www.fastly.com","ms":955,"path":"/.well-known/agent-card.json","html":true,"json":null,"present":false}},"rings":{"v":1,"candidates":[],"mcp":[],"a2a":null,"webmcp":{"model_context":false,"bridge":false,"mcp_json":false,"link_rel":false,"any":false},"handshakes":0,"get_path":{"status":404,"content_type":"text/html","ms":686,"error":null},"get_host":{"status":530,"content_type":"text/plain","ms":49,"error":null}},"derived":{"reachable":true,"robots_served":true,"robots_sitemap":true,"content_signal":false,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":[],"blocks_any_ai":false,"files":{"llms_txt":true,"agents_md":false,"agent_skills":false,"mcp_card":false,"api_catalog":false,"a2a_card":false},"any_capability":false,"ai_refused":[],"ai_thin":[],"rings":{"mcp_declared":false,"mcp_live":false,"mcp_auth":false,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":2,"fh":"www.fastly.com","fp_other_host":[],"fp":"df694c8900b1be79","fp_bits":{"gptbot":0,"claudebot":0,"oaisearch":0,"perplexity":0}},"derive_v":3,"frame":"top1k","shard":3},{"v":1,"kind":"crawlcheck-census-record","run":14,"domain":"googleusercontent.com","rank":37,"measured_at":"2026-10-15T05:30:03.842Z","home":{"browser":{"status":404,"content_type":"text/html","bytes":1561,"sha256":"23926e9185d8d43c02807a838ffb373cc1977726094a4e46807c66ada9dd7660","final_host":"googleusercontent.com","ms":125,"identity":"Mobile browser","html":true},"gptbot":{"status":404,"content_type":"text/html","bytes":1561,"sha256":"23926e9185d8d43c02807a838ffb373cc1977726094a4e46807c66ada9dd7660","final_host":"googleusercontent.com","ms":91,"identity":"GPTBot","html":true},"claudebot":{"status":404,"content_type":"text/html","bytes":1561,"sha256":"23926e9185d8d43c02807a838ffb373cc1977726094a4e46807c66ada9dd7660","final_host":"googleusercontent.com","ms":89,"identity":"ClaudeBot","html":true},"oaisearch":{"status":404,"content_type":"text/html","bytes":1561,"sha256":"23926e9185d8d43c02807a838ffb373cc1977726094a4e46807c66ada9dd7660","final_host":"googleusercontent.com","ms":43,"identity":"OAI-SearchBot","html":true},"perplexity":{"status":404,"content_type":"text/html","bytes":1561,"sha256":"23926e9185d8d43c02807a838ffb373cc1977726094a4e46807c66ada9dd7660","final_host":"googleusercontent.com","ms":41,"identity":"PerplexityBot","html":true}},"robots":{"status":404,"content_type":"text/html","bytes":1571,"sha256":"6adee83a3eba87d30fdca3997d27097697f2767805f222a0f78c3f04392004d2","final_host":"googleusercontent.com","ms":83,"html":true,"text":null,"agents":null},"sitemap":null,"files":null,"derived":{"reachable":false,"robots_served":false,"robots_sitemap":false,"content_signal":false,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":[],"blocks_any_ai":false,"files":{"llms_txt":false,"agents_md":false,"agent_skills":false,"mcp_card":false,"api_catalog":false,"a2a_card":false},"any_capability":false,"ai_refused":[],"ai_thin":[],"rings":{"mcp_declared":false,"mcp_live":false,"mcp_auth":false,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":2,"fh":"googleusercontent.com","fp_other_host":[]},"derive_v":3,"frame":"top1k","shard":4},{"v":1,"kind":"crawlcheck-census-record","run":14,"domain":"doubleclick.net","rank":38,"measured_at":"2026-10-15T05:31:08.958Z","home":{"browser":{"status":200,"content_type":"text/html","bytes":125285,"sha256":"3faf2e727c83fc1d37e3d4a2eb5a232b0d9a2e8254abe598cfb4876cd85ca5d4","final_host":"marketingplatform.google.com","ms":176,"identity":"Mobile browser","html":true,"fp":"2252358704c6095f"},"gptbot":{"status":200,"content_type":"text/html","bytes":125285,"sha256":"e9d1f39eb494fac0e198ea8a8b00a36d73cc4cc81ae6d872c89cbc0da61bee29","final_host":"marketingplatform.google.com","ms":54,"identity":"GPTBot","html":true,"fp":"2252358704c6095f"},"claudebot":{"status":200,"content_type":"text/html","bytes":125285,"sha256":"3cad0c7643ea692f917219887daeb4164af925b8e4f80e89d9528cee4963461b","final_host":"marketingplatform.google.com","ms":74,"identity":"ClaudeBot","html":true,"fp":"2252358704c6095f"},"oaisearch":{"status":200,"content_type":"text/html","bytes":125285,"sha256":"cc41d01dc196f48e0f85d45ab4a41218ff8b9ff6d2a7a8af1c545797fd813df6","final_host":"marketingplatform.google.com","ms":59,"identity":"OAI-SearchBot","html":true,"fp":"2252358704c6095f"},"perplexity":{"status":200,"content_type":"text/html","bytes":125285,"sha256":"a9780002f787793f99e7b3def50de38adcde54518df501aba015acb67b98ab6d","final_host":"marketingplatform.google.com","ms":61,"identity":"PerplexityBot","html":true,"fp":"2252358704c6095f"}},"robots":{"status":200,"content_type":"text/plain","bytes":6624,"sha256":"1aba570a1b32deb66941e0c29d4f37be71ee58aa1a624aa20754b3639538cf2c","final_host":"doubleclick.net","ms":76,"html":false,"text":"User-agent: *\nUser-agent: Yandex\nDisallow: /search\nAllow: /search/about\nAllow: /search/howsearchworks\nDisallow: /sdch\nDisallow: /groups\nDisallow: /index.html?\nDisallow: /?\nDisallow: /goto?\nAllow: /?hl=\nDisallow: /?hl=*&\nAllow: /?hl=*&gws_rd=ssl$\nDisallow: /?hl=*&*&gws_rd=ssl\nAllow: /?gws_rd=ssl$\nAllow: /?pt1=true$\nDisallow: /imgres\nDisallow: /u/\nDisallow: /setprefs\nDisallow: /m?\nDisallow: /m/\nAllow:    /m/finance\nDisallow: /wml?\nDisallow: /wml/?\nDisallow: /wml/search?\nDisallow: /xhtml?\nDisallow: /xhtml/?\nDisallow: /xhtml/search?\nDisallow: /xml?\nDisallow: /imode?\nDisallow: /imode/?\nDisallow: /imode/search?\nDisallow: /jsky?\nDisallow: /jsky/?\nDisallow: /jsky/search?\nDisallow: /pda?\nDisallow: /pda/?\nDisallow: /pda/search?\nDisallow: /local?\nDisallow: /local_url\nDisallow: /products?\nDisallow: /product_\nDisallow: /products_\nDisallow: /products;\nDisallow: /print\nDisallow: /books/\nDisallow: /bkshp?*dq=\nDisallow: /bkshp?*q=\nDisallow: /books?*dq=\nDisallow: /books?*q=\nDisallow: /books?*qtid=\nDisallow: /books?*output=\nDisallow: /books?*pg=\nDisallow: /books?*jtp=\nDisallow: /books?*jscmd=\nDisallow: /books?*buy=\nDisallow: /books?*zoom=\nAllow: /books/about\nAllow: /books?*zoom=1\nAllow: /books?*zoom=5\nAllow: /books/content?*zoom=1\nAllow: /books/content?*zoom=5\nDisallow: /patents?\nDisallow: /patents/download/\nDisallow: /patents/pdf/\nDisallow: /patents/related/\nDisallow: /scholar\nDisallow: /citations?\nAllow: /citations?user=\nAllow: /citations?view_op=new_profile\nAllow: /citations?view_op=top_venues\nAllow: /scholar_share\nDisallow: /s?\nDisallow: /maps?\nAllow: /maps?daddr=\nAllow: /maps?entry=wc\nAllow: /maps?f=\nAllow: /maps?hl=\nAllow: /maps?q=\nAllow: /maps?saddr=\nAllow: /maps?sid=\nAllow: /maps?*output=classic\nAllow: /maps?*file=\nDisallow: /mapslt?\nDisallow: /maphp?\nDisallow: /maps/\nAllow: /maps/$\nAllow: /maps/@\nAllow: /maps/?daddr=\nAllow: /maps/?entry=wc\nAllow: /maps/?f=\nAllow: /maps/?hl=\nAllow: /maps/?q=\nAllow: /maps/?saddr=\nAllow: /maps/?sid=\nAllow: /maps/api/staticmap?\nAllow: /maps/search/\nAllow: /maps/sitemap.xml\nAllow: /maps/sitemaps/\nAllow: /maps/dir/\nAllow: /maps/d/\nAllow: /maps/reserve\nAllow: /maps/about\nAllow: /maps/contrib/\nAllow: /maps/match\nAllow: /maps/ms?\nAllow: /maps/place/\nAllow: /maps/_/\nAllow: /search?*tbm=map\nAllow: /maps/vt?\nAllow: /maps/preview\nDisallow: /maps/api/js/\nAllow: /maps/api/js\nDisallow: /mld?\nDisallow: /staticmap?\nDisallow: /help/maps/streetview/partners/welcome/\nDisallow: /help/maps/indoormaps/partners/\nDisallow: /lochp?\nDisallow: /ie?\nDisallow: /uds/\nDisallow: /transit?\nDisallow: /trends?\nDisallow: /trends/music?\nDisallow: /trends/hottrends?\nDisallow: /trends/viz?\nDisallow: /trends/embed.js?\nDisallow: /trends/fetchComponent?\nDisallow: /trends/beta\nDisallow: /trends/topics\nDisallow: /trends/explore?\nDisallow: /trends/api\nDisallow: /musica\nDisallow: /musicl\nDisallow: /musics\nDisallow: /urchin_test/\nDisallow: /movies?\nDisallow: /wapsearch?\nDisallow: /reviews/search?\nDisallow: /cbk\nDisallow: /profiles/me\nDisallow: /s2/profiles/me\nAllow: /s2/profiles\nAllow: /s2/oz\nAllow: /s2/photos\nAllow: /s2/search/social\nAllow: /s2/static\nDisallow: /s2\nDisallow: /transconsole/portal/\nDisallow: /aclk\nDisallow: /tbproxy/\nDisallow: /support/forum/search?\nDisallow: /reviews/polls/\nDisallow: /hosted/images/\nDisallow: /accounts/ClientLogin\nDisallow: /accounts/ClientAuth\nDisallow: /accounts/o8\nAllow: /accounts/o8/id\nDisallow: /quality_form?\nDisallow: /labs/popgadget/search\nDisallow: /compressiontest/\nDisallow: /analytics/feeds/\nDisallow: /analytics/partners/comments/\nDisallow: /analytics/portal/\nDisallow: /analytics/uploads/\nAllow: /alerts/manage\nAllow: /alerts/remove\nDisallow: /alerts/\nAllow: /alerts/$\nDisallow: /phone/compare/?\nDisallow: /travel/clk\nDisallow: /travel/lodging/clk\nDisallow: /travel/entity\nDisallow: /travel/search\nDisallow: /travel/flights/booking\nDisallow: /travel/flights/s/\nDisallow: /travel/flights/search\nDisallow: /travel/hotels/stories\nDisallow: /travel/hotels/*/stories\nDisallow: /travel/story\nDisallow: /hotelfinder/rpc\nDisallow: /hotels/rpc\nDisallow: /evaluation/\nDisallow: /forms/perks/\nDisallow: /shopping/suppliers/search\nDisallow: /edu/cs4hs/\nDisallow: /trustedstores/s/\nDisallow: /trustedstores/tm2\nDisallow: /trustedstores/verify\nDisallow: /shopping?\nDisallow: /shopping/product/\nDisallow: /shopping/seller\nDisallow: /shopping/ratings/account/metrics\nDisallow: /shopping/ratings/merchant/immersivedetails\nDisallow: /shopping/reviewer\nDisallow: /shopping/search\nDisallow: /shopping/deals\nAllow: /shopping?udm=28$\nDisallow: /storefront\nDisallow: /storepicker\nDisallow: /about/careers/applications/candidate-prep\nDisallow: /about/careers/applications/connect-with-a-googler\nDisallow: /about/careers/applications/jobs/results?page=\nDisallow: /about/careers/applications/jobs/results/?page=\nDisallow: /about/careers/applications/jobs/results?*&page=\nDisallow: /about/careers/applications/jobs/results/?*&page=\nDisallow: /landing/signout.html\nDisallow: /gallery/\nDisallow: /landing/now/ontap/\nAllow: /maps/reserve\nAllow: /maps/reserve/partners\nDisallow: /maps/reserve/api/\nDisallow: /maps/reserve/search\nDisallow: /maps/reserve/bookings\nDisallow: /maps/reserve/settings\nDisallow: /maps/reserve/manage\nDisallow: /maps/reserve/payment\nDisallow: /maps/reserve/receipt\nDisallow: /maps/reserve/sellersignup\nDisallow: /maps/reserve/feedback\nDisallow: /maps/reserve/terms\nDisallow: /maps/reserve/m/\nDisallow: /maps/reserve/b/\nDisallow: /maps/reserve/partner-dashboard\nDisallow: /local/cars\nDisallow: /local/dealership/\nDisallow: /local/dining/\nDisallow: /local/place/products/\nDisallow: /local/place/reviews/\nDisallow: /local/place/rap/\nDisallow: /local/tab/\nDisallow: /localservices/\nAllow: /localservices/coupon/\nDisallow: /nonprofits/account/\nDisallow: /uviewer\nDisallow: /landing/cmsnext-root/\n\n# AdsBot\nUser-agent: AdsBot-Google\nDisallow: /maps/api/js/\nAllow: /maps/api/js\nDisallow: /maps/api/place/js/\nDisallow: /maps/api/staticmap\nDisallow: /maps/api/streetview\n\n# New user agent groups must also have a user agent reference in the global (*)\n# group. See \"Order of precedence\" section in\n# https://goo.gle/rep#order-of-precedence-for-user-agents\nUser-agent: Yandex\nDisallow: /about/careers/applications/jobs/results\nDisallow: /about/careers/applications-a/jobs/results\n\n# Crawlers of certain social media sites are allowed to access page markup when\n# google.com/imgres* links are shared. To learn more, please contact\n# images-robots-allowlist@google.com.\nUser-agent: facebookexternalhit\nUser-agent: Twitterbot\nAllow: /imgres\nAllow: /search\nDisallow: /groups\nDisallow: /hosted/images/\nDisallow: /m/\n\nSitemap: https://www.google.com/sitemap.xml\n","agents":{"GPTBot":true,"OAI-SearchBot":true,"ChatGPT-User":true,"ClaudeBot":true,"Claude-SearchBot":true,"PerplexityBot":true,"Google-Extended":true,"Applebot-Extended":true,"CCBot":true,"Bytespider":true,"Meta-ExternalAgent":true,"Amazonbot":true}},"sitemap":{"status":200,"content_type":"text/xml","bytes":2047,"sha256":"0bc80feb7f7900da84ecf0cafdc78fb4dc3a9550413b40722eb73c6c02d0f49c","final_host":"www.google.com","ms":31,"url":"https://www.google.com/sitemap.xml","declared":true,"xml":true},"files":{"llms_txt":{"status":404,"content_type":"text/html","bytes":1569,"sha256":"47d71ed470dfe1ca13eb2af8fa019b95871a1bd1381a518ed71359e69b9c3cc7","final_host":"doubleclick.net","ms":30,"path":"/llms.txt","html":true,"json":null,"present":false},"agents_md":{"status":404,"content_type":"text/html","bytes":1570,"sha256":"f08e316602488d86581ce85b3076a13c82fc3f97cdac3deba23801fb66ca908d","final_host":"doubleclick.net","ms":50,"path":"/agents.md","html":true,"json":null,"present":false},"agent_skills":{"status":404,"content_type":"text/html","bytes":1596,"sha256":"eee9f4f975e761645c6c385ec645439385500d0062bff7170ed09a8835d6f92b","final_host":"doubleclick.net","ms":50,"path":"/.well-known/agent-skills/index.json","html":true,"json":null,"present":false},"mcp_card":{"status":404,"content_type":"text/html","bytes":1593,"sha256":"e5db4a1d5f7596a7d62e54e192ab48bbf195116fa33a2c7e160fdaf6131945a8","final_host":"doubleclick.net","ms":51,"path":"/.well-known/mcp/server-card.json","html":true,"json":null,"present":false},"api_catalog":{"status":404,"content_type":"text/html","bytes":1584,"sha256":"7de25ab2e3ce288cf29d538f80ed48bf0d5759f87b613e29ed345db4675698aa","final_host":"doubleclick.net","ms":48,"path":"/.well-known/api-catalog","html":true,"json":null,"present":false},"a2a_card":{"status":404,"content_type":"text/html","bytes":1588,"sha256":"64d2189fbe1531e7fb6ab2e6040a932d87ae1feb811d4781df85ea7ef6b4d651","final_host":"doubleclick.net","ms":53,"path":"/.well-known/agent-card.json","html":true,"json":null,"present":false}},"rings":{"v":1,"candidates":[],"mcp":[],"a2a":null,"webmcp":{"model_context":false,"bridge":false,"mcp_json":false,"link_rel":false,"any":false},"handshakes":0,"get_path":{"status":404,"content_type":"text/html","ms":26,"error":null},"get_host":{"status":530,"content_type":"text/plain","ms":78,"error":null}},"derived":{"reachable":true,"robots_served":true,"robots_sitemap":true,"content_signal":false,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":[],"blocks_any_ai":false,"files":{"llms_txt":false,"agents_md":false,"agent_skills":false,"mcp_card":false,"api_catalog":false,"a2a_card":false},"any_capability":false,"ai_refused":[],"ai_thin":[],"rings":{"mcp_declared":false,"mcp_live":false,"mcp_auth":false,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":2,"fh":"marketingplatform.google.com","fp_other_host":[],"fp":"2252358704c6095f","fp_bits":{"gptbot":0,"claudebot":0,"oaisearch":0,"perplexity":0}},"derive_v":3,"frame":"top1k","shard":5},{"v":1,"kind":"crawlcheck-census-record","run":14,"domain":"cloud.microsoft","rank":39,"measured_at":"2026-10-15T05:33:03.639Z","home":{"browser":{"status":200,"content_type":"text/html","bytes":253905,"sha256":"73da3bc257582f40019baf730dfd202c94c95c87a10d0e108f6ab363de279f67","final_host":"m365.cloud.microsoft","ms":534,"identity":"Mobile browser","html":true,"fp":"5febec8904d1e979"},"gptbot":{"status":200,"content_type":"text/html","bytes":346967,"sha256":"bdbf5d6c62abfb52b08d4d3993c8b95064db1337a677fd9801ee83cec5829f00","final_host":"m365.cloud.microsoft","ms":217,"identity":"GPTBot","html":true,"fp":"dbcbec8980d1e979"},"claudebot":{"status":200,"content_type":"text/html","bytes":347081,"sha256":"94d203db16a6997886b70261884c8281a7b06987989a11517fa40f1bf05ef1cb","final_host":"m365.cloud.microsoft","ms":594,"identity":"ClaudeBot","html":true,"fp":"dbdbec8980c1e979"},"oaisearch":{"status":200,"content_type":"text/html","bytes":348382,"sha256":"6c92f60def7c2d2510e384827015f4700f54bcf7c92810dee884b44ee6cc7c2a","final_host":"m365.cloud.microsoft","ms":689,"identity":"OAI-SearchBot","html":true,"fp":"dbdbe48980c0e979"},"perplexity":{"status":200,"content_type":"text/html","bytes":348370,"sha256":"f9fed2b254ed21c4698da17b1e5480bf89c777aaaa7c08f718f76a1c45da8e2c","final_host":"m365.cloud.microsoft","ms":606,"identity":"PerplexityBot","html":true,"fp":"dbcbe48980c0e979"}},"robots":{"status":200,"content_type":"text/html","bytes":348521,"sha256":"2d0a03d3684fc7287d6db95d2e16cae4f4b4a5cd216a582ee787d3e79d359eca","final_host":"m365.cloud.microsoft","ms":549,"html":true,"text":null,"agents":null},"sitemap":{"status":200,"content_type":"text/html","bytes":348492,"sha256":"b27c3c34d98b7541b729771caea07ca387f6784080d4d5680f16995e531b362a","final_host":"m365.cloud.microsoft","ms":204,"url":"https://cloud.microsoft/sitemap.xml","declared":false,"xml":false},"files":{"llms_txt":{"status":200,"content_type":"text/html","bytes":348413,"sha256":"8360c99fef851ba2583003a53611a445df5bcbb6bc8c7348f4349695bc1259d5","final_host":"m365.cloud.microsoft","ms":475,"path":"/llms.txt","html":true,"json":null,"present":false},"agents_md":{"status":200,"content_type":"text/html","bytes":348399,"sha256":"9a21ef669ba42c49e3a5c740191fdcff62d17fb7540187d9107034d144e5d0a0","final_host":"m365.cloud.microsoft","ms":209,"path":"/agents.md","html":true,"json":null,"present":false},"agent_skills":{"status":200,"content_type":"text/html","bytes":348384,"sha256":"b13a2b0ba3ae37033784dc4f1dbd7332f4ba855a172033a213451967ad008ee9","final_host":"m365.cloud.microsoft","ms":202,"path":"/.well-known/agent-skills/index.json","html":true,"json":null,"present":false},"mcp_card":{"status":200,"content_type":"text/html","bytes":348500,"sha256":"324bd74282a69babcd773a007bcb6be0999ff478f6ab9cbb93e4cf94cf0cde1c","final_host":"m365.cloud.microsoft","ms":304,"path":"/.well-known/mcp/server-card.json","html":true,"json":null,"present":false},"api_catalog":{"status":200,"content_type":"text/html","bytes":347391,"sha256":"4916d62ab69aacf297faec78bbac0522d519d6d8b0dd52ddd4905601e1e61079","final_host":"m365.cloud.microsoft","ms":1100,"path":"/.well-known/api-catalog","html":true,"json":null,"present":false},"a2a_card":{"status":200,"content_type":"text/html","bytes":348415,"sha256":"1b2ceb3f828e87c5a948ac74edd834f842e46036976896afbb456051e1facaa6","final_host":"m365.cloud.microsoft","ms":280,"path":"/.well-known/agent-card.json","html":true,"json":null,"present":false}},"rings":{"v":1,"candidates":[],"mcp":[],"a2a":null,"webmcp":{"model_context":false,"bridge":false,"mcp_json":false,"link_rel":false,"any":false},"handshakes":0,"get_path":{"status":200,"content_type":"text/html","ms":477,"error":null},"get_host":{"status":530,"content_type":"text/plain","ms":25,"error":null}},"derived":{"reachable":true,"robots_served":false,"robots_sitemap":false,"content_signal":false,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":[],"blocks_any_ai":false,"files":{"llms_txt":false,"agents_md":false,"agent_skills":false,"mcp_card":false,"api_catalog":false,"a2a_card":false},"any_capability":false,"ai_refused":[],"ai_thin":[],"rings":{"mcp_declared":false,"mcp_live":false,"mcp_auth":false,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":2,"fh":"m365.cloud.microsoft","fp_other_host":[],"fp":"5febec8904d1e979","fp_bits":{"gptbot":5,"claudebot":7,"oaisearch":9,"perplexity":8}},"derive_v":3,"frame":"top1k","shard":6},{"v":1,"kind":"crawlcheck-census-record","run":14,"domain":"office.net","rank":40,"measured_at":"2026-10-15T05:34:03.814Z","home":{"browser":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"office.net","ms":10,"identity":"Mobile browser","html":false},"gptbot":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"office.net","ms":12,"identity":"GPTBot","html":false},"claudebot":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"office.net","ms":2,"identity":"ClaudeBot","html":false},"oaisearch":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"office.net","ms":1,"identity":"OAI-SearchBot","html":false},"perplexity":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"office.net","ms":2,"identity":"PerplexityBot","html":false}},"robots":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"office.net","ms":2,"html":false,"text":null,"agents":null},"sitemap":null,"files":null,"derived":{"reachable":false,"robots_served":false,"robots_sitemap":false,"content_signal":false,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":[],"blocks_any_ai":false,"files":{"llms_txt":false,"agents_md":false,"agent_skills":false,"mcp_card":false,"api_catalog":false,"a2a_card":false},"any_capability":false,"ai_refused":[],"ai_thin":[],"rings":{"mcp_declared":false,"mcp_live":false,"mcp_auth":false,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":2,"fh":"office.net","fp_other_host":[]},"derive_v":3,"frame":"top1k","shard":7},{"v":1,"kind":"crawlcheck-census-record","run":14,"domain":"microsoftonline.com","rank":41,"measured_at":"2026-10-15T05:26:03.595Z","home":{"browser":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"microsoftonline.com","ms":42,"identity":"Mobile browser","html":false},"gptbot":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"microsoftonline.com","ms":41,"identity":"GPTBot","html":false},"claudebot":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"microsoftonline.com","ms":1,"identity":"ClaudeBot","html":false},"oaisearch":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"microsoftonline.com","ms":1,"identity":"OAI-SearchBot","html":false},"perplexity":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"microsoftonline.com","ms":1,"identity":"PerplexityBot","html":false}},"robots":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"microsoftonline.com","ms":1,"html":false,"text":null,"agents":null},"sitemap":null,"files":null,"derived":{"reachable":false,"robots_served":false,"robots_sitemap":false,"content_signal":false,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":[],"blocks_any_ai":false,"files":{"llms_txt":false,"agents_md":false,"agent_skills":false,"mcp_card":false,"api_catalog":false,"a2a_card":false},"any_capability":false,"ai_refused":[],"ai_thin":[],"rings":{"mcp_declared":false,"mcp_live":false,"mcp_auth":false,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":2,"fh":"microsoftonline.com","fp_other_host":[]},"derive_v":3,"frame":"top1k","shard":0},{"v":1,"kind":"crawlcheck-census-record","run":14,"domain":"trafficmanager.net","rank":42,"measured_at":"2026-10-15T05:27:03.729Z","home":{"browser":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"trafficmanager.net","ms":26,"identity":"Mobile browser","html":false},"gptbot":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"trafficmanager.net","ms":26,"identity":"GPTBot","html":false},"claudebot":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"trafficmanager.net","ms":1,"identity":"ClaudeBot","html":false},"oaisearch":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"trafficmanager.net","ms":1,"identity":"OAI-SearchBot","html":false},"perplexity":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"trafficmanager.net","ms":1,"identity":"PerplexityBot","html":false}},"robots":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"trafficmanager.net","ms":1,"html":false,"text":null,"agents":null},"sitemap":null,"files":null,"derived":{"reachable":false,"robots_served":false,"robots_sitemap":false,"content_signal":false,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":[],"blocks_any_ai":false,"files":{"llms_txt":false,"agents_md":false,"agent_skills":false,"mcp_card":false,"api_catalog":false,"a2a_card":false},"any_capability":false,"ai_refused":[],"ai_thin":[],"rings":{"mcp_declared":false,"mcp_live":false,"mcp_auth":false,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":2,"fh":"trafficmanager.net","fp_other_host":[]},"derive_v":3,"frame":"top1k","shard":1},{"v":1,"kind":"crawlcheck-census-record","run":14,"domain":"sharepoint.com","rank":43,"measured_at":"2026-10-15T05:28:03.923Z","home":{"browser":{"status":200,"content_type":"text/html","bytes":504344,"sha256":"5267ccde276b4d20d463753f57252a73cdf33e398132ef4fcc244c027efb315b","final_host":"www.microsoft.com","ms":1438,"identity":"Mobile browser","html":true,"fp":"0e1bca224859a35e"},"gptbot":{"status":200,"content_type":"text/html","bytes":504344,"sha256":"ca14f6df1cdac2fddbe84fef481854c9ec92cbf1fb120268ee0114bd665873a4","final_host":"www.microsoft.com","ms":1014,"identity":"GPTBot","html":true,"fp":"1e1bca224859a35e"},"claudebot":{"status":200,"content_type":"text/html","bytes":504344,"sha256":"f4061df672a4613dd3c542b08d5b93df25258d55bf6d240a769666a4a2843fc7","final_host":"www.microsoft.com","ms":1113,"identity":"ClaudeBot","html":true,"fp":"1e1bca224859a35e"},"oaisearch":{"status":200,"content_type":"text/html","bytes":504344,"sha256":"d8a58d6ea6f6a037fad915b59e7f314e79c794131f70a8587f275a7e84a21f03","final_host":"www.microsoft.com","ms":1029,"identity":"OAI-SearchBot","html":true,"fp":"1e1bca224859a35e"},"perplexity":{"status":200,"content_type":"text/html","bytes":504344,"sha256":"71d208b83cb763541458eb276a0830eb7ade25d5b44beb547d27508619843719","final_host":"www.microsoft.com","ms":1134,"identity":"PerplexityBot","html":true,"fp":"1e1bca224859a35e"}},"robots":{"status":520,"content_type":"text/plain","bytes":16,"sha256":"b59e493586b4118c563aaa751e04a7f7bc1c9741d241e0c1d3ba9405c323c570","final_host":"www.microsoft.com","ms":744,"html":false,"text":null,"agents":null},"sitemap":{"status":520,"content_type":"text/plain","bytes":16,"sha256":"b59e493586b4118c563aaa751e04a7f7bc1c9741d241e0c1d3ba9405c323c570","final_host":"www.microsoft.com","ms":258,"url":"https://sharepoint.com/sitemap.xml","declared":false,"xml":false},"files":{"llms_txt":{"status":520,"content_type":"text/plain","bytes":16,"sha256":"b59e493586b4118c563aaa751e04a7f7bc1c9741d241e0c1d3ba9405c323c570","final_host":"www.microsoft.com","ms":296,"path":"/llms.txt","html":false,"json":null,"present":false},"agents_md":{"status":520,"content_type":"text/plain","bytes":16,"sha256":"b59e493586b4118c563aaa751e04a7f7bc1c9741d241e0c1d3ba9405c323c570","final_host":"www.microsoft.com","ms":280,"path":"/agents.md","html":false,"json":null,"present":false},"agent_skills":{"status":400,"content_type":null,"bytes":0,"sha256":"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855","final_host":"sharepoint.com","ms":184,"path":"/.well-known/agent-skills/index.json","html":false,"json":null,"present":false},"mcp_card":{"status":400,"content_type":null,"bytes":0,"sha256":"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855","final_host":"sharepoint.com","ms":496,"path":"/.well-known/mcp/server-card.json","html":false,"json":null,"present":false},"api_catalog":{"status":400,"content_type":null,"bytes":0,"sha256":"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855","final_host":"sharepoint.com","ms":192,"path":"/.well-known/api-catalog","html":false,"json":null,"present":false},"a2a_card":{"status":400,"content_type":null,"bytes":0,"sha256":"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855","final_host":"sharepoint.com","ms":500,"path":"/.well-known/agent-card.json","html":false,"json":null,"present":false}},"rings":{"v":1,"candidates":[],"mcp":[],"a2a":null,"webmcp":{"model_context":false,"bridge":false,"mcp_json":false,"link_rel":false,"any":false},"handshakes":0,"get_path":{"status":520,"content_type":"text/plain","ms":256,"error":null},"get_host":{"status":200,"content_type":"text/html","ms":3989,"error":null}},"derived":{"reachable":true,"robots_served":false,"robots_sitemap":false,"content_signal":false,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":[],"blocks_any_ai":false,"files":{"llms_txt":false,"agents_md":false,"agent_skills":false,"mcp_card":false,"api_catalog":false,"a2a_card":false},"any_capability":false,"ai_refused":[],"ai_thin":[],"rings":{"mcp_declared":false,"mcp_live":false,"mcp_auth":false,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":2,"fh":"www.microsoft.com","fp_other_host":[],"fp":"0e1bca224859a35e","fp_bits":{"gptbot":1,"claudebot":1,"oaisearch":1,"perplexity":1}},"derive_v":3,"frame":"top1k","shard":2},{"v":1,"kind":"crawlcheck-census-record","run":14,"domain":"netflix.com","rank":44,"measured_at":"2026-10-15T05:29:03.596Z","home":{"browser":{"status":200,"content_type":"text/html","bytes":620884,"sha256":"65f0ceb2fe466cc3c351915bc1e8da203b02843dc5244ea14d154595adbb2087","final_host":"www.netflix.com","ms":881,"identity":"Mobile browser","html":true,"fp":"80928c2d107ef8cf"},"gptbot":{"status":200,"content_type":"text/html","bytes":610281,"sha256":"6d35c7b427b9ceb352175265756c49042d8fb1e0b09bc5b6e72243ee094feea0","final_host":"www.netflix.com","ms":827,"identity":"GPTBot","html":true,"fp":"0092882d107eb9cf"},"claudebot":{"status":200,"content_type":"text/html","bytes":610290,"sha256":"4b37b462bcd2898c8d7c569e9d1e384cd2be44ae283473bdd78d44182a832b2f","final_host":"www.netflix.com","ms":668,"identity":"ClaudeBot","html":true,"fp":"0092882d107eb9cf"},"oaisearch":{"status":200,"content_type":"text/html","bytes":610308,"sha256":"1a6eb2e69dd1ac4e9490c1d2f46a26c1c7c9ddb623630a303118f8f3fac07d51","final_host":"www.netflix.com","ms":672,"identity":"OAI-SearchBot","html":true,"fp":"0092882d107eb9cf"},"perplexity":{"status":200,"content_type":"text/html","bytes":610380,"sha256":"3a5563461bf288eccf828905facc690c8443cd5c93644f0cd6dcf4693955b487","final_host":"www.netflix.com","ms":635,"identity":"PerplexityBot","html":true,"fp":"0092882d107eb9cf"}},"robots":{"status":200,"content_type":"text/plain","bytes":3790,"sha256":"1e14fa0284e9757b52b48dde38b390effae68afdd72447088e72f18f49b1aa09","final_host":"www.netflix.com","ms":432,"html":false,"text":"User-agent: *\nDisallow: /\n\nUser-agent: googlebot\nUser-agent: Googlebot-Video\nUser-agent: Applebot\nUser-agent: bingbot\nUser-agent: Baiduspider\nUser-agent: Baiduspider-mobile\nUser-agent: Baiduspider-video\nUser-agent: Baiduspider-image\nUser-agent: NaverBot\nUser-agent: Y!J-WSC\nUser-agent: YJ-WSC\nUser-agent: Yeti\nUser-agent: Yandex\nUser-agent: YandexBot\nUser-agent: YandexMobileBot\nUser-agent: YandexVideo\nUser-agent: YandexWebmaster\nUser-agent: YandexSitelinks\nUser-agent: SeznamBot\nUser-agent: facebookexternalhit\nUser-agent: GPTBot\nUser-agent: ChatGPT-User\nUser-agent: OAI-SearchBot\nUser-agent: Google-Extended\nUser-agent: ClaudeBot\nUser-agent: Claude-User\nUser-agent: Claude-SearchBot\nUser-agent: PerplexityBot\nAllow: /\n\nDisallow: /accountstatus\nDisallow: /AccountStatus\nDisallow: /aui/inbound\nDisallow: /authenticate\nDisallow: /autologin\nDisallow: /clearcookies\nDisallow: /companies\nDisallow: /editpayment\nDisallow: /emailunsubscribe\nDisallow: /error\nDisallow: /eula\nDisallow: /geooverride\nDisallow: /help\nDisallow: /imagelibrary\nDisallow: /learnmorelayer\nDisallow: /learnmorelayertv\nDisallow: /login\nDisallow: /loginhelp\nDisallow: /loginhelp/lookup\nDisallow: /loginhelpsucess\nDisallow: /LoginHelp\nDisallow: /password\nDisallow: /logout\nDisallow: /Logout\nDisallow: /mcd\nDisallow: /modernizr\nDisallow: /n/\nDisallow: /notamember\nDisallow: /notfound\nDisallow: /notices\nDisallow: /nrdapp\nDisallow: /optout\nDisallow: /overviewblockseeother\nDisallow: /popup/codewhatisthis\nDisallow: /popupdetails\nDisallow: /PopupDetails\nDisallow: /popupprivacypolicy\nDisallow: /privacypolicychanges\nDisallow: /registration\nDisallow: /rememberme\nDisallow: /signout\nDisallow: /signurl\nDisallow: /subscriptioncancel\nDisallow: /tastesurvey\nDisallow: /termsofusechanges\nDisallow: /tvsignup\nDisallow: /upcomingevents\nDisallow: /verifyidentity\nDisallow: /whysecure\n\nDisallow: /arabic\nDisallow: /Arabic\nDisallow: /chinese\nDisallow: /Chinese\nDisallow: /korean\nDisallow: /Korean\n\nDisallow: /airtel\nDisallow: /anan\nDisallow: /bouyguestelecom\nDisallow: /britishairways\nDisallow: /brutus\nDisallow: /comhem\nDisallow: /courts\nDisallow: /csl\nDisallow: /elisa\nDisallow: /entertain\nDisallow: /FireTV\nDisallow: /firetv\nDisallow: /freemonth\nDisallow: /kpn\nDisallow: /lg\nDisallow: /maxis\nDisallow: /Maxis\nDisallow: /meo\nDisallow: /Meo\nDisallow: /orangefrance\nDisallow: /Panasonic\nDisallow: /panasonic\nDisallow: /playstation\nDisallow: /proximus\nDisallow: /qantas\nDisallow: /samsung\nDisallow: /Sony\nDisallow: /sony\nDisallow: /talktalk\nDisallow: /tdc\nDisallow: /telenor\nDisallow: /telfort\nDisallow: /tim\nDisallow: /virginaustralia\nDisallow: /vodafone\nDisallow: /vodafonedemobilelaunch\nDisallow: /xboxone\nDisallow: /xfinity\nDisallow: /xs4all\nDisallow: /ziggo\n\nDisallow: /accountaccess\nDisallow: /AccountAccess\nDisallow: /BillingActivity\nDisallow: /browse\nDisallow: /browse/*\nAllow: /browse/genre/*\nDisallow: /CancelPlan\nDisallow: /ChangePlan\nDisallow: /changeplan\nDisallow: /deviceManagement\nDisallow: /DoNotTest\nDisallow: /EditProfiles\nDisallow: /email\nDisallow: /EmailPreferences\nDisallow: /entrytrap\nDisallow: /HdToggle\nDisallow: /LanguagePreferences\nDisallow: /ManageDevices\nDisallow: /ManageProfiles\nDisallow: /MoviesYouveSeen\nDisallow: /NewWatchInstantlyRSS\nDisallow: /NewWatchInstantlyRSS/*\nDisallow: /payment\nDisallow: /Payment\nDisallow: /phonenumber\nDisallow: /pin\nDisallow: /profiles\nDisallow: /profiles/*\nDisallow: /ProfilesGate\nDisallow: /search\nDisallow: /search/*\nDisallow: /viewingactivity\nDisallow: /WiViewingActivity\nDisallow: /yourAccount\nDisallow: /youraccount\nDisallow: /YourAccount\nDisallow: /YourAccountPayment\n\nUser-agent: AdsBot-Google\nUser-agent: Twitterbot\nUser-agent: Adidxbot\nAllow: /\n\nUser-agent: Yahoo Pipes 1.0\nUser-agent: Facebot\nDisallow: /\nAllow: /tudum\n\nSitemap: https://www.netflix.com/sitemap/index\n","agents":{"GPTBot":true,"OAI-SearchBot":true,"ChatGPT-User":true,"ClaudeBot":true,"Claude-SearchBot":true,"PerplexityBot":true,"Google-Extended":true,"Applebot-Extended":true,"CCBot":false,"Bytespider":false,"Meta-ExternalAgent":false,"Amazonbot":false}},"sitemap":{"status":200,"content_type":"text/xml","bytes":2531404,"sha256":"e212094e7d47fb1e202e67559bd5dedfa329b4420fd0e572b18ce3207f0fbdef","final_host":"www.netflix.com","ms":287,"url":"https://www.netflix.com/sitemap/index","declared":true,"xml":true},"files":{"llms_txt":{"status":200,"content_type":"text/html","bytes":23299,"sha256":"1f97f78c532517cc83fa23465c3c9b83412eb39731bb8ec3d5a678b03b5dbb86","final_host":"www.netflix.com","ms":193,"path":"/llms.txt","html":true,"json":null,"present":false},"agents_md":{"status":200,"content_type":"text/html","bytes":23300,"sha256":"eba82a82c7031d3952542fbae2a0ba7991186df57a1f7ae9ba69399a845d3336","final_host":"www.netflix.com","ms":209,"path":"/agents.md","html":true,"json":null,"present":false},"agent_skills":{"status":200,"content_type":"text/html","bytes":23325,"sha256":"f50ad4041480dcf8df506c9c3dcc0712a759cce6bbf2c3d3ac9622eae5d6fa7d","final_host":"www.netflix.com","ms":189,"path":"/.well-known/agent-skills/index.json","html":true,"json":null,"present":false},"mcp_card":{"status":200,"content_type":"text/html","bytes":23322,"sha256":"bce75558d7c7c2e4ebd7c95adeb0eeb0beab874e222a89e924bdcef197716518","final_host":"www.netflix.com","ms":188,"path":"/.well-known/mcp/server-card.json","html":true,"json":null,"present":false},"api_catalog":{"status":200,"content_type":"text/html","bytes":23308,"sha256":"933fea6bb544a870cc161bcd4b20087af82ac5d039f5810fed934243135b2a8d","final_host":"www.netflix.com","ms":183,"path":"/.well-known/api-catalog","html":true,"json":null,"present":false},"a2a_card":{"status":200,"content_type":"text/html","bytes":23312,"sha256":"7ece7ccf61f308a20e8e978e0a1961ff4764363d2ab9ae0609b649eaca503643","final_host":"www.netflix.com","ms":182,"path":"/.well-known/agent-card.json","html":true,"json":null,"present":false}},"rings":{"v":1,"candidates":[],"mcp":[],"a2a":null,"webmcp":{"model_context":false,"bridge":false,"mcp_json":false,"link_rel":false,"any":false},"handshakes":0,"get_path":{"status":200,"content_type":"text/html","ms":177,"error":null},"get_host":{"status":530,"content_type":"text/plain","ms":54,"error":null}},"derived":{"reachable":true,"robots_served":true,"robots_sitemap":true,"content_signal":false,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":["CCBot","Bytespider","Meta-ExternalAgent","Amazonbot"],"blocks_any_ai":true,"files":{"llms_txt":false,"agents_md":false,"agent_skills":false,"mcp_card":false,"api_catalog":false,"a2a_card":false},"any_capability":false,"ai_refused":[],"ai_thin":[],"rings":{"mcp_declared":false,"mcp_live":false,"mcp_auth":false,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":2,"fh":"www.netflix.com","fp_other_host":[],"fp":"80928c2d107ef8cf","fp_bits":{"gptbot":4,"claudebot":4,"oaisearch":4,"perplexity":4}},"derive_v":3,"frame":"top1k","shard":3},{"v":1,"kind":"crawlcheck-census-record","run":14,"domain":"gandi.net","rank":45,"measured_at":"2026-10-15T05:30:03.842Z","home":{"browser":{"status":200,"content_type":"text/html","bytes":189234,"sha256":"f7df022980957b459ee117dbe898db51214baaf68d39047b96765002f4fa297f","final_host":"www.gandi.net","ms":513,"identity":"Mobile browser","html":true,"fp":"363226cb9ef3f9f1"},"gptbot":{"status":200,"content_type":"text/html","bytes":189234,"sha256":"f7df022980957b459ee117dbe898db51214baaf68d39047b96765002f4fa297f","final_host":"www.gandi.net","ms":504,"identity":"GPTBot","html":true,"fp":"363226cb9ef3f9f1"},"claudebot":{"status":200,"content_type":"text/html","bytes":189234,"sha256":"f7df022980957b459ee117dbe898db51214baaf68d39047b96765002f4fa297f","final_host":"www.gandi.net","ms":530,"identity":"ClaudeBot","html":true,"fp":"363226cb9ef3f9f1"},"oaisearch":{"status":200,"content_type":"text/html","bytes":189234,"sha256":"f7df022980957b459ee117dbe898db51214baaf68d39047b96765002f4fa297f","final_host":"www.gandi.net","ms":522,"identity":"OAI-SearchBot","html":true,"fp":"363226cb9ef3f9f1"},"perplexity":{"status":200,"content_type":"text/html","bytes":189234,"sha256":"f7df022980957b459ee117dbe898db51214baaf68d39047b96765002f4fa297f","final_host":"www.gandi.net","ms":462,"identity":"PerplexityBot","html":true,"fp":"363226cb9ef3f9f1"}},"robots":{"status":200,"content_type":"text/plain","bytes":9023,"sha256":"3fd451a6143f6a84977d65af0ae1e26a153466423e8d7e08e672f3d1466e1cdc","final_host":"www.gandi.net","ms":402,"html":false,"text":"User-agent: *\r\n\r\n\r\n###### Sitemaps ######\r\nSitemap: https://www.gandi.net/sitemap.xml\r\nSitemap: https://news.gandi.net/sitemaps.xml\r\n\r\n###### Disalow ######\r\nDisallow: /*redirect=*\r\nDisallow: /*navigation=*\r\nDisallow: /*client_id=*\r\nDisallow: /*locale=*\r\nDisallow: /*tld=*\r\nDisallow: /*bulk=*\r\nDisallow: /*bu_id=*\r\nDisallow: /*opened=*\r\nDisallow: /*type=*\r\nDisallow: /*template=*\r\nDisallow: /*size=*\r\nDisallow: /*options=*\r\nDisallow: /*error=*\r\nDisallow: /*grid=*\r\nDisallow: /*currency=*\r\nDisallow: /*?c=*\r\nDisallow: /*&c=*\r\nDisallow: /*?sw=*\r\nDisallow: /*&sw=*\r\nDisallow: /*taxes=*\r\nDisallow: /pdf/*\r\nDisallow: /*/sample/*\r\nDisallow: */prices$\r\nDisallow: */rules$\r\nDisallow: */dates$\r\nDisallow: */news$\r\nAllow: */tld/news$\r\nDisallow: *?country=*\r\nDisallow: *?l=*\r\nDisallow: *?lang=*\r\nDisallow: /gcart/*\r\n\r\n\r\nUser-Agent: facebookexternalhit/1.1\r\nDisallow:\r\n\r\nUser-Agent: Facebot\r\nDisallow:\r\n\r\n###### Suppression robots ######\r\nUser-agent: AhrefsBot\r\nDisallow: /\r\nUser-agent: Alexibot\r\nDisallow: /\r\nUser-agent: Aqua_Products\r\nDisallow: /\r\nUser-agent: asterias\r\nDisallow: /\r\nUser-agent: b2w/0.1\r\nDisallow: /\r\nUser-agent: BackDoorBot/1.0\r\nDisallow: /\r\nUser-agent: BacklinkCrawler\r\nDisallow: /\r\nUser-agent: bdbrandprotect\r\nDisallow: /\r\nUser-agent: Black Hole\r\nDisallow: /\r\nUser-agent: BlowFish/1.0\r\nDisallow: /\r\nUser-agent: boitho.com-dc\r\nDisallow: /\r\nUser-agent: Bookmark search tool\r\nDisallow: /\r\nUser-agent: BotALot\r\nDisallow: /\r\nUser-agent: BotRightHere\r\nDisallow: /\r\nUser-agent: BPImageWalker\r\nDisallow: /\r\nUser-agent: BuiltBotTough\r\nDisallow: /\r\nUser-agent: Bullseye/1.0\r\nDisallow: /\r\nUser-agent: BunnySlippers\r\nDisallow: /\r\nUser-agent: CazoodleBot\r\nDisallow: /\r\nUser-agent: Cegbfeieh\r\nDisallow: /\r\nUser-agent: CheeseBot\r\nDisallow: /\r\nUser-agent: CherryPicker\r\nDisallow: /\r\nUser-agent: CherryPickerElite/1.0\r\nDisallow: /\r\nUser-agent: CherryPickerSE/1.0\r\nDisallow: /\r\nUser-agent: ConveraCrawler\r\nDisallow: /\r\nUser-agent: Copernic\r\nDisallow: /\r\nUser-agent: CopyRightCheck\r\nDisallow: /\r\nUser-agent: cosmos\r\nDisallow: /\r\nUser-agent: Crescent\r\nDisallow: /\r\nUser-agent: Crescent Internet ToolPak HTTP OLE Control v.1.0\r\nDisallow: /\r\nUser-agent: discobot\r\nDisallow: /\r\nUser-agent: DittoSpyder\r\nDisallow: /\r\nUser-agent: dotbot\r\nDisallow: /\r\nUser-agent: Download Ninja\r\nDisallow: /\r\nUser-agent: EmailCollector\r\nDisallow: /\r\nUser-agent: EmailSiphon\r\nDisallow: /\r\nUser-agent: EmailWolf\r\nDisallow: /\r\nUser-agent: EroCrawler\r\nDisallow: /\r\nUser-agent: ExtractorPro\r\nDisallow: /\r\nUser-agent: Ezooms\r\nDisallow: /\r\nUser-agent: FairAd Client\r\nDisallow: /\r\nUser-agent: Fasterfox\r\nDisallow: /\r\nUser-agent: Fetch\r\nDisallow: /\r\nUser-agent: findlinks\r\nDisallow: /\r\nUser-agent: Flaming AttackBot\r\nDisallow: /\r\nUser-agent: Foobot\r\nDisallow: /\r\nUser-agent: Gaisbot\r\nDisallow: /\r\nUser-agent: GetRight/4.2\r\nDisallow: /\r\nUser-agent: Gigabot\r\nDisallow: /\r\nUser-agent: grub\r\nDisallow: /\r\nUser-agent: grub-client\r\nDisallow: /\r\nUser-agent: Harvest/1.5\r\nDisallow: /\r\nUser-agent: heritrix\r\nDisallow: /\r\nUser-agent: hloader\r\nDisallow: /\r\nUser-agent: httplib\r\nDisallow: /\r\nUser-agent: HTTrack\r\nDisallow: / \r\nUser-agent: HTTrack 3.0\r\nDisallow: /\r\nUser-agent: HuaweiSymantecSpider\r\nDisallow: /\r\nUser-agent: humanlinks\r\nDisallow: /\r\nUser-agent: ichiro\r\nDisallow: /\r\nUser-agent: InfoNaviRobot\r\nDisallow: /\r\nUser-agent: IRLbot\r\nDisallow: /\r\nUser-agent: Iron33/1.0.2\r\nDisallow: /\r\nUser-agent: JennyBot\r\nDisallow: /\r\nUser-agent: kalooga\r\nDisallow: /\r\nUser-agent: Kenjin Spider\r\nDisallow: /\r\nUser-agent: Keyword Density/0.9\r\nDisallow: /\r\nUser-agent: larbin\r\nDisallow: /\r\nUser-agent: LexiBot\r\nDisallow: /\r\nUser-agent: LexxeBot\r\nDisallow: /\r\nUser-agent: libWeb/clsHTTP\r\nDisallow: /\r\nUser-agent: libwww\r\nDisallow: /\r\nUser-agent: LinkextractorPro\r\nDisallow: /\r\nUser-agent: linko\r\nDisallow: /\r\nUser-agent: LinkScan/8.1a Unix\r\nDisallow: /\r\nUser-agent: LinkWalker\r\nDisallow: /\r\nUser-agent: LNSpiderguy\r\nDisallow: /\r\nUser-agent: lwp-trivial\r\nDisallow: /\r\nUser-agent: lwp-trivial/1.34\r\nDisallow: /\r\nUser-agent: Mata Hari\r\nDisallow: /\r\nUser-agent: Microsoft URL Control\r\nDisallow: /\r\nUser-agent: Microsoft URL Control - 5.01.4511\r\nDisallow: /\r\nUser-agent: Microsoft URL Control - 6.00.8169\r\nDisallow: /\r\nUser-agent: Microsoft.URL.Control\r\nDisallow: /\r\nUser-agent: MIIxpc\r\nDisallow: /\r\nUser-agent: MIIxpc/4.2\r\nDisallow: /\r\nUser-agent: Mister PiX\r\nDisallow: /\r\nUser-agent: MJ12bot\r\nDisallow: /\r\nUser-agent: MLBot\r\nDisallow: /\r\nUser-agent: moget\r\nDisallow: /\r\nUser-agent: moget/2.1\r\nDisallow: /\r\nUser-agent: Mozilla/4.0 (compatible; BullsEye; Windows 95)\r\nDisallow: /\r\nUser-agent: MSIECrawler\r\nDisallow: /\r\nUser-agent: NaverBot\r\nDisallow: /\r\nUser-agent: NerdyBot\r\nDisallow: /\r\nUser-agent: Net\\ Vampire\r\nDisallow: /\r\nUser-agent: NetAnts\r\nDisallow: /\r\nUser-agent: NetMechanic\r\nDisallow: /\r\nUser-agent: NICErsPRO\r\nDisallow: /\r\nUser-agent: Nutch\r\nDisallow: /\r\nUser-agent: Ocelli\r\nDisallow: /\r\nUser-agent: Offline Explorer\r\nDisallow: /\r\nUser-agent: Openbot\r\nDisallow: /\r\nUser-agent: Openfind\r\nDisallow: /\r\nUser-agent: Openfind data gatherer\r\nDisallow: /\r\nUser-agent: Oracle Ultra Search\r\nDisallow: /\r\nUser-agent: penthesilea\r\nDisallow: /\r\nUser-agent: PerMan\r\nDisallow: /\r\nUser-agent: PhpDig\r\nDisallow: /\r\nUser-agent: ProPowerBot/2.14\r\nDisallow: /\r\nUser-agent: ProWebWalker\r\nDisallow: /\r\nUser-agent: psbot\r\nDisallow: /\r\nUser-agent: Python-urllib\r\nDisallow: /\r\nUser-agent: QueryN Metasearch\r\nDisallow: /\r\nUser-agent: Radiation Retriever 1.1\r\nDisallow: /\r\nUser-agent: RepoMonkey\r\nDisallow: /\r\nUser-agent: RepoMonkey Bait & Tackle/v1.01\r\nDisallow: /\r\nUser-agent: RMA\r\nDisallow: /\r\nUser-agent: rogerbot\r\nDisallow: /\r\nUser-agent: SCOUTjET\r\nDisallow: /\r\nUser-agent: SearchDaimon.com-dc\r\nDisallow: /\r\nUser-agent: searchpreview\r\nDisallow: /\r\nUser-agent: SEOENGBot\r\nDisallow: /\r\nUser-agent: ShopWiki\r\nDisallow: /\r\nUser-agent: sitecheck.internetseer.com\r\nDisallow: /\r\nUser-agent: SiteSnagger\r\nDisallow: /\r\nUser-agent: Snapbot\r\nDisallow: /\r\nUser-agent: Sogou\\ web\\ spider\r\nDisallow: /\r\nUser-agent: Sosospider\r\nDisallow: /\r\nUser-agent: SpankBot\r\nDisallow: /\r\nUser-agent: spanner\r\nDisallow: /\r\nUser-agent: spbot\r\nDisallow: /\r\nUser-agent: SurveyBot\r\nDisallow: /\r\nUser-agent: suzuran\r\nDisallow: /\r\nUser-agent: Szukacz/1.4\r\nDisallow: /\r\nUser-agent: Teleport\r\nDisallow: /\r\nUser-agent: TeleportPro\r\nDisallow: /\r\nUser-agent: Telesoft\r\nDisallow: /\r\nUser-agent: Templeton\r\nDisallow: /\r\nUser-agent: The Intraformant\r\nDisallow: /\r\nUser-agent: TheNomad\r\nDisallow: /\r\nUser-agent: TightTwatBot\r\nDisallow: /\r\nUser-agent: Titan\r\nDisallow: /\r\nUser-agent: toCrawl/UrlDispatcher\r\nDisallow: /\r\nUser-agent: True_Robot\r\nDisallow: /\r\nUser-agent: True_Robot/1.0\r\nDisallow: /\r\nUser-agent: turingos\r\nDisallow: /\r\nUser-agent: TurnitinBot\r\nDisallow: /\r\nUser-agent: TurnitinBot/1.5\r\nDisallow: /\r\nUser-agent: TwengaBot\r\nDisallow: /\r\nUser-agent: Twiceler\r\nDisallow: /\r\nUser-agent: Updownerbot\r\nDisallow: /\r\nUser-agent: URL Control\r\nDisallow: /\r\nUser-agent: URL_Spider_Pro\r\nDisallow: /\r\nUser-agent: URLy Warning\r\nDisallow: /\r\nUser-agent: VCI\r\nDisallow: /\r\nUser-agent: VCI WebViewer VCI WebViewer Win32\r\nDisallow: /\r\nUser-agent: Web Image Collector\r\nDisallow: /\r\nUser-agent: WebAuto\r\nDisallow: /\r\nUser-agent: WebBandit\r\nDisallow: /\r\nUser-agent: WebBandit/3.50\r\nDisallow: /\r\nUser-agent: WebCapture 2.0\r\nDisallow: /\r\nUser-agent: WebCopier\r\nDisallow: /\r\nUser-agent: WebCopier v.2.2\r\nDisallow: /\r\nUser-agent: WebCopier v3.2a\r\nDisallow: /\r\nUser-agent: WebEnhancer\r\nDisallow: /\r\nUser-agent: WebmasterWorldForumBot\r\nDisallow: /\r\nUser-agent: WebSauger\r\nDisallow: /\r\nUser-agent: Website Quester\r\nDisallow: /\r\nUser-agent: Webster Pro\r\nDisallow: /\r\nUser-agent: WebStripper\r\nDisallow: /\r\nUser-agent: WebZIP\r\nDisallow: /\r\nUser-agent: WebZip/4.0\r\nDisallow: /\r\nUser-agent: WebZIP/4.21\r\nDisallow: /\r\nUser-agent: WebZIP/5.0\r\nDisallow: /\r\nUser-agent: wget\r\nDisallow: /\r\nUser-agent: Wget/1.5.3\r\nDisallow: /\r\nUser-agent: Wget/1.6\r\nDisallow: /\r\nUser-agent: woozweb-monitoring\r\nDisallow: /\r\nUser-agent: woriobot\r\nDisallow: /\r\nUser-agent: wprocketbot\r\nAllow: /\r\nUser-agent: WWW-Collector-E\r\nDisallow: /\r\nUser-agent: Xenu\r\nDisallow: /\r\nUser-agent: Xenu's\r\nDisallow: /\r\nUser-agent: Xenu's Link Sleuth 1.1c\r\nDisallow: /\r\nUser-agent: YACYBIT\r\nDisallow: /\r\nUser-agent: yacybot\r\nDisallow: /\r\nUser-agent: Yandex\r\nDisallow: /\r\nUser-agent: YandexBot\r\nDisallow: /\r\nUser-agent: Yeti\r\nDisallow: /\r\nUser-agent: YodaoBot\r\nDisallow: /\r\nUser-agent: Zealbot\r\nDisallow: /\r\nUser-agent: Zeus\r\nDisallow: /\r\nUser-agent: Zeus 32297 Webster Pro V2.9 Win32\r\nDisallow: /\r\nUser-agent: Zeus Link Scout\r\nDisallow: /\r\nUser-agent: ZyBorg\r\nDisallow: /\r\nUser-agent: Mozilla/4.0 (compatible; BullsEye; Windows 95)\r\nDisallow: /\r\nUser-agent: NerdyBot\r\nDisallow: /\r\nUser-agent: WebAuto\r\nDisallow: /\r\nUser-agent: wprocketbot\r\nDisallow: /\r\nUser-agent: DIIbot\r\nDisallow: /\r\nUser-agent: eCatch\r\nDisallow: /\r\nUser-agent: Jetbot\r\nDisallow: /\r\nUser-agent: Openfind data gathere\r\nDisallow: /\r\nUser-agent: QuepasaCreep\r\nDisallow: /\r\nUser-agent: Roverbot\r\nDisallow: /\r\nUser-agent: Foobot\r\nDisallow: /\r\nUser-agent: RepoMonkey Bait & Tackle/v1.01\r\nDisallow: /\r\nUser-agent: ExtractorPro\r\nDisallow: /","agents":{"GPTBot":true,"OAI-SearchBot":true,"ChatGPT-User":true,"ClaudeBot":true,"Claude-SearchBot":true,"PerplexityBot":true,"Google-Extended":true,"Applebot-Extended":true,"CCBot":true,"Bytespider":true,"Meta-ExternalAgent":true,"Amazonbot":true}},"sitemap":{"status":200,"content_type":"text/xml","bytes":529,"sha256":"20e45cfdab4c73574ebe3829d032be6b389f600aac558cddcc91ecbe5f9681f9","final_host":"www.gandi.net","ms":41,"url":"https://www.gandi.net/sitemap.xml","declared":true,"xml":true},"files":{"llms_txt":{"status":404,"content_type":"text/html","bytes":118057,"sha256":"65bd387183652f66562a17e458a200df2aa477360cfa5c1e7325c4d80c911c66","final_host":"www.gandi.net","ms":306,"path":"/llms.txt","html":true,"json":null,"present":false},"agents_md":{"status":404,"content_type":"text/html","bytes":118058,"sha256":"afb2c185f2e48c627a1f800d39a05caf6e814db3edadb60787005d7bea062868","final_host":"www.gandi.net","ms":304,"path":"/agents.md","html":true,"json":null,"present":false},"agent_skills":{"status":404,"content_type":"text/html","bytes":118084,"sha256":"b50fc7797eec4ec49f36cc3048a3facb31bd6bf3dd2a843ca9e045b6828dfb2c","final_host":"www.gandi.net","ms":279,"path":"/.well-known/agent-skills/index.json","html":true,"json":null,"present":false},"mcp_card":{"status":404,"content_type":"text/html","bytes":118081,"sha256":"e3661a08c7ab3af5246d6c9ae425c39c4e79cee00d8d96faa70cfea50c141752","final_host":"www.gandi.net","ms":283,"path":"/.well-known/mcp/server-card.json","html":true,"json":null,"present":false},"api_catalog":{"status":404,"content_type":"text/html","bytes":118072,"sha256":"83f3a8c49cd64e19a113d6fcf955aec4f27afa2c15f0d296ec5f7550f89005b0","final_host":"www.gandi.net","ms":274,"path":"/.well-known/api-catalog","html":true,"json":null,"present":false},"a2a_card":{"status":404,"content_type":"text/html","bytes":118076,"sha256":"f209be29c77ed21ba846f9ae1a28382184e6bf7ca63fa3feaa40ed28d5b94d88","final_host":"www.gandi.net","ms":285,"path":"/.well-known/agent-card.json","html":true,"json":null,"present":false}},"rings":{"v":1,"candidates":[],"mcp":[],"a2a":null,"webmcp":{"model_context":false,"bridge":false,"mcp_json":false,"link_rel":false,"any":false},"handshakes":0,"get_path":{"status":200,"content_type":"text/html","ms":172,"error":null},"get_host":{"status":530,"content_type":"text/plain","ms":152,"error":null}},"derived":{"reachable":true,"robots_served":true,"robots_sitemap":true,"content_signal":false,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":[],"blocks_any_ai":false,"files":{"llms_txt":false,"agents_md":false,"agent_skills":false,"mcp_card":false,"api_catalog":false,"a2a_card":false},"any_capability":false,"ai_refused":[],"ai_thin":[],"rings":{"mcp_declared":false,"mcp_live":false,"mcp_auth":false,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":2,"fh":"www.gandi.net","fp_other_host":[],"fp":"363226cb9ef3f9f1","fp_bits":{"gptbot":0,"claudebot":0,"oaisearch":0,"perplexity":0}},"derive_v":3,"frame":"top1k","shard":4},{"v":1,"kind":"crawlcheck-census-record","run":14,"domain":"cloudfront.net","rank":46,"measured_at":"2026-10-15T05:31:08.958Z","home":{"browser":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"cloudfront.net","ms":42,"identity":"Mobile browser","html":false},"gptbot":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"cloudfront.net","ms":34,"identity":"GPTBot","html":false},"claudebot":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"cloudfront.net","ms":25,"identity":"ClaudeBot","html":false},"oaisearch":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"cloudfront.net","ms":19,"identity":"OAI-SearchBot","html":false},"perplexity":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"cloudfront.net","ms":4,"identity":"PerplexityBot","html":false}},"robots":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"cloudfront.net","ms":3,"html":false,"text":null,"agents":null},"sitemap":null,"files":null,"derived":{"reachable":false,"robots_served":false,"robots_sitemap":false,"content_signal":false,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":[],"blocks_any_ai":false,"files":{"llms_txt":false,"agents_md":false,"agent_skills":false,"mcp_card":false,"api_catalog":false,"a2a_card":false},"any_capability":false,"ai_refused":[],"ai_thin":[],"rings":{"mcp_declared":false,"mcp_live":false,"mcp_auth":false,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":2,"fh":"cloudfront.net","fp_other_host":[]},"derive_v":3,"frame":"top1k","shard":5},{"v":1,"kind":"crawlcheck-census-record","run":14,"domain":"youtu.be","rank":47,"measured_at":"2026-10-15T05:33:03.639Z","home":{"browser":{"status":200,"content_type":"text/html","bytes":370219,"sha256":"261cbde0d41f3720e5b8485fc2b56a204203ea487731a646952102f35b1f913e","final_host":"m.youtube.com","ms":443,"identity":"Mobile browser","html":true},"gptbot":{"status":200,"content_type":"text/html","bytes":993559,"sha256":"2c1684a373678bd7fe5c85d5656110e582c6c6fc4a0f1a7b03456260637435bf","final_host":"www.youtube.com","ms":333,"identity":"GPTBot","html":true},"claudebot":{"status":200,"content_type":"text/html","bytes":994025,"sha256":"ff1f0515880744d900db20d536854c1f9a9923e2590d54c8e30f7329e5e282cf","final_host":"www.youtube.com","ms":328,"identity":"ClaudeBot","html":true},"oaisearch":{"status":200,"content_type":"text/html","bytes":998438,"sha256":"e3d18cf90c48f549fd260b8be32d3e39ea6ad4088709664cc552dcf8a0c7f167","final_host":"www.youtube.com","ms":271,"identity":"OAI-SearchBot","html":true},"perplexity":{"status":200,"content_type":"text/html","bytes":951408,"sha256":"47eb7d1c3d7328e3dd47e4e01816d8221a4da87ae9f4e83cf515e236d3455b05","final_host":"www.youtube.com","ms":243,"identity":"PerplexityBot","html":true}},"robots":{"status":200,"content_type":"text/plain","bytes":511,"sha256":"c290485c8002c51c24c95765948dbcec9ec88c1f181638d245418a36194fd1a0","final_host":"youtu.be","ms":122,"html":false,"text":"# robots.txt file for youtu.be\n\nUser-agent: *\nDisallow: /api/\nDisallow: /comment\nDisallow: /feeds/videos.xml\nDisallow: /get_video\nDisallow: /get_video_info\nDisallow: /get_midroll_info\nDisallow: /live_chat\nDisallow: /login\nDisallow: /results\nDisallow: /signup\nDisallow: /t/terms\nDisallow: /timedtext_video\nDisallow: /verify_age\nDisallow: /watch_ajax\nDisallow: /watch_fragments_ajax\nDisallow: /watch_popup\nDisallow: /watch_queue_ajax\nDisallow: /youtubei/\n\nAllow: /apple-app-site-association\nAllow: /.well-known/*\n","agents":{"GPTBot":true,"OAI-SearchBot":true,"ChatGPT-User":true,"ClaudeBot":true,"Claude-SearchBot":true,"PerplexityBot":true,"Google-Extended":true,"Applebot-Extended":true,"CCBot":true,"Bytespider":true,"Meta-ExternalAgent":true,"Amazonbot":true}},"sitemap":{"status":200,"content_type":"text/html","bytes":889940,"sha256":"ae015605548e7b82cf42674bc21797a5ebe8796f9f0734402067c9d441f744b4","final_host":"www.youtube.com","ms":146,"url":"https://youtu.be/sitemap.xml","declared":false,"xml":false},"files":{"llms_txt":{"status":200,"content_type":"text/html","bytes":890627,"sha256":"e2ceab588dbbb394407ee07d5768307a8456c45e0ca7a25d99b778947c262091","final_host":"www.youtube.com","ms":198,"path":"/llms.txt","html":true,"json":null,"present":false},"agents_md":{"status":200,"content_type":"text/html","bytes":890292,"sha256":"39919bc20ac478ce5d65e0de3d9b899a4400cf5c069c0a863ef9929ec708ec2f","final_host":"www.youtube.com","ms":173,"path":"/agents.md","html":true,"json":null,"present":false},"agent_skills":{"status":200,"content_type":"text/html","bytes":890991,"sha256":"3a4041167fd113b95303db8713d1e4bc981dc1f132011522f5bb6ef1757447dd","final_host":"www.youtube.com","ms":137,"path":"/.well-known/agent-skills/index.json","html":true,"json":null,"present":false},"mcp_card":{"status":200,"content_type":"text/html","bytes":885197,"sha256":"0f7f1952c2f073325cee398323208e94d56f60ffd30bb50aba75764140a83210","final_host":"www.youtube.com","ms":241,"path":"/.well-known/mcp/server-card.json","html":true,"json":null,"present":false},"api_catalog":{"status":200,"content_type":"text/html","bytes":890872,"sha256":"d99d4c18b1c0974a8dfec6fcd0148aaf295cabd40aea2b9629eff99e6c052aac","final_host":"www.youtube.com","ms":178,"path":"/.well-known/api-catalog","html":true,"json":null,"present":false},"a2a_card":{"status":200,"content_type":"text/html","bytes":890414,"sha256":"736fb2351666aebd91d0acd2f0717209c98b798bd5410c50f746b9ad006049ad","final_host":"www.youtube.com","ms":227,"path":"/.well-known/agent-card.json","html":true,"json":null,"present":false}},"rings":{"v":1,"candidates":[],"mcp":[],"a2a":null,"webmcp":{"model_context":false,"bridge":false,"mcp_json":false,"link_rel":false,"any":false},"handshakes":0,"get_path":{"status":200,"content_type":"text/html","ms":146,"error":null},"get_host":{"status":530,"content_type":"text/plain","ms":99,"error":null}},"derived":{"reachable":true,"robots_served":true,"robots_sitemap":false,"content_signal":false,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":[],"blocks_any_ai":false,"files":{"llms_txt":false,"agents_md":false,"agent_skills":false,"mcp_card":false,"api_catalog":false,"a2a_card":false},"any_capability":false,"ai_refused":[],"ai_thin":[],"rings":{"mcp_declared":false,"mcp_live":false,"mcp_auth":false,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":2,"fh":"m.youtube.com","fp_other_host":[]},"derive_v":3,"frame":"top1k","shard":6},{"v":1,"kind":"crawlcheck-census-record","run":14,"domain":"x.com","rank":48,"measured_at":"2026-10-15T05:34:03.814Z","home":{"browser":{"status":200,"content_type":"text/html","bytes":42199,"sha256":"01e0a4090513c18485fc817a1584186bcadb0bf5981a8782800300e2acc45761","final_host":"x.com","ms":186,"identity":"Mobile browser","html":true,"fp":"8b4892564ad225ec"},"gptbot":{"status":402,"content_type":"application/json","bytes":55,"sha256":"35e642f0e1bd426b9d8b0ee0284aefe801873dc13c18819fbeeaf2bf47136664","final_host":"x.com","ms":4,"identity":"GPTBot","html":false},"claudebot":{"status":403,"content_type":null,"bytes":0,"sha256":"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855","final_host":"x.com","ms":134,"identity":"ClaudeBot","html":null},"oaisearch":{"status":402,"content_type":"application/json","bytes":55,"sha256":"35e642f0e1bd426b9d8b0ee0284aefe801873dc13c18819fbeeaf2bf47136664","final_host":"x.com","ms":5,"identity":"OAI-SearchBot","html":false},"perplexity":{"status":402,"content_type":"application/json","bytes":55,"sha256":"35e642f0e1bd426b9d8b0ee0284aefe801873dc13c18819fbeeaf2bf47136664","final_host":"x.com","ms":4,"identity":"PerplexityBot","html":false}},"robots":{"status":200,"content_type":"text/plain","bytes":2678,"sha256":"14a72e33b3e9af07e1e8f5f645bfa00ed3fe9f9c87249d861d369d33d3f3ddc6","final_host":"x.com","ms":124,"html":false,"text":"# Google / Bing Search Engine Robots\n# ==================================\n# Shared group so Googlebot and Bingbot always get the same rules.\nUser-agent: Googlebot\nUser-agent: Bingbot\n\nAllow: /*?s=\nAllow: /*?t=\nAllow: /*?ref_src=\nAllow: /hashtag/*?src=\nAllow: /search?q=%23\nAllow: /i/api/\n# Same length as the /*/likes and /*/media rules; RFC 9309 breaks the tie in\n# favor of Allow, so /i/api/ fetches stay crawlable.\nAllow: /i/api/*\nDisallow: /*?lang=en-ss\nAllow: /*?lang=\nDisallow: /search/realtime\nDisallow: /search/users\nDisallow: /search/*/grid\n\nDisallow: /*/analytics\nDisallow: /*/followers\nDisallow: /*/following\nDisallow: /*/verified_followers\n\nDisallow: /account/deactivated\nDisallow: /settings/deactivated\n\n# Only `*` and `$` are metacharacters here (RFC 9309); regex classes match\n# literally. /photo is anchored so /status/*/photo/1 media pages stay open.\nDisallow: /*/likes\nDisallow: /*/likes?\nDisallow: /*/retweets\nDisallow: /*/retweets?\nDisallow: /*/media\nDisallow: /*/media?\nDisallow: /*/photo$\nDisallow: /*/photo?\nAllow: /*?\n\nUser-agent: facebookexternalhit\n\nAllow: /*?lang=\nAllow: /*?s=\nAllow: /*?t=\nAllow: /*?ref_src=\nAllow: /hashtag/*?src=\nAllow: /search?*cashtagRestId=\nAllow: /i/api/\nAllow: /i/api/*\nDisallow: /search?q=\nDisallow: /search/realtime\nDisallow: /search/users\nDisallow: /search/*/grid\n\nDisallow: /*?\nDisallow: /*/followers\nDisallow: /*/following\nDisallow: /*/verified_followers\n\nDisallow: /account/deactivated\nDisallow: /settings/deactivated\n\nDisallow: /*/likes\nDisallow: /*/likes?\nDisallow: /*/retweets\nDisallow: /*/retweets?\nDisallow: /*/media\nDisallow: /*/media?\nDisallow: /*/photo$\nDisallow: /*/photo?\n\nUser-Agent: Google-Extended\nDisallow: *\n\nUser-Agent: FacebookBot\nDisallow: *\n\nUser-agent: Discordbot\nDisallow: *\n\n# Meta AI / product crawlers (product tokens; versions like /1.1 are ignored\n# under RFC 9309 token matching).\nUser-agent: meta-webindexer\nDisallow: *\n\nUser-agent: meta-externalagent\nDisallow: *\n\nUser-agent: meta-externalads\nDisallow: *\n\nUser-agent: meta-externalfetcher\nDisallow: *\n\n# Every bot that might possibly read and respect this file\n# ========================================================\nUser-agent: *\nDisallow: /\n\n\n# WHAT-4882 - Keep notification-email links (/i/u) out of search results.\n# Named crawlers stay un-blocked on purpose: they must crawl /i/u to see its\n# X-Robots-Tag noindex (the robots.txt Noindex directive died in 2019).\nDisallow: /i/u\n\n# Wait 1 second between successive requests. See ONBOARD-2698 for details.\nCrawl-delay: 1\n\n# Independent of user agent. Links in the sitemap are full URLs using https://\n# and need to match the protocol of the sitemap.\nSitemap: https://x.com/sitemap.xml\n","agents":{"GPTBot":false,"OAI-SearchBot":false,"ChatGPT-User":false,"ClaudeBot":false,"Claude-SearchBot":false,"PerplexityBot":false,"Google-Extended":false,"Applebot-Extended":false,"CCBot":false,"Bytespider":false,"Meta-ExternalAgent":false,"Amazonbot":false}},"sitemap":{"status":404,"content_type":"text/html","bytes":36630,"sha256":"cb8e00052f7a661da8725c90ba7cb402c2a7ae7d28df85008f98695a0ce42f3a","final_host":"x.com","ms":211,"url":"https://x.com/sitemap.xml","declared":true,"xml":false},"files":{"llms_txt":{"status":404,"content_type":"text/html","bytes":36612,"sha256":"a48d12936119b82134869d6858379abc419fd70a5f06be732bdf2639118a8dd5","final_host":"x.com","ms":917,"path":"/llms.txt","html":true,"json":null,"present":false},"agents_md":{"status":404,"content_type":"text/html","bytes":36618,"sha256":"5b949e6677939730abe5da261f40d00465049e058b338795985d6ea02a3af5cf","final_host":"x.com","ms":230,"path":"/agents.md","html":true,"json":null,"present":false},"agent_skills":{"status":404,"content_type":"application/json","bytes":68,"sha256":"643770b397d393e8b83954adc74ed48dd2e27ab3abab48f2b79472f07808eeb0","final_host":"x.com","ms":475,"path":"/.well-known/agent-skills/index.json","html":false,"json":null,"present":false},"mcp_card":{"status":404,"content_type":"application/json","bytes":68,"sha256":"643770b397d393e8b83954adc74ed48dd2e27ab3abab48f2b79472f07808eeb0","final_host":"x.com","ms":142,"path":"/.well-known/mcp/server-card.json","html":false,"json":null,"present":false},"api_catalog":{"status":404,"content_type":"text/html","bytes":3656,"sha256":"b4056ca07cbe8a3c71d16206c45c9d5515c74912647eb67100390f95a8a88600","final_host":"x.com","ms":307,"path":"/.well-known/api-catalog","html":true,"json":null,"present":false},"a2a_card":{"status":404,"content_type":"application/json","bytes":68,"sha256":"643770b397d393e8b83954adc74ed48dd2e27ab3abab48f2b79472f07808eeb0","final_host":"x.com","ms":216,"path":"/.well-known/agent-card.json","html":false,"json":null,"present":false}},"rings":{"v":1,"candidates":[],"mcp":[],"a2a":null,"webmcp":{"model_context":false,"bridge":false,"mcp_json":false,"link_rel":false,"any":false},"handshakes":0,"get_path":{"status":200,"content_type":"text/html","ms":883,"error":null},"get_host":{"status":530,"content_type":"text/plain","ms":109,"error":null}},"derived":{"reachable":true,"robots_served":true,"robots_sitemap":true,"content_signal":false,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":["GPTBot","OAI-SearchBot","ChatGPT-User","ClaudeBot","Claude-SearchBot","PerplexityBot","Google-Extended","Applebot-Extended","CCBot","Bytespider","Meta-ExternalAgent","Amazonbot"],"blocks_any_ai":true,"files":{"llms_txt":false,"agents_md":false,"agent_skills":false,"mcp_card":false,"api_catalog":false,"a2a_card":false},"any_capability":false,"ai_refused":["gptbot","claudebot","oaisearch","perplexity"],"ai_thin":[],"rings":{"mcp_declared":false,"mcp_live":false,"mcp_auth":false,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":2,"fh":"x.com","fp_other_host":[],"fp":"8b4892564ad225ec","fp_bits":{}},"derive_v":3,"frame":"top1k","shard":7},{"v":1,"kind":"crawlcheck-census-record","run":14,"domain":"wordpress.org","rank":49,"measured_at":"2026-10-15T05:26:03.595Z","home":{"browser":{"status":200,"content_type":"text/html","bytes":177653,"sha256":"8e38cac8d87df7b29ba974aedce0a6d5b30d642f285b7c1b11fe5aff726e07fc","final_host":"wordpress.org","ms":580,"identity":"Mobile browser","html":true,"fp":"39fbafd592739fe0"},"gptbot":{"status":200,"content_type":"text/html","bytes":177653,"sha256":"4f86c68d9e409a7106e0ced7b9b471b9d182ee9c90c1abe8fd9c684df14f4eac","final_host":"wordpress.org","ms":872,"identity":"GPTBot","html":true,"fp":"39fbafd592739fe0"},"claudebot":{"status":200,"content_type":"text/html","bytes":177653,"sha256":"8e38cac8d87df7b29ba974aedce0a6d5b30d642f285b7c1b11fe5aff726e07fc","final_host":"wordpress.org","ms":708,"identity":"ClaudeBot","html":true,"fp":"39fbafd592739fe0"},"oaisearch":{"status":200,"content_type":"text/html","bytes":177653,"sha256":"8e38cac8d87df7b29ba974aedce0a6d5b30d642f285b7c1b11fe5aff726e07fc","final_host":"wordpress.org","ms":243,"identity":"OAI-SearchBot","html":true,"fp":"39fbafd592739fe0"},"perplexity":{"status":200,"content_type":"text/html","bytes":177653,"sha256":"8e38cac8d87df7b29ba974aedce0a6d5b30d642f285b7c1b11fe5aff726e07fc","final_host":"wordpress.org","ms":195,"identity":"PerplexityBot","html":true,"fp":"39fbafd592739fe0"}},"robots":{"status":200,"content_type":"text/plain","bytes":976,"sha256":"65185a701db3ab195bcb2f86657300e13153f758af110ac0ffeeea5dc19ab5dc","final_host":"wordpress.org","ms":488,"html":false,"text":"Sitemap: https://wordpress.org/sitemap.xml\nSitemap: https://wordpress.org/news-sitemap.xml\nSitemap: https://wordpress.org/themes/sitemap.xml\nSitemap: https://wordpress.org/plugins/sitemap.xml\nSitemap: https://wordpress.org/news/sitemap.xml\nSitemap: https://wordpress.org/showcase/sitemap.xml\nSitemap: https://wordpress.org/documentation/sitemap.xml\nSitemap: https://wordpress.org/patterns/sitemap.xml\nSitemap: https://wordpress.org/photos/sitemap.xml\nUser-agent: *\nDisallow: /wp-admin/\nAllow: /wp-admin/admin-ajax.php\nAllow: /wp-admin/load-scripts.php\nAllow: /wp-admin/load-styles.php\n\nUser-agent: *\nDisallow: /search\nDisallow: /?s=\n\nUser-agent: GPTBot\nAllow: /\n\nUser-agent: ClaudeBot\nAllow: /\n\nUser-agent: anthropic-ai\nAllow: /\n\nUser-agent: Google-Extended\nAllow: /\n\nUser-agent: Applebot-Extended\nAllow: /\n\nUser-agent: PerplexityBot\nAllow: /\n\nUser-agent: Bytespider\nAllow: /\n\nUser-agent: CCBot\nAllow: /\n\nUser-agent: Copilot\nAllow: /\n\nUser-agent: *\nDisallow: /plugins/search/\n","agents":{"GPTBot":true,"OAI-SearchBot":true,"ChatGPT-User":true,"ClaudeBot":true,"Claude-SearchBot":true,"PerplexityBot":true,"Google-Extended":true,"Applebot-Extended":true,"CCBot":true,"Bytespider":true,"Meta-ExternalAgent":true,"Amazonbot":true}},"sitemap":{"status":200,"content_type":"text/xml","bytes":631,"sha256":"479014f463805774f733ee235f7bcc7ca0c303186ad67bcf4be583ebaafe1b09","final_host":"wordpress.org","ms":287,"url":"https://wordpress.org/sitemap.xml","declared":true,"xml":true},"files":{"llms_txt":{"status":200,"content_type":"text/plain","bytes":5765,"sha256":"731a4c979c79b5a350375f8b089fc26ed70b20d395de39ad305b716de233b4af","final_host":"wordpress.org","ms":173,"path":"/llms.txt","html":false,"json":null,"present":true},"agents_md":{"status":404,"content_type":"text/html","bytes":124625,"sha256":"b65f28f2b18a072eb0eeb29d40f58020ee3939f4d41c91c71fff7a6793d5d680","final_host":"wordpress.org","ms":347,"path":"/agents.md","html":true,"json":null,"present":false},"agent_skills":{"status":404,"content_type":"text/html","bytes":124677,"sha256":"f230a26133dd78e232b259c248250562765776d2413188137b3293ff6cd1854e","final_host":"wordpress.org","ms":422,"path":"/.well-known/agent-skills/index.json","html":true,"json":null,"present":false},"mcp_card":{"status":404,"content_type":"text/html","bytes":124671,"sha256":"2f460ea1f550627ff061f18c0946904d36217195985e884556cb9bc9300252af","final_host":"wordpress.org","ms":375,"path":"/.well-known/mcp/server-card.json","html":true,"json":null,"present":false},"api_catalog":{"status":404,"content_type":"text/html","bytes":124653,"sha256":"6b8b5dd5a691eacce52babd47b5511d5245d8e9316f428361d4b0692c96d6f50","final_host":"wordpress.org","ms":341,"path":"/.well-known/api-catalog","html":true,"json":null,"present":false},"a2a_card":{"status":404,"content_type":"text/html","bytes":124661,"sha256":"9fa7a4930b34ca56d07c9da7dae2f4b631d515173c479b031a0fddafe5b0cd98","final_host":"wordpress.org","ms":353,"path":"/.well-known/agent-card.json","html":true,"json":null,"present":false}},"rings":{"v":1,"candidates":[],"mcp":[],"a2a":null,"webmcp":{"model_context":false,"bridge":false,"mcp_json":false,"link_rel":false,"any":false},"handshakes":0,"get_path":{"status":404,"content_type":"text/html","ms":339,"error":null},"get_host":{"status":200,"content_type":"text/html","ms":920,"error":null}},"derived":{"reachable":true,"robots_served":true,"robots_sitemap":true,"content_signal":false,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":[],"blocks_any_ai":false,"files":{"llms_txt":true,"agents_md":false,"agent_skills":false,"mcp_card":false,"api_catalog":false,"a2a_card":false},"any_capability":false,"ai_refused":[],"ai_thin":[],"rings":{"mcp_declared":false,"mcp_live":false,"mcp_auth":false,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":2,"fh":"wordpress.org","fp_other_host":[],"fp":"39fbafd592739fe0","fp_bits":{"gptbot":0,"claudebot":0,"oaisearch":0,"perplexity":0}},"derive_v":3,"frame":"top1k","shard":0},{"v":1,"kind":"crawlcheck-census-record","run":14,"domain":"digicert.com","rank":50,"measured_at":"2026-10-15T05:27:03.729Z","home":{"browser":{"status":200,"content_type":"text/html","bytes":69354,"sha256":"15039f50f48769b5f625ae4b2c326ea0b9f95ca4316a9e354ba61a3b1728b0e5","final_host":"www.digicert.com","ms":727,"identity":"Mobile browser","html":true,"fp":"9119de7a9b99bf96"},"gptbot":{"status":200,"content_type":"text/html","bytes":69354,"sha256":"77957134565f62457e28d4733908cab9fae8228021d9da39fb930a2e930372a0","final_host":"www.digicert.com","ms":718,"identity":"GPTBot","html":true,"fp":"9119de7a9b99bf96"},"claudebot":{"status":200,"content_type":"text/html","bytes":69220,"sha256":"b856a6992d51842d60dba6715f5a911cbd18ff8572021fc1f1e947fd83888193","final_host":"www.digicert.com","ms":694,"identity":"ClaudeBot","html":true,"fp":"9119de7a9b99bf96"},"oaisearch":{"status":200,"content_type":"text/html","bytes":69220,"sha256":"b856a6992d51842d60dba6715f5a911cbd18ff8572021fc1f1e947fd83888193","final_host":"www.digicert.com","ms":601,"identity":"OAI-SearchBot","html":true,"fp":"9119de7a9b99bf96"},"perplexity":{"status":200,"content_type":"text/html","bytes":924,"sha256":"b7daed2ede46fcad11594a6759fb77461906030c0440608217ba2f9c23d9787f","final_host":"www.digicert.com","ms":550,"identity":"PerplexityBot","html":true}},"robots":{"status":200,"content_type":"text/plain","bytes":2657,"sha256":"a139a54387d10bfd57f5223202cb56558934677d20460b789d4445d2549537f7","final_host":"www.digicert.com","ms":799,"html":false,"text":"# Welcome to DigiCert\r\n# Think you have what it takes to join the best team in the world?\r\n# Check out our careers while you're here\r\n# https://www.digicert.com/careers\r\n# Or check out our YouTube channel for our latest\r\n# https://www.youtube.com/user/DigiCertSSL\r\n\r\nUser-agent: *\r\nDisallow: /account/sso/idp\r\nDisallow: */test/\r\nDisallow: */ni/\r\nDisallow: */gated-content-pdfs/\r\nDisallow: /content/dam/digicert/pdfs/ni/\r\nDisallow: /account/saml-guest-request/idp\r\nDisallow: /webinars/*\r\nDisallow: /seals/\r\nDisallow: /link/\r\nDisallow: /secure/*\r\nDisallow: /services/*\r\nDisallow: /in-console-ssm\r\nDisallow: /jp/sid-partner*\r\nDisallow: /ppc/\r\nDisallow: /ru/\r\nDisallow: /ar/\r\nDisallow: /dc/blog/*\r\nDisallow: /dc/news/*\r\nDisallow: /author/*\r\nDisallow: /tag/*\r\nDisallow: /clients/*\r\nDisallow: /voucher/*\r\nDisallow: /bios*\r\nDisallow: /direct/*\r\nDisallow: /*thank\r\nDisallow: /*.xls$\r\nDisallow: /*.docx$\r\nDisallow: /language-masters/*\r\nDisallow: /link*\r\nDisallow: /*feed\r\n\r\nAllow: /*.js\r\nAllow: /*.css\r\n\r\n# AI / answer-engine crawlers\r\n\r\nUser-agent: GPTBot\r\nAllow: /\r\n\r\nUser-agent: OAI-SearchBot\r\nAllow: /\r\n\r\nUser-agent: ChatGPT-User\r\nAllow: /\r\n\r\nUser-agent: PerplexityBot\r\nAllow: /\r\n\r\nUser-agent: Perplexity-User\r\nAllow: /\r\n\r\nUser-agent: ClaudeBot\r\nAllow: /\r\n\r\nUser-agent: Claude-SearchBot\r\nAllow: /\r\n\r\nUser-agent: Claude-User\r\nAllow: /\r\n\r\nUser-agent: Googlebot-Image\r\nAllow: /\r\n\r\nUser-agent: AdsBot-Google\r\nAllow: /\r\n\r\nUser-agent: Google-Extended\r\nAllow: /\r\n\r\nSitemap: https://www.digicert.com/sitemap.xml\r\nSitemap: https://www.digicert.com/cn/sitemap.xml\r\nSitemap: https://www.digicert.com/de/sitemap.xml\r\nSitemap: https://www.digicert.com/es/sitemap.xml\r\nSitemap: https://www.digicert.com/fr/sitemap.xml\r\nSitemap: https://www.digicert.com/it/sitemap.xml\r\nSitemap: https://www.digicert.com/jp/sitemap.xml\r\nSitemap: https://www.digicert.com/kr/sitemap.xml\r\nSitemap: https://www.digicert.com/nl/sitemap.xml\r\nSitemap: https://www.digicert.com/tw/sitemap.xml\r\nSitemap: https://www.digicert.com/pt/sitemap.xml\r\nSitemap: https://www.digicert.com/faq/sitemap.xml\r\nSitemap: https://www.digicert.com/news/sitemap.xml\r\nSitemap: https://www.digicert.com/blog/sitemap.xml\r\nSitemap: https://www.digicert.com/eds-sitemap.xml\r\n\r\n# Bot Requests\r\n\r\nUser-agent: XoviBot\r\nDisallow: /\r\n\r\nUser-agent: BLEXBot\r\nDisallow: /\r\n\r\nUser-agent: DotBot\r\nDisallow: /\r\n\r\nUser-agent: MJ12bot\r\nDisallow: /\r\n\r\nUser-agent: Xenu's\r\nDisallow: /\r\n\r\nUser-agent: Xenu's Link Sleuth 1.1c\r\nDisallow: /\r\n\r\nUser-agent: Zeus\r\nDisallow: /\r\n\r\nUser-agent: Zeus 32297 Webster Pro V2.9 Win32\r\nDisallow: /\r\n\r\nUser-agent: Updownerbot\r\nDisallow: /\r\n\r\nUser-agent: Blekkobot\r\nDisallow: /\r\n\r\nUser-agent: 008\r\nDisallow: /","agents":{"GPTBot":true,"OAI-SearchBot":true,"ChatGPT-User":true,"ClaudeBot":true,"Claude-SearchBot":true,"PerplexityBot":true,"Google-Extended":true,"Applebot-Extended":true,"CCBot":true,"Bytespider":true,"Meta-ExternalAgent":true,"Amazonbot":true}},"sitemap":{"status":200,"content_type":"text/html","bytes":925,"sha256":"5c5aab6f8045a6adbef3374ddf9436e8075fcdc920b1e69a0f7f56a8aa694677","final_host":"www.digicert.com","ms":28,"url":"https://www.digicert.com/sitemap.xml","declared":true,"xml":false},"files":{"llms_txt":{"status":200,"content_type":"text/plain","bytes":8395,"sha256":"02169ec84a351a1d16d7d35ae9186f3edaa67dc827c2a7b55f9cea8f32abd439","final_host":"www.digicert.com","ms":986,"path":"/llms.txt","html":false,"json":null,"present":true},"agents_md":{"status":200,"content_type":"text/html","bytes":929,"sha256":"6d4831cc83e5cf0fa1540a3f35c77d5a7c622d08e160fc5ff83a7a71d15ce7f9","final_host":"www.digicert.com","ms":139,"path":"/agents.md","html":true,"json":null,"present":false},"agent_skills":{"status":200,"content_type":"text/html","bytes":929,"sha256":"6ec192ccf059561e465952bb40fd30f9c6bc776b3b13c1f8f06f5ed0537228e2","final_host":"www.digicert.com","ms":136,"path":"/.well-known/agent-skills/index.json","html":true,"json":null,"present":false},"mcp_card":{"status":200,"content_type":"text/html","bytes":931,"sha256":"e0a0a91a26823d3a0421c62ffcb441852283fff60e25783f3e79c499db125363","final_host":"www.digicert.com","ms":136,"path":"/.well-known/mcp/server-card.json","html":true,"json":null,"present":false},"api_catalog":{"status":200,"content_type":"text/html","bytes":925,"sha256":"6229aaaa336595faf1f048a831d9e6fe8c8b8b8cc858a68c6a95bcba97562df2","final_host":"www.digicert.com","ms":139,"path":"/.well-known/api-catalog","html":true,"json":null,"present":false},"a2a_card":{"status":200,"content_type":"text/html","bytes":925,"sha256":"486a3c1c82d07ac3166bfca964ea2e61d2118dd2b46ef55c0d7040cf062283ad","final_host":"www.digicert.com","ms":138,"path":"/.well-known/agent-card.json","html":true,"json":null,"present":false}},"rings":{"v":1,"candidates":[],"mcp":[],"a2a":null,"webmcp":{"model_context":false,"bridge":false,"mcp_json":false,"link_rel":false,"any":false},"handshakes":0,"get_path":{"status":200,"content_type":"text/html","ms":499,"error":null},"get_host":{"status":404,"content_type":null,"ms":658,"error":null}},"derived":{"reachable":true,"robots_served":true,"robots_sitemap":true,"content_signal":false,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":[],"blocks_any_ai":false,"files":{"llms_txt":true,"agents_md":false,"agent_skills":false,"mcp_card":false,"api_catalog":false,"a2a_card":false},"any_capability":false,"ai_refused":[],"ai_thin":["perplexity"],"rings":{"mcp_declared":false,"mcp_live":false,"mcp_auth":false,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":2,"fh":"www.digicert.com","fp_other_host":[],"fp":"9119de7a9b99bf96","fp_bits":{"gptbot":0,"claudebot":0,"oaisearch":0}},"derive_v":3,"frame":"top1k","shard":1},{"v":1,"kind":"crawlcheck-census-record","run":14,"domain":"skype.com","rank":51,"measured_at":"2026-10-15T05:28:03.923Z","home":{"browser":{"status":200,"content_type":"text/html","bytes":61987,"sha256":"aaa8993334881a56d2d53be76501e4d8d36881e8bb0185ba9c9a7002d00db9ab","final_host":"teams.live.com","ms":719,"identity":"Mobile browser","html":true,"fp":"f13ce2b0825475b1"},"gptbot":{"status":200,"content_type":"text/html","bytes":61987,"sha256":"56cac325ff5c87740ae20e0d8c866708d8ec3f0bf149d51dff0515429218b111","final_host":"teams.live.com","ms":553,"identity":"GPTBot","html":true,"fp":"f13ce2b0825475b1"},"claudebot":{"status":200,"content_type":"text/html","bytes":61987,"sha256":"979fb28eb159da6052c660fa44f954381418fe64c9672428e6a75f4fbaea48c6","final_host":"teams.live.com","ms":541,"identity":"ClaudeBot","html":true,"fp":"f13ce2b0825475b1"},"oaisearch":{"status":200,"content_type":"text/html","bytes":61987,"sha256":"ae2ddbbd46358adfb7b7ec26badf81d0f4cac490d46cd22d668a0b432ae7689b","final_host":"teams.live.com","ms":498,"identity":"OAI-SearchBot","html":true,"fp":"f13ce2b0825475b1"},"perplexity":{"status":200,"content_type":"text/html","bytes":61987,"sha256":"9e80fd605e54a30afa712cadb41a38a311da5aeff41008d9396585620f67fb09","final_host":"teams.live.com","ms":585,"identity":"PerplexityBot","html":true,"fp":"f13ce2b0825475b1"}},"robots":{"status":200,"content_type":"text/html","bytes":61987,"sha256":"9c69feeb47f1281a7da3674411b4b1c233348d98a67e7eecd052b181f6d656e2","final_host":"teams.live.com","ms":5690,"html":true,"text":null,"agents":null},"sitemap":{"status":200,"content_type":"text/html","bytes":61987,"sha256":"192f9e5ed793fa82e3af757cb0036752495c0f9c2e01e22f09c01e1b8f201455","final_host":"teams.live.com","ms":222,"url":"https://skype.com/sitemap.xml","declared":false,"xml":false},"files":{"llms_txt":{"status":200,"content_type":"text/html","bytes":61987,"sha256":"8d3ac30d2108279c36aaf9b8201162b26ba3d77f798d6dc388d50248a241b1cb","final_host":"teams.live.com","ms":89,"path":"/llms.txt","html":true,"json":null,"present":false},"agents_md":{"status":200,"content_type":"text/html","bytes":61987,"sha256":"8ac50300b12cb890ebc34c30494b91a75493d01dd6a9f2d59a40312f8dfd7663","final_host":"teams.live.com","ms":190,"path":"/agents.md","html":true,"json":null,"present":false},"agent_skills":{"status":200,"content_type":"text/html","bytes":61987,"sha256":"8cc21a773c43ba40e41a8ed45b2e8b8c79d1df531438f7e5411a830a352097b0","final_host":"teams.live.com","ms":964,"path":"/.well-known/agent-skills/index.json","html":true,"json":null,"present":false},"mcp_card":{"status":200,"content_type":"text/html","bytes":61987,"sha256":"51899d9908cd4a92936a4899b853bff2933ab2e24397669835da1cf1cc4e7e1a","final_host":"teams.live.com","ms":192,"path":"/.well-known/mcp/server-card.json","html":true,"json":null,"present":false},"api_catalog":{"status":200,"content_type":"text/html","bytes":61987,"sha256":"906fdf8fd3458344cd6c9e0e7715d824c8765c0423a6d65d263382cf6718c1ea","final_host":"teams.live.com","ms":303,"path":"/.well-known/api-catalog","html":true,"json":null,"present":false},"a2a_card":{"status":200,"content_type":"text/html","bytes":61987,"sha256":"9dedc7b6166892fd57ed7163f1cc738d802ce24d3abf95690d1c78e7f11d31a0","final_host":"teams.live.com","ms":106,"path":"/.well-known/agent-card.json","html":true,"json":null,"present":false}},"rings":{"v":1,"candidates":[],"mcp":[],"a2a":null,"webmcp":{"model_context":false,"bridge":false,"mcp_json":false,"link_rel":false,"any":false},"handshakes":0,"get_path":{"status":200,"content_type":"text/html","ms":163,"error":null},"get_host":{"status":530,"content_type":"text/plain","ms":65,"error":null}},"derived":{"reachable":true,"robots_served":false,"robots_sitemap":false,"content_signal":false,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":[],"blocks_any_ai":false,"files":{"llms_txt":false,"agents_md":false,"agent_skills":false,"mcp_card":false,"api_catalog":false,"a2a_card":false},"any_capability":false,"ai_refused":[],"ai_thin":[],"rings":{"mcp_declared":false,"mcp_live":false,"mcp_auth":false,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":2,"fh":"teams.live.com","fp_other_host":[],"fp":"f13ce2b0825475b1","fp_bits":{"gptbot":0,"claudebot":0,"oaisearch":0,"perplexity":0}},"derive_v":3,"frame":"top1k","shard":2},{"v":1,"kind":"crawlcheck-census-record","run":14,"domain":"icloud.com","rank":52,"measured_at":"2026-10-15T05:29:03.596Z","home":{"browser":{"status":200,"content_type":"text/html","bytes":31328,"sha256":"5b599fd901cbd3c4b00c14c97a83bce4118980b75447d499aa38ac16e21c829a","final_host":"www.icloud.com","ms":347,"identity":"Mobile browser","html":true},"gptbot":{"status":200,"content_type":"text/html","bytes":31328,"sha256":"5b599fd901cbd3c4b00c14c97a83bce4118980b75447d499aa38ac16e21c829a","final_host":"www.icloud.com","ms":158,"identity":"GPTBot","html":true},"claudebot":{"status":200,"content_type":"text/html","bytes":31328,"sha256":"5b599fd901cbd3c4b00c14c97a83bce4118980b75447d499aa38ac16e21c829a","final_host":"www.icloud.com","ms":67,"identity":"ClaudeBot","html":true},"oaisearch":{"status":200,"content_type":"text/html","bytes":31328,"sha256":"5b599fd901cbd3c4b00c14c97a83bce4118980b75447d499aa38ac16e21c829a","final_host":"www.icloud.com","ms":69,"identity":"OAI-SearchBot","html":true},"perplexity":{"status":200,"content_type":"text/html","bytes":31328,"sha256":"5b599fd901cbd3c4b00c14c97a83bce4118980b75447d499aa38ac16e21c829a","final_host":"www.icloud.com","ms":71,"identity":"PerplexityBot","html":true}},"robots":{"status":200,"content_type":"text/plain","bytes":1289,"sha256":"7c0da5b622a8703a69e156d3df581d1a2c5d4106a0da2e85c4db8f18c5a9e9c8","final_host":"www.icloud.com","ms":499,"html":false,"text":"\nSitemap: https://www.icloud.com/sitemap.xml\nSitemap: https://www.icloud.com/sitemap-info.xml\n\nUser-agent: *\n\nDisallow: /unsupported_browser\nDisallow: /unsupported_mobile\nDisallow: /enable_cookies\nDisallow: /maintenance\nDisallow: /not_found\nDisallow: /unavailable\nDisallow: /attachment/\nDisallow: /download/\nDisallow: /invites/\nDisallow: /iw/\nDisallow: /bookmarks_extension/\nDisallow: /family_sharing/\nDisallow: /calendar/eventreply/\nDisallow: /fmf_getting_started/\nDisallow: /fmf_location_update/\nDisallow: /fmf_location_update_unsubscribe_undo/\nDisallow: /fmf_unsubscribe/\nDisallow: /fmf_unsubscribe_error/\nDisallow: /fmf_unsubscribe_success/\nDisallow: /fmf_unsubscribe_undo/\nDisallow: /fmf_unsupported_device/\nDisallow: /photos_get_started/\nDisallow: /sharedalbum_unsubscribe/\nDisallow: /sharedalbum_unsubscribe_error/\nDisallow: /sharedalbum_unsubscribe_undo/\nDisallow: /sharedalbum_unsupported_device/\nDisallow: /photostream_unsubscribe/\nDisallow: /iclouddrive/\nDisallow: /notes/\nDisallow: /photos/\nDisallow: /reminders/\nDisallow: /share/\nDisallow: /pages/\nDisallow: /numbers/\nDisallow: /keynote/\nDisallow: /keynote-live/\nDisallow: /sharedalbum/\nDisallow: /ca/ics/\nDisallow: /shared/\n\nUser-agent: Googlebot\nAllow: /\n\nUser-agent: facebookexternalhit\nAllow: /invites/\nAllow: /opengraph/\n","agents":{"GPTBot":true,"OAI-SearchBot":true,"ChatGPT-User":true,"ClaudeBot":true,"Claude-SearchBot":true,"PerplexityBot":true,"Google-Extended":true,"Applebot-Extended":true,"CCBot":true,"Bytespider":true,"Meta-ExternalAgent":true,"Amazonbot":true}},"sitemap":{"status":200,"content_type":"text/xml","bytes":3442,"sha256":"b73212a376f2901d9ff3ec1bb8d59999116437dc44b19691c5ccc0fbcce60430","final_host":"www.icloud.com","ms":217,"url":"https://www.icloud.com/sitemap.xml","declared":true,"xml":true},"files":{"llms_txt":{"status":404,"content_type":"text/html","bytes":3250,"sha256":"60b391103f3f6b7ac9e37bd76d1ad2ac9c0b3fa775afb0ed276aa94908251b45","final_host":"www.icloud.com","ms":145,"path":"/llms.txt","html":true,"json":null,"present":false},"agents_md":{"status":404,"content_type":"text/html","bytes":3250,"sha256":"60b391103f3f6b7ac9e37bd76d1ad2ac9c0b3fa775afb0ed276aa94908251b45","final_host":"www.icloud.com","ms":157,"path":"/agents.md","html":true,"json":null,"present":false},"agent_skills":{"status":404,"content_type":"text/html","bytes":3250,"sha256":"60b391103f3f6b7ac9e37bd76d1ad2ac9c0b3fa775afb0ed276aa94908251b45","final_host":"www.icloud.com","ms":168,"path":"/.well-known/agent-skills/index.json","html":true,"json":null,"present":false},"mcp_card":{"status":404,"content_type":"text/html","bytes":3250,"sha256":"60b391103f3f6b7ac9e37bd76d1ad2ac9c0b3fa775afb0ed276aa94908251b45","final_host":"www.icloud.com","ms":235,"path":"/.well-known/mcp/server-card.json","html":true,"json":null,"present":false},"api_catalog":{"status":404,"content_type":"text/html","bytes":3250,"sha256":"60b391103f3f6b7ac9e37bd76d1ad2ac9c0b3fa775afb0ed276aa94908251b45","final_host":"www.icloud.com","ms":235,"path":"/.well-known/api-catalog","html":true,"json":null,"present":false},"a2a_card":{"status":404,"content_type":"text/html","bytes":3250,"sha256":"60b391103f3f6b7ac9e37bd76d1ad2ac9c0b3fa775afb0ed276aa94908251b45","final_host":"www.icloud.com","ms":239,"path":"/.well-known/agent-card.json","html":true,"json":null,"present":false}},"rings":{"v":1,"candidates":[],"mcp":[],"a2a":null,"webmcp":{"model_context":false,"bridge":false,"mcp_json":false,"link_rel":false,"any":false},"handshakes":0,"get_path":{"status":404,"content_type":"text/html","ms":194,"error":null},"get_host":{"status":530,"content_type":"text/plain","ms":49,"error":null}},"derived":{"reachable":true,"robots_served":true,"robots_sitemap":true,"content_signal":false,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":[],"blocks_any_ai":false,"files":{"llms_txt":false,"agents_md":false,"agent_skills":false,"mcp_card":false,"api_catalog":false,"a2a_card":false},"any_capability":false,"ai_refused":[],"ai_thin":[],"rings":{"mcp_declared":false,"mcp_live":false,"mcp_auth":false,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":2,"fh":"www.icloud.com","fp_other_host":[]},"derive_v":3,"frame":"top1k","shard":3},{"v":1,"kind":"crawlcheck-census-record","run":14,"domain":"pinterest.com","rank":53,"measured_at":"2026-10-15T05:30:03.842Z","home":{"browser":{"status":200,"content_type":"text/html","bytes":1203001,"sha256":"47608a5359a9cac4b3665d0c6f20b3acdd243bc579f9adb45fb5e4ee3934bdf6","final_host":"www.pinterest.com","ms":310,"identity":"Mobile browser","html":true,"fp":"414138e1c61d07db"},"gptbot":{"status":403,"content_type":"text/html","bytes":2987,"sha256":"5f4fa7efdbba4d333f1b04a3f235b14a7cf00d40ab49b84f087e2e127703e4b2","final_host":"www.pinterest.com","ms":188,"identity":"GPTBot","html":true},"claudebot":{"status":200,"content_type":"text/html","bytes":1507378,"sha256":"00c2f3b6ca110b1c380e6713384477da2ce1db8eab447201276f8cb9b0a1ef7e","final_host":"www.pinterest.com","ms":255,"identity":"ClaudeBot","html":true},"oaisearch":{"status":403,"content_type":"text/html","bytes":2987,"sha256":"5f9cc8ac22862b1d6c3857ef3d3fd27690e80209a526182b953455e79825b226","final_host":"www.pinterest.com","ms":180,"identity":"OAI-SearchBot","html":true},"perplexity":{"status":403,"content_type":"text/html","bytes":2987,"sha256":"1e0fc40d450c81c4f6912d8e3c716c61825a2c801798b3c7511447b5726d4a9e","final_host":"www.pinterest.com","ms":205,"identity":"PerplexityBot","html":true}},"robots":{"status":200,"content_type":"text/plain","bytes":43476,"sha256":"511e08a0a7fccd95ef0258ac912eda7962816959a7007f5547a3588ca782992f","final_host":"www.pinterest.com","ms":302,"html":false,"text":"# Allowlist your bot: https://help.pinterest.com/bot-submission-form\n#\n# Pinterest is hiring!\n#\n# Learn about the SEO work that we're doing at https://medium.com/@Pinterest_Engineering/demystifying-seo-with-experiments-a183b325cf4c\n#\n# Check out some of our available positions at https://www.pinterestcareers.com/\n\nUser-agent: 2checkout\nUser-agent: AASA-Bot\nUser-agent: AccessibleWebBot\nUser-agent: Adagiobot\nUser-agent: adbeat_bot\nUser-agent: ADmantX\nUser-agent: AdometryBot\nUser-agent: Adsense-Snapshot-Google\nUser-agent: AdsTxtCrawler\nUser-agent: Adyen\nUser-agent: AhrefsBot\nUser-agent: AhrefsSiteAudit\nUser-agent: Akismet\nUser-agent: Algolia Crawler\nUser-agent: AmazonProductDiscovery\nUser-agent: Aolbot-News\nUser-agent: APIs-Google\nUser-agent: AppInsights\nUser-agent: Applebot\nUser-agent: AppleNewsBot\nUser-agent: Archive-It\nUser-agent: archive.org_bot\nUser-agent: ArchiveBot\nUser-agent: ArchiveTeam-ArchiveBot\nUser-agent: AspiegelBot\nUser-agent: Audisto Crawler\nUser-agent: Awario\nUser-agent: Baidu-ADS\nUser-agent: Baiduspider\nUser-agent: Barkrowler\nUser-agent: bingbot\nUser-agent: BingPreview\nUser-agent: bitlybot\nUser-agent: bl.uk_ldfc_bot\nUser-agent: BLP_bbot\nUser-agent: bordermanager\nUser-agent: botify\nUser-agent: BrandVerity\nUser-agent: Bravebot\nUser-agent: BrightEdge Crawler\nUser-agent: BrightEdge-Crawler\nUser-agent: BUbiNG\nUser-agent: CaliperBot\nUser-agent: CapitalOneBot\nUser-agent: Catchpoint\nUser-agent: cf-qualys-scanner\nUser-agent: Chrome-Lighthouse\nUser-agent: CJBot\nUser-agent: ClarityBot\nUser-agent: Clickagy Intelligence Bot v2\nUser-agent: Cloudflare-AI-Search\nUser-agent: CloudflareBrowserRenderingCrawler\nUser-agent: coccocbot\nUser-agent: ContextualBot\nUser-agent: CookieReports\nUser-agent: Coveobot\nUser-agent: CriteoBot\nUser-agent: curly-\nUser-agent: cXensebot\nUser-agent: Datadog\nUser-agent: DataForSeoBot\nUser-agent: Daum\nUser-agent: demandbase\nUser-agent: Discordbot\nUser-agent: DotBot\nUser-agent: DoubleVerifyBot\nUser-agent: Doximity-Pipeline\nUser-agent: Dragonbot\nUser-agent: Dratabot\nUser-agent: DuckAssistBot\nUser-agent: DuckDuckBot\nUser-agent: DuckDuckGo-Favicons-Bot\nUser-agent: DVBot\nUser-agent: eContext\nUser-agent: ecurly Webhook\nUser-agent: EngageBDR\nUser-agent: europarchive.org\nUser-agent: evidon\nUser-agent: ExaleadCloudview\nUser-agent: FacebookBot\nUser-agent: facebookcatalog\nUser-agent: facebookexternalhit\nUser-agent: Facebot\nUser-agent: FastDAST\nUser-agent: FeedBurner\nUser-agent: Feedfetcher-Google\nUser-agent: FlipboardProxy\nUser-agent: FlippBot\nUser-agent: Freshping\nUser-agent: GeedoProductSearch\nUser-agent: Google Favicon\nUser-agent: Google News\nUser-agent: Google Page Speed Insights\nUser-agent: Google Wireless Transcoder\nUser-agent: Google-Apps-Script\nUser-agent: Google-Certificates-Bridge\nUser-agent: Google-CWS\nUser-agent: Google-Display-Ads-Bot\nUser-agent: Google-Firebase\nUser-agent: Google-HotelAdsVerifier\nUser-agent: Google-InspectionTool\nUser-agent: Google-NotebookLM\nUser-agent: Google-PageRenderer\nUser-agent: Google-PhysicalWeb\nUser-agent: Google-Pinpoint\nUser-agent: Google-Read-Aloud\nUser-agent: Google-Safety\nUser-agent: Google-Site-Verification\nUser-agent: Google-speakr\nUser-agent: Google-Structured-Data-Testing\nUser-agent: Google-Trust-Services\nUser-agent: google_partner_monitoring\nUser-agent: GoogleAssociationService\nUser-agent: GoogleImageProxy\nUser-agent: GoogleMessages\nUser-agent: GoogleProducer\nUser-agent: GoogleSecurityScanner\nUser-agent: grafana/synthetic-monitoring\nUser-agent: GroovinaAdsbot\nUser-agent: GuestpostsBot\nUser-agent: heritrix\nUser-agent: Hotjar\nUser-agent: HubSpot\nUser-agent: ia_archiver\nUser-agent: IABot\nUser-agent: IAS-Crawler\nUser-agent: ias_crawler\nUser-agent: ias_wombles\nUser-agent: INA dlweb\nUser-agent: infolink\nUser-agent: InteractiveAdvertisingBureau\nUser-agent: ip-label\nUser-agent: J-BRW\nUser-agent: J-DLC\nUser-agent: James BOT\nUser-agent: JobRapido\nUser-agent: Jooblebot\nUser-agent: Jumio\nUser-agent: kakaotalk-scrap\nUser-agent: KargoBot-Artemis\nUser-agent: Leikibot\nUser-agent: LiftoffDroid\nUser-agent: line-poker\nUser-agent: Linguee Bot\nUser-agent: linkchecker.pro\nUser-agent: LOCKSS cache\nUser-agent: LogicMonitor SiteMonitor\nUser-agent: Lyonl\nUser-agent: magpie-crawler\nUser-agent: Mastodon\nUser-agent: MaxPoint Bot\nUser-agent: MBCrawler\nUser-agent: mediaboardbot\nUser-agent: MediaMonitoringBot\nUser-agent: Mediatoolkitbot\nUser-agent: MeltwaterNews\nUser-agent: meta-externalfetcher\nUser-agent: Metorik\nUser-agent: MgidBot\nUser-agent: MicrosoftPreview\nUser-agent: MixrankBot\nUser-agent: MJ12bot\nUser-agent: moatbot\nUser-agent: MojeekBot\nUser-agent: Monsidobot\nUser-agent: Moreover\nUser-agent: Morningscore Bot\nUser-agent: MSNPTC\nUser-agent: msrbot\nUser-agent: MTRobot\nUser-agent: netarkivindsamling\nUser-agent: NetpeakCheckerBot\nUser-agent: NeumobBot\nUser-agent: Neustar\nUser-agent: Nielsen\nUser-agent: NostoCrawlerBot\nUser-agent: NumeratorBot\nUser-agent: OdklBot\nUser-agent: OKX-dolphin-crawler\nUser-agent: OnCrawl\nUser-agent: OneTrust\nUser-agent: OPBot\nUser-agent: OSZKbot\nUser-agent: outbrain\nUser-agent: Paddle\nUser-agent: PageFreezer\nUser-agent: Pandalytics\nUser-agent: parsely.com\nUser-agent: ParselySharesBot\nUser-agent: PayPal\nUser-agent: peer39_crawler\nUser-agent: Pinterest\nUser-agent: Pinterestbot\nUser-agent: Plukkie\nUser-agent: Prerender\nUser-agent: PriceSpider\nUser-agent: ProjectShield-UrlCheck\nUser-agent: proximic\nUser-agent: QualifiedBot\nUser-agent: Quantcastbot\nUser-agent: Qwantbot\nUser-agent: Qwantify\nUser-agent: Razorpay\nUser-agent: Rigor\nUser-agent: RiverglassScanner\nUser-agent: RKG Url Verifier\nUser-agent: rogerbot-crawler\nUser-agent: RuxitSynthetic\nUser-agent: scanalert\nUser-agent: search.marginalia.nu\nUser-agent: SearchAtlas Bot\nUser-agent: SeekportBot\nUser-agent: SemanticJuice\nUser-agent: SemrushBot\nUser-agent: SemrushBot-BA\nUser-agent: SemrushBot-OCOB\nUser-agent: SemrushBot-SA\nUser-agent: SemrushBot-SI\nUser-agent: SemrushBot-SWA\nUser-agent: sentry\nUser-agent: SentryUptimeBot\nUser-agent: seo-audit-check-bot\nUser-agent: SeobilityBot\nUser-agent: SERankingBacklinksBot\nUser-agent: SeznamBot\nUser-agent: SinceraSyntheticUser\nUser-agent: SISTRIX Optimizer\nUser-agent: SiteAuditBot\nUser-agent: sitebulb\nUser-agent: SiteCheckerBotCrawler\nUser-agent: Siteimprove-SiteCheck\nUser-agent: SiteSearch360\nUser-agent: SkypeUriPreview\nUser-agent: Slack-ImgProxy\nUser-agent: Slackbot\nUser-agent: Slurp\nUser-agent: SmarshBot\nUser-agent: SmartBriefBot\nUser-agent: Snap URL Preview Service\nUser-agent: SnapPreviewBot\nUser-agent: Sogou\nUser-agent: solarwinds\nUser-agent: spbot\nUser-agent: Splunk Synthetics\nUser-agent: SpringBot\nUser-agent: SSL Labs\nUser-agent: ssr.ias-lab.de\nUser-agent: Stape\nUser-agent: SteamChat\nUser-agent: Storebot-Google\nUser-agent: Stripe\nUser-agent: Sucuri\nUser-agent: TagInspector\nUser-agent: TelegramBot\nUser-agent: Teoma\nUser-agent: trendictionbot\nUser-agent: trovitBot\nUser-agent: Tumblr\nUser-agent: twibble.io\nUser-agent: TwinWaveScanner\nUser-agent: um-LN\nUser-agent: VaultPress\nUser-agent: vercel-screenshot\nUser-agent: Viber\nUser-agent: VigLink\nUser-agent: VocusBot\nUser-agent: W3C-checklink\nUser-agent: W3C_Validator\nUser-agent: Wappalyzer\nUser-agent: WhatsApp\nUser-agent: woorankreview\nUser-agent: Y!J-ASR\nUser-agent: Y!J-BRJ/YATS crawler\nUser-agent: Y!J-BRU/VSIDX\nUser-agent: Y!J-BRW\nUser-agent: Y!J-BRY/YATSH crawler\nUser-agent: Y!J-BRZ/YATSHA crawler\nUser-agent: Y!J-MMP/dscv\nUser-agent: Y!J-WSC\nUser-agent: YaDirectFetcher\nUser-agent: Yahoo Ad monitoring\nUser-agent: Yahoo Link Preview\nUser-agent: Yahoo! Slurp\nUser-agent: Yahoo-Japan-ASR\nUser-agent: YahooCacheSystem\nUser-agent: YahooExternalCache\nUser-agent: yahoofeedseeker\nUser-agent: YahooMailProxy\nUser-agent: YaK\nUser-agent: Yandex\nUser-agent: Yeti\nUser-agent: YextBot\nUser-agent: YioopBot\nUser-agent: YJ-SAD\nAllow: /_ngjs/resource/*/get/\nAllow: /business/create/\nAllow: /resource/*/get/\nDisallow: /*/*/*/_tools/*\nDisallow: /*/*/*/more_ideas/\nDisallow: /*/*/_tools/*\nDisallow: /*/*/activity/*\nDisallow: /*/*/group/\nDisallow: /*/*/invite/\nDisallow: /*/*/more_ideas/*\nDisallow: /*/?*amp_client_id*\nDisallow: /*/?z=1\nDisallow: /*/__wishlist__/*\nDisallow: /*/_activities/*\nDisallow: /*/_activity/*\nDisallow: /*/_community/*\nDisallow: /*/_created/*\nDisallow: /*/_followers/*\nDisallow: /*/_following/*\nDisallow: /*/_quick_saves/*\nDisallow: /*/_saved/*\nDisallow: /*/_shop/*\nDisallow: /*/_shopping_list/*\nDisallow: /*/communities/\nDisallow: /*/discovered/*\nDisallow: /*/followers/*\nDisallow: /*/following/*\nDisallow: /*/likes/*\nDisallow: /*/organize/*\nDisallow: /*/pins/\nDisallow: /*/profile-about/\nDisallow: /*/scheduled-pin/*\nDisallow: /*/scheduled-pins/*\nDisallow: /*/story_pins/*\nDisallow: /*/tagged/*\nDisallow: /*/tried/*\nDisallow: /*/video_pins/*\nDisallow: /?show_error=true\nDisallow: /_/_/*\nDisallow: /_/_/pixel/*\nDisallow: /_ngjs/resource/\nDisallow: /about/*\nDisallow: /addboard/\nDisallow: /admin/\nDisallow: /age-appeal/*\nDisallow: /amp/*\nDisallow: /appealed-pin/\nDisallow: /autocomplete/\nDisallow: /board/*\nDisallow: /board_activity/*\nDisallow: /board_activity_comment/*\nDisallow: /bookmarklet/\nDisallow: /branded-pins/\nDisallow: /business/ad_create/\nDisallow: /business/business-access/*\nDisallow: /business/business-hierarchy/*\nDisallow: /business/business-manager/*\nDisallow: /business/catalogs/*\nDisallow: /business/commerce-integrations/*\nDisallow: /business/community-information/\nDisallow: /business/convert/\nDisallow: /business/create-linked-account/\nDisallow: /business/hub/\nDisallow: /business/invite/*\nDisallow: /business/onboarding/\nDisallow: /business/pin_create/\nDisallow: /business/referral/*\nDisallow: /business/site/\nDisallow: /business/switch-account/\nDisallow: /business/top-ads/\nDisallow: /business/update/\nDisallow: /business/verified-merchant/status/\nDisallow: /christmas-spotlight/\nDisallow: /close-account/\nDisallow: /communities/*\nDisallow: /community/*\nDisallow: /community_comment/*\nDisallow: /community_post/*\nDisallow: /connect/*/\nDisallow: /content-claiming/*\nDisallow: /conversation/*\nDisallow: /conversations/*\nDisallow: /convert-business/\nDisallow: /convert-personal/\nDisallow: /copyright-reporting/*\nDisallow: /create_pin/\nDisallow: /creative-studio/\nDisallow: /creative_editor/\nDisallow: /creator-spotlight/*\nDisallow: /creator_hub/_earn\nDisallow: /creator_hub/_partnerships\nDisallow: /csrf_error/\nDisallow: /ct.html\nDisallow: /deactivate-account/\nDisallow: /delete_post/\nDisallow: /discover/*\nDisallow: /edit/*\nDisallow: /editorial/*/*/\nDisallow: /email/*\nDisallow: /email_verification_error/\nDisallow: /engagement/*\nDisallow: /explore/*/?p=*\nDisallow: /facebook/\nDisallow: /fb.html\nDisallow: /follow_board/\nDisallow: /follow_user/\nDisallow: /google_gdn.html\nDisallow: /google_search.html\nDisallow: /hanukkah-spotlight/\nDisallow: /holiday-spotlight/\nDisallow: /holidays-celebrations/\nDisallow: /homefeed/\nDisallow: /homefeed/*\nDisallow: /idea-ads-tool/*\nDisallow: /idea-pin-builder/*\nDisallow: /idea-pin-invite/\nDisallow: /install-shuffles/\nDisallow: /invite/\nDisallow: /invited/\nDisallow: /join/?next=*\nDisallow: /join/signup/?next=*\nDisallow: /kwanzaa-spotlight/\nDisallow: /lens-search/*\nDisallow: /linked/*\nDisallow: /live-session/*\nDisallow: /login/?next=*\nDisallow: /logout/\nDisallow: /m/\nDisallow: /new_invite/\nDisallow: /news_hub/*\nDisallow: /notifications/\nDisallow: /oauth/line/*\nDisallow: /oauth/login/*\nDisallow: /oauth_error/\nDisallow: /offsite/\nDisallow: /parental-consent/confirmation/\nDisallow: /parental-passcode/\nDisallow: /password/reset/\nDisallow: /password/reset/*\nDisallow: /pin-builder/*\nDisallow: /pin-communities/*\nDisallow: /pin-creation-tool/*\nDisallow: /pin-editor/\nDisallow: /pin/*/activities\nDisallow: /pin/*/activity/\nDisallow: /pin/*/analytics\nDisallow: /pin/*/brand-catalog/\nDisallow: /pin/*/brand-catalog/*\nDisallow: /pin/*/comments\nDisallow: /pin/*/email\nDisallow: /pin/*/feedback\nDisallow: /pin/*/idea-pin/related-products/*\nDisallow: /pin/*/likes\nDisallow: /pin/*/promote\nDisallow: /pin/*/reaction\nDisallow: /pin/*/related-products\nDisallow: /pin/*/related-products/*\nDisallow: /pin/*/repin\nDisallow: /pin/*/repins\nDisallow: /pin/*/share\nDisallow: /pin/*/shop-the-look/*\nDisallow: /pin/*/tag\nDisallow: /pin/*/takes\nDisallow: /pin/*/visual-search\nDisallow: /pin/*/visual-shop\nDisallow: /pin/create/*\nDisallow: /pin/find/\nDisallow: /pin/find?url=*\nDisallow: /pin_shop/*\nDisallow: /pinterest-predicts/\nDisallow: /pinterest-wellbeing/*\nDisallow: /pinterestwellbeing/*\nDisallow: /place/*\nDisallow: /prefs/\nDisallow: /product-catalogs/\nDisallow: /product-catalogs/*\nDisallow: /product-feed/\nDisallow: /pw/*\nDisallow: /quiz/*\nDisallow: /recently-viewed/\nDisallow: /refresh_stored_accounts\nDisallow: /refresh_token\nDisallow: /register/\nDisallow: /reports-and-violations/\nDisallow: /request-data/\nDisallow: /resource/\nDisallow: /safe-redirect/\nDisallow: /search/\nDisallow: /search?\nDisallow: /secure/*\nDisallow: /sendHeaderEarly/\nDisallow: /settings/*\nDisallow: /shop_pins/*\nDisallow: /signup/*\nDisallow: /socialmanager/*\nDisallow: /source/*\nDisallow: /sso/*\nDisallow: /story-pin-builder/*\nDisallow: /story-pin-invite/\nDisallow: /story_feed/\nDisallow: /story_pins/*\nDisallow: /suspension-appeal/*\nDisallow: /sw-shell.html\nDisallow: /taggingtool/*\nDisallow: /thanksgiving-spotlight/\nDisallow: /theme/*\nDisallow: /tv/*\nDisallow: /unauth-profile/\nDisallow: /unlink/\nDisallow: /upload-lens-image\nDisallow: /upload-lens-image-history\nDisallow: /upload-shuffle-draft\nDisallow: /v1/urls/count.json\nDisallow: /v3/pidgets/log/\nDisallow: /website/confirm/\nDisallow: /website/confirm/download*\nDisallow: /welcome/\nDisallow: /your-shop/\nDisallow: /your-shop/*\n\nUser-agent: Googlebot\nAllow: /_ngjs/resource/*/get/\nAllow: /business/create/\nAllow: /resource/*/get/\nDisallow: /*/*/*/_tools/*\nDisallow: /*/*/*/more_ideas/\nDisallow: /*/*/_tools/*\nDisallow: /*/*/activity/*\nDisallow: /*/*/group/\nDisallow: /*/*/invite/\nDisallow: /*/*/more_ideas/*\nDisallow: /*/?*amp_client_id*\nDisallow: /*/?z=1\nDisallow: /*/__wishlist__/*\nDisallow: /*/_activities/*\nDisallow: /*/_activity/*\nDisallow: /*/_community/*\nDisallow: /*/_created/*\nDisallow: /*/_followers/*\nDisallow: /*/_following/*\nDisallow: /*/_quick_saves/*\nDisallow: /*/_saved/*\nDisallow: /*/_shop/*\nDisallow: /*/_shopping_list/*\nDisallow: /*/communities/\nDisallow: /*/discovered/*\nDisallow: /*/followers/*\nDisallow: /*/following/*\nDisallow: /*/likes/*\nDisallow: /*/organize/*\nDisallow: /*/pins/\nDisallow: /*/profile-about/\nDisallow: /*/scheduled-pin/*\nDisallow: /*/scheduled-pins/*\nDisallow: /*/story_pins/*\nDisallow: /*/tagged/*\nDisallow: /*/tried/*\nDisallow: /*/video_pins/*\nDisallow: /?show_error=true\nDisallow: /_/_/*\nDisallow: /_/_/pixel/*\nDisallow: /_ngjs/resource/\nDisallow: /about/*\nDisallow: /addboard/\nDisallow: /admin/\nDisallow: /age-appeal/*\nDisallow: /amp/*\nDisallow: /appealed-pin/\nDisallow: /autocomplete/\nDisallow: /board/*\nDisallow: /board_activity/*\nDisallow: /board_activity_comment/*\nDisallow: /bookmarklet/\nDisallow: /branded-pins/\nDisallow: /business/ad_create/\nDisallow: /business/business-access/*\nDisallow: /business/business-hierarchy/*\nDisallow: /business/business-manager/*\nDisallow: /business/catalogs/*\nDisallow: /business/commerce-integrations/*\nDisallow: /business/community-information/\nDisallow: /business/convert/\nDisallow: /business/create-linked-account/\nDisallow: /business/hub/\nDisallow: /business/invite/*\nDisallow: /business/onboarding/\nDisallow: /business/pin_create/\nDisallow: /business/referral/*\nDisallow: /business/site/\nDisallow: /business/switch-account/\nDisallow: /business/top-ads/\nDisallow: /business/update/\nDisallow: /business/verified-merchant/status/\nDisallow: /christmas-spotlight/\nDisallow: /close-account/\nDisallow: /communities/*\nDisallow: /community/*\nDisallow: /community_comment/*\nDisallow: /community_post/*\nDisallow: /connect/*/\nDisallow: /content-claiming/*\nDisallow: /conversation/*\nDisallow: /conversations/*\nDisallow: /convert-business/\nDisallow: /convert-personal/\nDisallow: /copyright-reporting/*\nDisallow: /create_pin/\nDisallow: /creative-studio/\nDisallow: /creative_editor/\nDisallow: /creator-spotlight/*\nDisallow: /creator_hub/_earn\nDisallow: /creator_hub/_partnerships\nDisallow: /csrf_error/\nDisallow: /ct.html\nDisallow: /deactivate-account/\nDisallow: /delete_post/\nDisallow: /discover/*\nDisallow: /edit/*\nDisallow: /editorial/*/*/\nDisallow: /email/*\nDisallow: /email_verification_error/\nDisallow: /engagement/*\nDisallow: /explore/*\nDisallow: /explore/*/?p=*\nDisallow: /facebook/\nDisallow: /fb.html\nDisallow: /follow_board/\nDisallow: /follow_user/\nDisallow: /google_gdn.html\nDisallow: /google_search.html\nDisallow: /hanukkah-spotlight/\nDisallow: /holiday-spotlight/\nDisallow: /holidays-celebrations/\nDisallow: /homefeed/\nDisallow: /homefeed/*\nDisallow: /idea-ads-tool/*\nDisallow: /idea-pin-builder/*\nDisallow: /idea-pin-invite/\nDisallow: /install-shuffles/\nDisallow: /invite/\nDisallow: /invited/\nDisallow: /join/?next=*\nDisallow: /join/signup/?next=*\nDisallow: /kwanzaa-spotlight/\nDisallow: /lens-search/*\nDisallow: /linked/*\nDisallow: /live-session/*\nDisallow: /login/?next=*\nDisallow: /logout/\nDisallow: /m/\nDisallow: /new_invite/\nDisallow: /news_hub/*\nDisallow: /notifications/\nDisallow: /oauth/line/*\nDisallow: /oauth/login/*\nDisallow: /oauth_error/\nDisallow: /offsite/\nDisallow: /parental-consent/confirmation/\nDisallow: /parental-passcode/\nDisallow: /password/reset/\nDisallow: /password/reset/*\nDisallow: /pin-builder/*\nDisallow: /pin-communities/*\nDisallow: /pin-creation-tool/*\nDisallow: /pin-editor/\nDisallow: /pin/*/activities\nDisallow: /pin/*/activity/\nDisallow: /pin/*/analytics\nDisallow: /pin/*/brand-catalog/\nDisallow: /pin/*/brand-catalog/*\nDisallow: /pin/*/comments\nDisallow: /pin/*/email\nDisallow: /pin/*/feedback\nDisallow: /pin/*/idea-pin/related-products/*\nDisallow: /pin/*/likes\nDisallow: /pin/*/promote\nDisallow: /pin/*/reaction\nDisallow: /pin/*/related-products\nDisallow: /pin/*/related-products/*\nDisallow: /pin/*/repin\nDisallow: /pin/*/repins\nDisallow: /pin/*/share\nDisallow: /pin/*/shop-the-look/*\nDisallow: /pin/*/tag\nDisallow: /pin/*/takes\nDisallow: /pin/*/visual-search\nDisallow: /pin/*/visual-shop\nDisallow: /pin/create/*\nDisallow: /pin/find/\nDisallow: /pin/find?url=*\nDisallow: /pin_shop/*\nDisallow: /pinterest-predicts/\nDisallow: /pinterest-wellbeing/*\nDisallow: /pinterestwellbeing/*\nDisallow: /place/*\nDisallow: /prefs/\nDisallow: /product-catalogs/\nDisallow: /product-catalogs/*\nDisallow: /product-feed/\nDisallow: /pw/*\nDisallow: /quiz/*\nDisallow: /recently-viewed/\nDisallow: /refresh_stored_accounts\nDisallow: /refresh_token\nDisallow: /register/\nDisallow: /reports-and-violations/\nDisallow: /request-data/\nDisallow: /resource/\nDisallow: /safe-redirect/\nDisallow: /search/\nDisallow: /search?\nDisallow: /secure/*\nDisallow: /sendHeaderEarly/\nDisallow: /settings/*\nDisallow: /shop_pins/*\nDisallow: /signup/*\nDisallow: /socialmanager/*\nDisallow: /source/*\nDisallow: /sso/*\nDisallow: /story-pin-builder/*\nDisallow: /story-pin-invite/\nDisallow: /story_feed/\nDisallow: /story_pins/*\nDisallow: /suspension-appeal/*\nDisallow: /sw-shell.html\nDisallow: /taggingtool/*\nDisallow: /thanksgiving-spotlight/\nDisallow: /theme/*\nDisallow: /tv/*\nDisallow: /unauth-profile/\nDisallow: /unlink/\nDisallow: /upload-lens-image\nDisallow: /upload-lens-image-history\nDisallow: /upload-shuffle-draft\nDisallow: /v1/urls/count.json\nDisallow: /v3/pidgets/log/\nDisallow: /website/confirm/\nDisallow: /website/confirm/download*\nDisallow: /welcome/\nDisallow: /your-shop/\nDisallow: /your-shop/*\n\nUser-agent: Twitterbot\nAllow: /_ngjs/resource/*/get/\nAllow: /business/create/\nAllow: /pin/*/feedback\nAllow: /pin/*/sent\nAllow: /resource/*/get/\nDisallow: /*/*/*/_tools/*\nDisallow: /*/*/*/more_ideas/\nDisallow: /*/*/_tools/*\nDisallow: /*/*/activity/*\nDisallow: /*/*/group/\nDisallow: /*/*/invite/\nDisallow: /*/*/more_ideas/*\nDisallow: /*/?*amp_client_id*\nDisallow: /*/?z=1\nDisallow: /*/__wishlist__/*\nDisallow: /*/_activities/*\nDisallow: /*/_activity/*\nDisallow: /*/_community/*\nDisallow: /*/_created/*\nDisallow: /*/_followers/*\nDisallow: /*/_following/*\nDisallow: /*/_quick_saves/*\nDisallow: /*/_saved/*\nDisallow: /*/_shop/*\nDisallow: /*/_shopping_list/*\nDisallow: /*/communities/\nDisallow: /*/discovered/*\nDisallow: /*/followers/*\nDisallow: /*/following/*\nDisallow: /*/likes/*\nDisallow: /*/organize/*\nDisallow: /*/pins/\nDisallow: /*/profile-about/\nDisallow: /*/scheduled-pin/*\nDisallow: /*/scheduled-pins/*\nDisallow: /*/story_pins/*\nDisallow: /*/tagged/*\nDisallow: /*/tried/*\nDisallow: /*/video_pins/*\nDisallow: /?show_error=true\nDisallow: /_/_/*\nDisallow: /_/_/pixel/*\nDisallow: /_ngjs/resource/\nDisallow: /about/*\nDisallow: /addboard/\nDisallow: /admin/\nDisallow: /age-appeal/*\nDisallow: /amp/*\nDisallow: /appealed-pin/\nDisallow: /autocomplete/\nDisallow: /board/*\nDisallow: /board_activity/*\nDisallow: /board_activity_comment/*\nDisallow: /bookmarklet/\nDisallow: /branded-pins/\nDisallow: /business/ad_create/\nDisallow: /business/business-access/*\nDisallow: /business/business-hierarchy/*\nDisallow: /business/business-manager/*\nDisallow: /business/catalogs/*\nDisallow: /business/commerce-integrations/*\nDisallow: /business/community-information/\nDisallow: /business/convert/\nDisallow: /business/create-linked-account/\nDisallow: /business/hub/\nDisallow: /business/invite/*\nDisallow: /business/onboarding/\nDisallow: /business/pin_create/\nDisallow: /business/referral/*\nDisallow: /business/site/\nDisallow: /business/switch-account/\nDisallow: /business/top-ads/\nDisallow: /business/update/\nDisallow: /business/verified-merchant/status/\nDisallow: /christmas-spotlight/\nDisallow: /close-account/\nDisallow: /communities/*\nDisallow: /community/*\nDisallow: /community_comment/*\nDisallow: /community_post/*\nDisallow: /connect/*/\nDisallow: /content-claiming/*\nDisallow: /conversation/*\nDisallow: /conversations/*\nDisallow: /convert-business/\nDisallow: /convert-personal/\nDisallow: /copyright-reporting/*\nDisallow: /create_pin/\nDisallow: /creative-studio/\nDisallow: /creative_editor/\nDisallow: /creator-spotlight/*\nDisallow: /creator_hub/_earn\nDisallow: /creator_hub/_partnerships\nDisallow: /csrf_error/\nDisallow: /ct.html\nDisallow: /deactivate-account/\nDisallow: /delete_post/\nDisallow: /discover/*\nDisallow: /edit/*\nDisallow: /editorial/*/*/\nDisallow: /email/*\nDisallow: /email_verification_error/\nDisallow: /engagement/*\nDisallow: /explore/*\nDisallow: /explore/*/?p=*\nDisallow: /facebook/\nDisallow: /fb.html\nDisallow: /follow_board/\nDisallow: /follow_user/\nDisallow: /google_gdn.html\nDisallow: /google_search.html\nDisallow: /hanukkah-spotlight/\nDisallow: /holiday-spotlight/\nDisallow: /holidays-celebrations/\nDisallow: /homefeed/\nDisallow: /homefeed/*\nDisallow: /idea-ads-tool/*\nDisallow: /idea-pin-builder/*\nDisallow: /idea-pin-invite/\nDisallow: /install-shuffles/\nDisallow: /invite/\nDisallow: /invited/\nDisallow: /join/?next=*\nDisallow: /join/signup/?next=*\nDisallow: /kwanzaa-spotlight/\nDisallow: /lens-search/*\nDisallow: /linked/*\nDisallow: /live-session/*\nDisallow: /login/?next=*\nDisallow: /logout/\nDisallow: /m/\nDisallow: /new_invite/\nDisallow: /news_hub/*\nDisallow: /notifications/\nDisallow: /oauth/line/*\nDisallow: /oauth/login/*\nDisallow: /oauth_error/\nDisallow: /offsite/\nDisallow: /parental-consent/confirmation/\nDisallow: /parental-passcode/\nDisallow: /password/reset/\nDisallow: /password/reset/*\nDisallow: /pin-builder/*\nDisallow: /pin-communities/*\nDisallow: /pin-creation-tool/*\nDisallow: /pin-editor/\nDisallow: /pin/*/activities\nDisallow: /pin/*/activity/\nDisallow: /pin/*/analytics\nDisallow: /pin/*/brand-catalog/\nDisallow: /pin/*/brand-catalog/*\nDisallow: /pin/*/comments\nDisallow: /pin/*/email\nDisallow: /pin/*/idea-pin/related-products/*\nDisallow: /pin/*/likes\nDisallow: /pin/*/promote\nDisallow: /pin/*/reaction\nDisallow: /pin/*/related-products\nDisallow: /pin/*/related-products/*\nDisallow: /pin/*/repin\nDisallow: /pin/*/repins\nDisallow: /pin/*/share\nDisallow: /pin/*/shop-the-look/*\nDisallow: /pin/*/tag\nDisallow: /pin/*/takes\nDisallow: /pin/*/visual-search\nDisallow: /pin/*/visual-shop\nDisallow: /pin/create/*\nDisallow: /pin/find/\nDisallow: /pin/find?url=*\nDisallow: /pin_shop/*\nDisallow: /pinterest-predicts/\nDisallow: /pinterest-wellbeing/*\nDisallow: /pinterestwellbeing/*\nDisallow: /place/*\nDisallow: /prefs/\nDisallow: /product-catalogs/\nDisallow: /product-catalogs/*\nDisallow: /product-feed/\nDisallow: /pw/*\nDisallow: /quiz/*\nDisallow: /recently-viewed/\nDisallow: /refresh_stored_accounts\nDisallow: /refresh_token\nDisallow: /register/\nDisallow: /reports-and-violations/\nDisallow: /request-data/\nDisallow: /resource/\nDisallow: /safe-redirect/\nDisallow: /search/\nDisallow: /search?\nDisallow: /secure/*\nDisallow: /sendHeaderEarly/\nDisallow: /settings/*\nDisallow: /shop_pins/*\nDisallow: /signup/*\nDisallow: /socialmanager/*\nDisallow: /source/*\nDisallow: /sso/*\nDisallow: /story-pin-builder/*\nDisallow: /story-pin-invite/\nDisallow: /story_feed/\nDisallow: /story_pins/*\nDisallow: /suspension-appeal/*\nDisallow: /sw-shell.html\nDisallow: /taggingtool/*\nDisallow: /thanksgiving-spotlight/\nDisallow: /theme/*\nDisallow: /tv/*\nDisallow: /unauth-profile/\nDisallow: /unlink/\nDisallow: /upload-lens-image\nDisallow: /upload-lens-image-history\nDisallow: /upload-shuffle-draft\nDisallow: /v1/urls/count.json\nDisallow: /v3/pidgets/log/\nDisallow: /website/confirm/\nDisallow: /website/confirm/download*\nDisallow: /welcome/\nDisallow: /your-shop/\nDisallow: /your-shop/*\n\nUser-agent: Mediapartners-Google\nUser-agent: AmazonAdBot\nAllow: /homefeed/\nAllow: /homefeed/*\nAllow: /pin/\nAllow: /search/*\nDisallow: /\n\nUser-agent: Google-adstxt\nUser-agent: IAB-Tech-Lab\nAllow: /ads.txt\nAllow: /app-ads.txt\nDisallow: /\n\nUser-agent: adidxbot\nUser-agent: AdsBot-Google\nUser-agent: AdsBot-Google-Mobile\nUser-agent: AxonBot\nUser-agent: Google Web Preview\nUser-agent: Google-Ads-Overview\nUser-agent: Google-AdWords-Express\nUser-agent: Google-Adwords-Instant\nUser-agent: LinkedInBot\nUser-agent: meta-externalads\nUser-agent: SnapchatAds\nUser-agent: Taboolabot\nAllow: /\n\nUser-agent: YandexAdditional\nUser-agent: YandexAdditionalBot\nDisallow: /\n\nUser-agent: *\nDisallow: /\n\nSitemap: https://www.pinterest.com/v3_sitemaps/seasonal_events_sitemap_20261125_international_day_for_the_elimination_of_violence_www.pinterest.com.xml\nSitemap: https://www.pinterest.com/v3_sitemaps/seasonal_events_sitemap_20261225_catholic_christmas_day_www.pinterest.com.xml\nSitemap: https://www.pinterest.com/v3_sitemaps/seasonal_events_sitemap_20261226_day_after_christmas_day_www.pinterest.com.xml\nSitemap: https://www.pinterest.com/v3_sitemaps/seasonal_events_sitemap_20261224_catholic_protestant_christmas_eve_www.pinterest.com.xml\nSitemap: https://www.pinterest.com/v3_sitemaps/seasonal_events_sitemap_20261224_christmas_eve_from_2pm_www.pinterest.com.xml\nSitemap: https://www.pinterest.com/v3_sitemaps/seasonal_events_sitemap_20261210_human_rights_day_www.pinterest.com.xml\nSitemap: https://www.pinterest.com/v3_sitemaps/seasonal_events_sitemap_20261226_st_stephen_s_day_www.pinterest.com.xml\nSitemap: https://www.pinterest.com/v3_sitemaps/seasonal_events_sitemap_20261206_hanukkah_day_2_www.pinterest.com.xml\nSitemap: https://www.pinterest.com/v3_sitemaps/seasonal_events_sitemap_20261213_third_advent_sunday_www.pinterest.com.xml\nSitemap: https://www.pinterest.com/v3_sitemaps/seasonal_events_sitemap_20261129_international_day_of_solidarity_with_the_palestini_www.pinterest.com.xml\nSitemap: https://www.pinterest.com/v3_sitemaps/seasonal_events_sitemap_20261228_day_off_for_boxing_day_www.pinterest.com.xml\nSitemap: https://www.pinterest.com/v3_sitemaps/seasonal_events_sitemap_20261225_first_day_of_western_christmas_www.pinterest.com.xml\nSitemap: https://www.pinterest.com/v3_sitemaps/seasonal_events_sitemap_20261122_sunday_of_the_dead_www.pinterest.com.xml\nSitemap: https://www.pinterest.com/v3_sitemaps/ideas_hub_world_cup_terms_sitemap_www.pinterest.com.xml\nSitemap: https://www.pinterest.com/v3_sitemaps/seasonal_events_sitemap_20261201_givingtuesday_www.pinterest.com.xml\nSitemap: https://www.pinterest.com/v3_sitemaps/seasonal_events_sitemap_20261225_family_day_www.pinterest.com.xml\nSitemap: https://www.pinterest.com/v3_sitemaps/seasonal_events_sitemap_20261205_st_nicholas_eve_sinterklaas_www.pinterest.com.xml\nSitemap: https://www.pinterest.com/v3_sitemaps/seasonal_events_sitemap_20261224_yap_constitution_day_www.pinterest.com.xml\nSitemap: https://www.pinterest.com/v3_sitemaps/seasonal_events_sitemap_20261127_day_after_thanksgiving_www.pinterest.com.xml\nSitemap: https://www.pinterest.com/v3_sitemaps/seasonal_events_sitemap_20261207_international_civil_aviation_day_www.pinterest.com.xml\nSitemap: https://www.pinterest.com/v3_sitemaps/qem_klp_vlm_vase_medium_www.pinterest.com.xml\nSitemap: https://www.pinterest.com/v3_sitemaps/seasonal_events_sitemap_20261205_hanukkah_day_1_www.pinterest.com.xml\nSitemap: https://www.pinterest.com/v3_sitemaps/seasonal_events_sitemap_20261123_labor_thanksgiving_day_www.pinterest.com.xml\nSitemap: https://www.pinterest.com/v3_sitemaps/seasonal_events_sitemap_20261226_day_after_christmas_www.pinterest.com.xml\nSitemap: https://www.pinterest.com/v3_sitemaps/seasonal_events_sitemap_20261228_christmas_holiday_www.pinterest.com.xml\nSitemap: https://www.pinterest.com/v3_sitemaps/seasonal_events_sitemap_20261226_day_of_goodwill_www.pinterest.com.xml\nSitemap: https://www.pinterest.com/v3_sitemaps/seasonal_events_sitemap_20261212_last_day_of_hanukkah_www.pinterest.com.xml\nSitemap: https://www.pinterest.com/v3_sitemaps/seasonal_events_sitemap_20261225_catholic_protestant_christmas_day_www.pinterest.com.xml\nSitemap: https://www.pinterest.com/v3_sitemaps/seasonal_events_sitemap_20261205_chanukah_hanukkah_first_day_www.pinterest.com.xml\nSitemap: https://www.pinterest.com/v3_sitemaps/seasonal_events_sitemap_20261130_day_of_remembrance_for_all_victims_of_chemical_war_www.pinterest.com.xml\nSitemap: https://www.pinterest.com/v3_sitemaps/seasonal_events_sitemap_20261225_quaid_e_azam_day_www.pinterest.com.xml\nSitemap: https://www.pinterest.com/v3_sitemaps/seasonal_events_sitemap_20261231_new_year_holiday_www.pinterest.com.xml\nSitemap: https://www.pinterest.com/v3_sitemaps/qem_klp_vlm_navboost_sample_high_www.pinterest.com.xml\nSitemap: https://www.pinterest.com/v3_sitemaps/seasonal_events_sitemap_20261203_international_day_of_persons_with_disabilities_www.pinterest.com.xml\nSitemap: https://www.pinterest.com/v3_sitemaps/seasonal_events_sitemap_20261201_world_aids_day_www.pinterest.com.xml\nSitemap: https://www.pinterest.com/v3_sitemaps/seasonal_events_sitemap_20261206_christmas_season_begins_www.pinterest.com.xml\nSitemap: https://www.pinterest.com/v3_sitemaps/seasonal_events_sitemap_20261226_second_day_of_christmas_www.pinterest.com.xml\nSitemap: https://www.pinterest.com/v3_sitemaps/seasonal_events_sitemap_20261124_guru_nanak_s_birthday_www.pinterest.com.xml\nSitemap: https://www.pinterest.com/v3_sitemaps/seasonal_events_sitemap_20261231_substitute_holiday_for_new_year_holiday_www.pinterest.com.xml\nSitemap: https://www.pinterest.com/v3_sitemaps/seasonal_events_sitemap_20261231_old_year_s_night_www.pinterest.com.xml\nSitemap: https://www.pinterest.com/v3_sitemaps/seasonal_events_sitemap_20261226_kwanzaa_first_day_www.pinterest.com.xml\nSitemap: https://www.pinterest.com/v3_sitemaps/seasonal_events_sitemap_20261230_tamu_lhosar_www.pinterest.com.xml\nSitemap: https://www.pinterest.com/v3_sitemaps/seasonal_events_sitemap_20261122_albanian_alphabet_day_albanian_community_www.pinterest.com.xml\nSitemap: https://www.pinterest.com/v3_sitemaps/seasonal_events_sitemap_20261128_flag_and_independence_day_www.pinterest.com.xml\nSitemap: https://www.pinterest.com/v3_sitemaps/seasonal_events_sitemap_20261206_second_advent_sunday_www.pinterest.com.xml\nSitemap: https://www.pinterest.com/v3_sitemaps/seasonal_events_sitemap_20261226_independence_and_unity_day_www.pinterest.com.xml\nSitemap: https://www.pinterest.com/v3_sitemaps/seasonal_events_sitemap_20261207_hanukkah_day_3_www.pinterest.com.xml\nSitemap: https://www.pinterest.com/v3_sitemaps/seasonal_events_sitemap_20261228_substitute_bank_holiday_for_boxing_day_www.pinterest.com.xml\nSitemap: https://www.pinterest.com/v3_sitemaps/seasonal_events_sitemap_20261124_vietnam_culture_day_www.pinterest.com.xml\nSitemap: https://www.pinterest.com/v3_sitemaps/seasonal_events_sitemap_20261225_christmas_www.pinterest.com.xml\nSitemap: https://www.pinterest.com/v3_sitemaps/seasonal_events_sitemap_20261228_day_of_the_holy_innocents_www.pinterest.com.xml\nSitemap: https://www.pinterest.com/v3_sitemaps/seasonal_events_sitemap_20261127_native_american_heritage_day_www.pinterest.com.xml\nSitemap: https://www.pinterest.com/v3_sitemaps/ideas_hub_expansion_v2_sitemap_www.pinterest.com.xml\nSitemap: https://www.pinterest.com/v3_sitemaps/ideas_hub_gsc_terms_sitemap_www.pinterest.com.xml\nSitemap: https://www.pinterest.com/v3_sitemaps/seasonal_events_sitemap_20261226_national_day_of_thanksgiving_www.pinterest.com.xml\nSitemap: https://www.pinterest.com/v3_sitemaps/seasonal_events_sitemap_20261121_world_television_day_www.pinterest.com.xml\nSitemap: https://www.pinterest.com/v3_sitemaps/seasonal_events_sitemap_20261226_catholic_protestant_boxing_day_www.pinterest.com.xml\nSitemap: https://www.pinterest.com/v3_sitemaps/qem_klp_vlm_navboost_sample_medium_www.pinterest.com.xml\nSitemap: https://www.pinterest.com/v3_sitemaps/seasonal_events_sitemap_20261229_peace_day_www.pinterest.com.xml\nSitemap: https://www.pinterest.com/v3_sitemaps/seasonal_events_sitemap_20261204_international_day_of_banks_www.pinterest.com.xml\nSitemap: https://www.pinterest.com/v3_sitemaps/seasonal_events_sitemap_20261123_day_off_for_alphabet_day_www.pinterest.com.xml\nSitemap: https://www.pinterest.com/v3_sitemaps/qem_klp_vlm_vase_low_www.pinterest.com.xml\nSitemap: https://www.pinterest.com/v3_sitemaps/seasonal_events_sitemap_20261209_hanukkah_day_5_www.pinterest.com.xml\nSitemap: https://www.pinterest.com/v3_sitemaps/seasonal_events_sitemap_20261207_pearl_harbor_remembrance_day_www.pinterest.com.xml\nSitemap: https://www.pinterest.com/v3_sitemaps/seasonal_events_sitemap_20261210_hanukkah_day_6_rosh_chodesh_tevet_www.pinterest.com.xml\nSitemap: https://www.pinterest.com/v3_sitemaps/seasonal_events_sitemap_20261124_guru_nanak_jayanti_www.pinterest.com.xml\nSitemap: https://www.pinterest.com/v3_sitemaps/seasonal_events_sitemap_20261206_st_nicholas_day_www.pinterest.com.xml\nSitemap: https://www.pinterest.com/v3_sitemaps/seasonal_events_sitemap_20261205_first_day_of_hanukkah_www.pinterest.com.xml\nSitemap: https://www.pinterest.com/v3_sitemaps/seasonal_events_sitemap_20261224_christmas_eve_joint_holiday_www.pinterest.com.xml\nSitemap: https://www.pinterest.com/v3_sitemaps/seasonal_events_sitemap_20261201_freedom_and_democracy_day_www.pinterest.com.xml\nSitemap: https://www.pinterest.com/v3_sitemaps/seasonal_events_sitemap_20261231_december_31_bank_holiday_www.pinterest.com.xml\nSitemap: https://www.pinterest.com/v3_sitemaps/qem_klp_vlm_vase_high_www.pinterest.com.xml\nSitemap: https://www.pinterest.com/v3_sitemaps/seasonal_events_sitemap_20261231_new_year_s_eve_afternoon_www.pinterest.com.xml\nSitemap: https://www.pinterest.com/v3_sitemaps/seasonal_events_sitemap_20261122_alphabet_day_www.pinterest.com.xml\nSitemap: https://www.pinterest.com/v3_sitemaps/seasonal_events_sitemap_20261205_world_soil_day_www.pinterest.com.xml\nSitemap: https://www.pinterest.com/v3_sitemaps/seasonal_events_sitemap_20261228_boxing_day_observed_www.pinterest.com.xml\nSitemap: https://www.pinterest.com/v3_sitemaps/seasonal_events_sitemap_20261231_new_year_s_eve_from_2pm_www.pinterest.com.xml\nSitemap: https://www.pinterest.com/v3_sitemaps/seasonal_events_sitemap_20261226_day_after_christmas_christians_only_www.pinterest.com.xml\nSitemap: https://www.pinterest.com/v3_sitemaps/ideas_hub_vlm_terms_sitemap_auto_publish_www.pinterest.com.xml\nSitemap: https://www.pinterest.com/v3_sitemaps/seasonal_events_sitemap_20261122_christ_the_king_day_www.pinterest.com.xml\nSitemap: https://www.pinterest.com/v3_sitemaps/seasonal_events_sitemap_20261231_western_new_year_s_eve_www.pinterest.com.xml\nSitemap: https://www.pinterest.com/v3_sitemaps/seasonal_events_sitemap_20261231_international_new_year_s_eve_www.pinterest.com.xml\nSitemap: https://www.pinterest.com/v3_sitemaps/seasonal_events_sitemap_20261227_feast_of_the_holy_family_www.pinterest.com.xml\nSitemap: https://www.pinterest.com/v3_sitemaps/seasonal_events_sitemap_20261212_hanukkah_day_8_www.pinterest.com.xml\nSitemap: https://www.pinterest.com/v3_sitemaps/seasonal_events_sitemap_20261214_green_monday_www.pinterest.com.xml\nSitemap: https://www.pinterest.com/v3_sitemaps/seasonal_events_sitemap_20261208_feast_of_the_immaculate_conception_www.pinterest.com.xml\nSitemap: https://www.pinterest.com/v3_sitemaps/seasonal_events_sitemap_20261031_halloween_www.pinterest.com.xml\nSitemap: https://www.pinterest.com/v3_sitemaps/seasonal_events_sitemap_20261230_rizal_day_www.pinterest.com.xml\nSitemap: https://www.pinterest.com/v3_sitemaps/ideas_hub_v_expansion_sitemap_www.pinterest.com.xml\nSitemap: https://www.pinterest.com/v3_sitemaps/seasonal_events_sitemap_20261206_saint_nicholas_day_www.pinterest.com.xml\nSitemap: https://www.pinterest.com/v3_sitemaps/qem_klp_vlm_navboost_sample_low_www.pinterest.com.xml\nSitemap: https://www.pinterest.com/v3_sitemaps/seasonal_events_sitemap_20261127_presidents_day_observed_www.pinterest.com.xml\nSitemap: https://www.pinterest.com/v3_sitemaps/seasonal_events_sitemap_20261226_christmas_second_day_www.pinterest.com.xml\nSitemap: https://www.pinterest.com/v3_sitemaps/seasonal_events_sitemap_20261226_christmas_day_observed_www.pinterest.com.xml\nSitemap: https://www.pinterest.com/v3_sitemaps/seasonal_events_sitemap_20261226_synaxis_of_the_mother_of_god_www.pinterest.com.xml\nSitemap: https://www.pinterest.com/v3_sitemaps/seasonal_events_sitemap_20261130_cyber_monday_www.pinterest.com.xml\nSitemap: https://www.pinterest.com/v3_sitemaps/seasonal_events_sitemap_20261231_world_azerbaijanis_solidarity_day_www.pinterest.com.xml\nSitemap: https://www.pinterest.com/v3_sitemaps/seasonal_events_sitemap_20261129_first_sunday_of_advent_www.pinterest.com.xml\nSitemap: https://www.pinterest.com/v3_sitemaps/today_article_sitemap_www.pinterest.com.xml\nSitemap: https://www.pinterest.com/v3_sitemaps/seasonal_events_sitemap_20261205_international_volunteer_day_for_economic_and_socia_www.pinterest.com.xml\nSitemap: https://www.pinterest.com/v3_sitemaps/ideas_hub_v_expansion_sitemap_batch_2_www.pinterest.com.xml\nSitemap: https://www.pinterest.com/v3_sitemaps/seasonal_events_sitemap_20261210_international_human_rights_day_www.pinterest.com.xml\nSitemap: https://www.pinterest.com/v3_sitemaps/seasonal_events_sitemap_20261208_hanukkah_day_4_www.pinterest.com.xml\nSitemap: https://www.pinterest.com/v3_sitemaps/seasonal_events_sitemap_20261227_international_day_of_epidemic_preparedness_www.pinterest.com.xml\nSitemap: https://www.pinterest.com/v3_sitemaps/seasonal_events_sitemap_20261212_last_day_of_chanukah_www.pinterest.com.xml\nSitemap: https://www.pinterest.com/v3_sitemaps/seasonal_events_sitemap_20261127_aviation_day_www.pinterest.com.xml\nSitemap: https://www.pinterest.com/v3_sitemaps/seasonal_events_sitemap_20261129_first_advent_sunday_www.pinterest.com.xml\nSitemap: https://www.pinterest.com/v3_sitemaps/seasonal_events_sitemap_20261206_cayman_thanksgiving_www.pinterest.com.xml\nSitemap: https://www.pinterest.com/v3_sitemaps/ideas_hub_answers_terms_sitemap_www.pinterest.com.xml\nSitemap: https://www.pinterest.com/v3_sitemaps/seasonal_events_sitemap_20261127_black_friday_www.pinterest.com.xml\nSitemap: https://www.pinterest.com/v3_sitemaps/seasonal_events_sitemap_20261226_2nd_christmas_day_www.pinterest.com.xml\nSitemap: https://www.pinterest.com/v3_sitemaps/seasonal_events_sitemap_20261207_the_feast_of_st_ambrose_milan_www.pinterest.com.xml\nSitemap: https://www.pinterest.com/v3_sitemaps/seasonal_events_sitemap_20261224_washington_s_birthday_www.pinterest.com.xml\nSitemap: https://www.pinterest.com/v3_sitemaps/gift_board_link_sitemap_www.pinterest.com.xml\nSitemap: https://www.pinterest.com/v3_sitemaps/image_pin_sitemap_recent_engagement_tier_0_www.pinterest.com.xml\nSitemap: https://www.pinterest.com/v3_sitemaps/image_pin_sitemap_high_engagement_tier_0_www.pinterest.com.xml\nSitemap: https://www.pinterest.com/v3_sitemaps/image_pin_high_quality_tier_0_www.pinterest.com.xml\nSitemap: https://www.pinterest.com/v3_sitemaps/board_link_sitemap_tier_3_quality_uncrawled_www.pinterest.com.xml\nSitemap: https://www.pinterest.com/v3_sitemaps/priority_segment_2026q3_www.pinterest.com.xml\nSitemap: https://www.pinterest.com/v3_sitemaps/board_link_sitemap_high_cohesion_tier_0_www.pinterest.com.xml\nSitemap: https://www.pinterest.com/v3_sitemaps/board_link_sitemap_priority_segment_2026q3_www.pinterest.com.xml\nSitemap: https://www.pinterest.com/v3_sitemaps/user_link_sitemap_www.pinterest.com.xml\nSitemap: https://www.pinterest.com/v3_sitemaps/board_link_sitemap_www.pinterest.com.xml\nSitemap: https://www.pinterest.com/v3_sitemaps/seasonal_halloween_www.pinterest.com.xml\nSitemap: https://www.pinterest.com/v3_sitemaps/image_pin_sitemap_high_engagement_tier_1_www.pinterest.com.xml\nSitemap: https://www.pinterest.com/v3_sitemaps/video_pin_sitemap_www.pinterest.com.xml\nSitemap: https://www.pinterest.com/v3_sitemaps/board_link_sitemap_recent_engagement_tier_1_www.pinterest.com.xml\nSitemap: https://www.pinterest.com/v3_sitemaps/board_link_sitemap_recent_engagement_tier_0_www.pinterest.com.xml\nSitemap: https://www.pinterest.com/v3_sitemaps/board_link_sitemap_high_cohesion_tier_1_www.pinterest.com.xml\nSitemap: https://www.pinterest.com/v3_sitemaps/board_link_sitemap_seasonal_halloween_www.pinterest.com.xml\nSitemap: https://www.pinterest.com/v3_sitemaps/product_pin_sitemap_www.pinterest.com.xml\nSitemap: https://www.pinterest.com/v3_sitemaps/page_quality_model_high_score_www.pinterest.com.xml\nSitemap: https://www.pinterest.com/v3_sitemaps/board_link_sitemap_tier_2_quality_uncrawled_www.pinterest.com.xml\nSitemap: https://www.pinterest.com/v3_sitemaps/board_link_sitemap_tier_0_quality_uncrawled_www.pinterest.com.xml\nSitemap: https://www.pinterest.com/v3_sitemaps/image_pin_sitemap_www.pinterest.com.xml\nSitemap: https://www.pinterest.com/v3_sitemaps/video_product_pin_sitemap_www.pinterest.com.xml\nSitemap: https://www.pinterest.com/v3_sitemaps/board_link_sitemap_tier_1_quality_uncrawled_www.pinterest.com.xml\nSitemap: https://www.pinterest.com/v3_sitemaps/image_pin_high_quality_tier_1_www.pinterest.com.xml\nSitemap: https://www.pinterest.com/v3_sitemaps/image_pin_sitemap_recently_created_www.pinterest.com.xml\nSitemap: https://www.pinterest.com/v3_sitemaps/image_pin_sitemap_recent_engagement_tier_1_www.pinterest.com.xml\nSitemap: https://www.pinterest.com/v3_sitemaps/seasonal_halloween_segment_www.pinterest.com.xml\nSitemap: https://www.pinterest.com/v3_sitemaps/christmas_pin_www.pinterest.com.xml\nSitemap: https://www.pinterest.com/v3_sitemaps/new_years_eve_pin_www.pinterest.com.xml\nSitemap: https://www.pinterest.com/v3_sitemaps/christmas_pin_reinforce_www.pinterest.com.xml\nSitemap: https://www.pinterest.com/v3_sitemaps/new_years_eve_pin_newset_www.pinterest.com.xml\nSitemap: https://www.pinterest.com/v3_sitemaps/christmas_pin_newset_www.pinterest.com.xml\nSitemap: https://www.pinterest.com/v3_sitemaps/new_years_eve_pin_reinforce_www.pinterest.com.xml\nSitemap: https://www.pinterest.com/v3_sitemaps/seasonal_events_sitemap_www.pinterest.com.xml\nSitemap: https://www.pinterest.com/v3_sitemaps/answers_page_sitemap_www.pinterest.com.xml\nSitemap: https://www.pinterest.com/v3_sitemaps/christmas_new_year_eve_pin_www.pinterest.com.xml","agents":{"GPTBot":false,"OAI-SearchBot":false,"ChatGPT-User":false,"ClaudeBot":false,"Claude-SearchBot":false,"PerplexityBot":false,"Google-Extended":false,"Applebot-Extended":true,"CCBot":false,"Bytespider":false,"Meta-ExternalAgent":false,"Amazonbot":false}},"sitemap":{"status":404,"content_type":"text/html","bytes":2987,"sha256":"f12a9c5ad733495c85dfe076a1df1ffb9389f13560260a3334a7c493e40325de","final_host":"www.pinterest.com","ms":191,"url":"https://www.pinterest.com/v3_sitemaps/seasonal_events_sitemap_20261125_international_day_for_the_elimination_of_violence_www.pinterest.com.xml","declared":true,"xml":false},"files":{"llms_txt":{"status":200,"content_type":"text/html","bytes":1272618,"sha256":"ad8ce20265a433eab89d1a3973e480529926ba08d06a80435f41cce56c768fe8","final_host":"www.pinterest.com","ms":324,"path":"/llms.txt","html":true,"json":null,"present":false},"agents_md":{"status":200,"content_type":"text/html","bytes":1272937,"sha256":"334d2a0f1f90f84f24439fd514f8cdb7a2a2201ba5a463d2fbd932ddf67c70fa","final_host":"www.pinterest.com","ms":226,"path":"/agents.md","html":true,"json":null,"present":false},"agent_skills":{"status":200,"content_type":"text/html","bytes":1277129,"sha256":"8dac352a89e1ccb26c211ec632441c09973bec7abfdfae1eebdbe23cbe6de1e8","final_host":"www.pinterest.com","ms":335,"path":"/.well-known/agent-skills/index.json","html":true,"json":null,"present":false},"mcp_card":{"status":200,"content_type":"text/html","bytes":1277155,"sha256":"289724b2cc6d0e3bb202ff735059515d3954cdc1b983ff51ae58aae116d9229c","final_host":"www.pinterest.com","ms":315,"path":"/.well-known/mcp/server-card.json","html":true,"json":null,"present":false},"api_catalog":{"status":200,"content_type":"text/html","bytes":1275316,"sha256":"c1a7e8ef9d694235bce2defbc81a7bf68dbe29fcde6e14ca0c855d4ebf81f55b","final_host":"www.pinterest.com","ms":387,"path":"/.well-known/api-catalog","html":true,"json":null,"present":false},"a2a_card":{"status":200,"content_type":"text/html","bytes":1275297,"sha256":"742075db505cdb722db350c789ceee70c73c605efcd9cb31413b1840e7434c87","final_host":"www.pinterest.com","ms":243,"path":"/.well-known/agent-card.json","html":true,"json":null,"present":false}},"rings":{"v":1,"candidates":[],"mcp":[],"a2a":null,"webmcp":{"model_context":false,"bridge":false,"mcp_json":false,"link_rel":false,"any":false},"handshakes":0,"get_path":{"status":200,"content_type":"text/html","ms":574,"error":null},"get_host":{"status":530,"content_type":"text/plain","ms":92,"error":null}},"derived":{"reachable":true,"robots_served":true,"robots_sitemap":true,"content_signal":false,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":["GPTBot","OAI-SearchBot","ChatGPT-User","ClaudeBot","Claude-SearchBot","PerplexityBot","Google-Extended","CCBot","Bytespider","Meta-ExternalAgent","Amazonbot"],"blocks_any_ai":true,"files":{"llms_txt":false,"agents_md":false,"agent_skills":false,"mcp_card":false,"api_catalog":false,"a2a_card":false},"any_capability":false,"ai_refused":["gptbot","oaisearch","perplexity"],"ai_thin":[],"rings":{"mcp_declared":false,"mcp_live":false,"mcp_auth":false,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":2,"fh":"www.pinterest.com","fp_other_host":[],"fp":"414138e1c61d07db","fp_bits":{}},"derive_v":3,"frame":"top1k","shard":4},{"v":1,"kind":"crawlcheck-census-record","run":14,"domain":"tiktok.com","rank":54,"measured_at":"2026-10-15T05:31:08.958Z","home":{"browser":{"status":200,"content_type":"text/html","bytes":171147,"sha256":"8db4331d44bbe0204562694902a6e58292287b7d8ad1c58e742dcb88646caa40","final_host":"www.tiktok.com","ms":301,"identity":"Mobile browser","html":true},"gptbot":{"status":200,"content_type":"text/html","bytes":376649,"sha256":"85ca13ab0486ae43d5e06b0b0ca16035801601086e8258b48dc2f7c5909dd436","final_host":"www.tiktok.com","ms":360,"identity":"GPTBot","html":true},"claudebot":{"status":200,"content_type":"text/html","bytes":368434,"sha256":"65af87112bb2563d75c1c34ff1e0c67ad323f024857e542202f9fdf950fcfc50","final_host":"www.tiktok.com","ms":351,"identity":"ClaudeBot","html":true},"oaisearch":{"status":200,"content_type":"text/html","bytes":368391,"sha256":"c7e79086044797848234ba9ccd37408a72a35f21d2c56f6f1926193514e1e863","final_host":"www.tiktok.com","ms":358,"identity":"OAI-SearchBot","html":true},"perplexity":{"status":200,"content_type":"text/html","bytes":339632,"sha256":"90de2cbdec03c57d7374425c5a51ce196eb93939b5162b45b237bdfce6110198","final_host":"www.tiktok.com","ms":419,"identity":"PerplexityBot","html":true}},"robots":{"status":200,"content_type":"text/plain","bytes":1567,"sha256":"835705a2ecb862f49a813b3aa4061e0edfefbb8318316b371f8fa2dba4018802","final_host":"www.tiktok.com","ms":259,"html":false,"text":"User-agent: Baiduspider\nUser-agent: 360Spider\nUser-agent: Sogouspider\nUser-agent: Yisouspider\nUser-agent: PetalBot\nUser-agent: GPTBot\nUser-agent: OAI-SearchBot\nUser-agent: anthropic-ai\nUser-agent: ClaudeBot\nUser-agent: PerplexityBot\nUser-agent: Google-Extended\nUser-agent: Applebot-Extended\nUser-agent: meta-externalagent\nUser-agent: DuckAssistBot\nUser-agent: ChatGPT-User\nUser-agent: Gemini-Deep-Research\nUser-agent: GoogleAgent-Mariner\nUser-agent: Google-NotebookLM\nUser-agent: CCBot\nUser-agent: Claude-User\nUser-agent: Claude-SearchBot\nUser-agent: Perplexity-User\nUser-agent: AI2Bot\nUser-agent: MistralAI-User\nUser-agent: Bytespider\nDisallow: /\n\nUser-agent: *\nAllow: /foryou\nAllow: /discover\nAllow: /about\nAllow: /legal\nAllow: /safety\nAllow: /forgood\nAllow: /community-guidelines\nAllow: /tag\nAllow: /amp\nAllow: /transparency\nAllow: /share\nAllow: /music\nDisallow: /inapp\nDisallow: /auth\nDisallow: /embed/@\nDisallow: /embed/v2\nDisallow: /embed/curated\nDisallow: /link\nDisallow: */directory/\nDisallow: /search/video?\nDisallow: /search/user?q=\nDisallow: /shop/view/product/\nDisallow: /sgtm/g/collect\nDisallow: /api/share/settings\nDisallow: /api/recommend/embed_videos\nDisallow: /discover/trending/detail/\nDisallow: /search?\nDisallow: /search/live?\n\nUser-agent: Bingbot\nDisallow: /discover\nDisallow: /inapp\nDisallow: /auth\nDisallow: /embed/@\nDisallow: /embed/v2\nDisallow: /embed/curated\nDisallow: /link\nDisallow: */directory/\nDisallow: /shop/view/product/\nDisallow: /sgtm/g/collect\nDisallow: /api/share/settings\nDisallow: /api/recommend/embed_videos\nDisallow: /search?\n","agents":{"GPTBot":false,"OAI-SearchBot":false,"ChatGPT-User":false,"ClaudeBot":false,"Claude-SearchBot":false,"PerplexityBot":false,"Google-Extended":false,"Applebot-Extended":false,"CCBot":false,"Bytespider":false,"Meta-ExternalAgent":false,"Amazonbot":true}},"sitemap":{"status":404,"content_type":"text/plain","bytes":9,"sha256":"0019dfc4b32d63c1392aa264aed2253c1e0c2fb09216f8e2cc269bbfb8bb49b5","final_host":"www.tiktok.com","ms":229,"url":"https://tiktok.com/sitemap.xml","declared":false,"xml":false},"files":{"llms_txt":{"status":200,"content_type":"text/html","bytes":33589,"sha256":"a222a5a76560e1a573ef39647c25cfedc8de4c3c8eda01ab24d7478c1d122657","final_host":"www.tiktok.com","ms":251,"path":"/llms.txt","html":true,"json":null,"present":false},"agents_md":{"status":200,"content_type":"text/html","bytes":33589,"sha256":"a222a5a76560e1a573ef39647c25cfedc8de4c3c8eda01ab24d7478c1d122657","final_host":"www.tiktok.com","ms":335,"path":"/agents.md","html":true,"json":null,"present":false},"agent_skills":{"status":200,"content_type":"text/html","bytes":33589,"sha256":"a222a5a76560e1a573ef39647c25cfedc8de4c3c8eda01ab24d7478c1d122657","final_host":"www.tiktok.com","ms":245,"path":"/.well-known/agent-skills/index.json","html":true,"json":null,"present":false},"mcp_card":{"status":200,"content_type":"text/html","bytes":33589,"sha256":"a222a5a76560e1a573ef39647c25cfedc8de4c3c8eda01ab24d7478c1d122657","final_host":"www.tiktok.com","ms":391,"path":"/.well-known/mcp/server-card.json","html":true,"json":null,"present":false},"api_catalog":{"status":200,"content_type":"text/html","bytes":33589,"sha256":"a222a5a76560e1a573ef39647c25cfedc8de4c3c8eda01ab24d7478c1d122657","final_host":"www.tiktok.com","ms":279,"path":"/.well-known/api-catalog","html":true,"json":null,"present":false},"a2a_card":{"status":200,"content_type":"text/html","bytes":33589,"sha256":"a222a5a76560e1a573ef39647c25cfedc8de4c3c8eda01ab24d7478c1d122657","final_host":"www.tiktok.com","ms":232,"path":"/.well-known/agent-card.json","html":true,"json":null,"present":false}},"rings":{"v":1,"candidates":[],"mcp":[],"a2a":null,"webmcp":{"model_context":false,"bridge":false,"mcp_json":false,"link_rel":false,"any":false},"handshakes":0,"get_path":{"status":200,"content_type":"text/html","ms":242,"error":null},"get_host":{"status":530,"content_type":"text/plain","ms":55,"error":null}},"derived":{"reachable":true,"robots_served":true,"robots_sitemap":false,"content_signal":false,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":["GPTBot","OAI-SearchBot","ChatGPT-User","ClaudeBot","Claude-SearchBot","PerplexityBot","Google-Extended","Applebot-Extended","CCBot","Bytespider","Meta-ExternalAgent"],"blocks_any_ai":true,"files":{"llms_txt":false,"agents_md":false,"agent_skills":false,"mcp_card":false,"api_catalog":false,"a2a_card":false},"any_capability":false,"ai_refused":[],"ai_thin":[],"rings":{"mcp_declared":false,"mcp_live":false,"mcp_auth":false,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":2,"fh":"www.tiktok.com","fp_other_host":[]},"derive_v":3,"frame":"top1k","shard":5},{"v":1,"kind":"crawlcheck-census-record","run":14,"domain":"whatsapp.com","rank":55,"measured_at":"2026-10-15T05:33:03.639Z","home":{"browser":{"status":400,"content_type":"text/html","bytes":6278,"sha256":"50eff5ad5a798e51f6df6179c35aecc2b8148113e8f27f1292e3491915c43636","final_host":"www.whatsapp.com","ms":207,"identity":"Mobile browser","html":true},"gptbot":{"status":200,"content_type":"text/html","bytes":274607,"sha256":"1e44224e6f04181fb04b926cff71e006173dcaab512453a3f31f5370e6ce0110","final_host":"www.whatsapp.com","ms":480,"identity":"GPTBot","html":true,"fp":"031ccbb7a3071b56"},"claudebot":{"status":200,"content_type":"text/html","bytes":274566,"sha256":"3764b8d8214c3842653a469ac39bd393b73755de8004700d9a393bf944a3f44f","final_host":"www.whatsapp.com","ms":396,"identity":"ClaudeBot","html":true,"fp":"031ccbb7a3071b56"},"oaisearch":{"status":200,"content_type":"text/html","bytes":274614,"sha256":"caaa322ac41e99e89ecd1606d7dcf2a3e345411539d476eff79eabb6181b1f0a","final_host":"www.whatsapp.com","ms":379,"identity":"OAI-SearchBot","html":true,"fp":"031ccbb7a3071b56"},"perplexity":{"status":200,"content_type":"text/html","bytes":291924,"sha256":"1ae7fdc9aaa711e40c92cf219751cd62cd35026879f11d9b3aa63c5546b09d05","final_host":"www.whatsapp.com","ms":314,"identity":"PerplexityBot","html":true,"fp":"231acaf7a30f4516"}},"robots":{"status":200,"content_type":"text/plain","bytes":1180,"sha256":"b0dc29a03f811635b16c416f0c890b4acad4112533b78feed9bdb13de7156cf9","final_host":"www.whatsapp.com","ms":116,"html":false,"text":"# Notice: Collection of data on Facebook through automated means is\n# prohibited unless you have express written permission from Facebook\n# and may only be conducted for the limited purpose contained in said\n# permission.\n# See: http://www.facebook.com/apps/site_scraping_tos_terms.php\n\nUser-agent: Amazonbot\nDisallow: /\n\nUser-agent: Applebot-Extended\nDisallow: /\n\nUser-agent: Brightbot\nDisallow: /\n\nUser-agent: ClaudeBot\nDisallow: /\n\nUser-agent: Google-Extended\nDisallow: /\n\nUser-agent: GPTBot\nDisallow: /\n\nUser-agent: PerplexityBot\nDisallow: /\n\nUser-agent: PetalBot\nDisallow: /\n\nUser-agent: Scrapy\nDisallow: /\n\nUser-agent: uptimerobot\nDisallow: /\n\nUser-agent: viberbot\nDisallow: /\n\nUser-agent: YaK\nDisallow: /\n\nUser-agent: Yandex\nDisallow: /\n\nUser-agent: Yeti\nDisallow: /\n\nUser-agent: facebookexternalhit\nAllow: *\n\nUser-agent: meta-externalads\nAllow: *\n\nUser-agent: *\nDisallow: /*cursor=\nDisallow: /*fb_comment_id=\nDisallow: /ajax/\nDisallow: /tealium/\nDisallow: /intern/\nDisallow: /internal/\nDisallow: /login/\nDisallow: /oidc/callback/\nDisallow: /*.php\nDisallow: /android/third_party\nDisallow: /settings\n\nSitemap: https://www.whatsapp.com/sitemap/www_whatsapp_com_sitemap.xml.gz","agents":{"GPTBot":false,"OAI-SearchBot":true,"ChatGPT-User":true,"ClaudeBot":false,"Claude-SearchBot":true,"PerplexityBot":false,"Google-Extended":false,"Applebot-Extended":false,"CCBot":true,"Bytespider":true,"Meta-ExternalAgent":true,"Amazonbot":false}},"sitemap":null,"files":null,"derived":{"reachable":false,"robots_served":false,"robots_sitemap":false,"content_signal":false,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":[],"blocks_any_ai":false,"files":{"llms_txt":false,"agents_md":false,"agent_skills":false,"mcp_card":false,"api_catalog":false,"a2a_card":false},"any_capability":false,"ai_refused":[],"ai_thin":[],"rings":{"mcp_declared":false,"mcp_live":false,"mcp_auth":false,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":2,"fh":"www.whatsapp.com","fp_other_host":[]},"derive_v":3,"frame":"top1k","shard":6},{"v":1,"kind":"crawlcheck-census-record","run":14,"domain":"googledomains.com","rank":56,"measured_at":"2026-10-15T05:34:03.814Z","home":{"browser":{"status":200,"content_type":"text/html","bytes":5354,"sha256":"bd6285f79bbfc33187e5b3d849a4a6af7cc56cd96f325649c9adeb2d53a011cc","final_host":"domains.google","ms":365,"identity":"Mobile browser","html":true,"fp":"e51811cd1864f4d8"},"gptbot":{"status":200,"content_type":"text/html","bytes":5354,"sha256":"a527aa9d161d5eb89233f66833e3f1aaf9dec69f6530780b79e88c68d8e53916","final_host":"domains.google","ms":362,"identity":"GPTBot","html":true,"fp":"e51811cd1864f4d8"},"claudebot":{"status":200,"content_type":"text/html","bytes":5354,"sha256":"4bea988f91aad573e7f97765e7ec94ded49a18aaf457ba4cecf66335f562ba53","final_host":"domains.google","ms":336,"identity":"ClaudeBot","html":true,"fp":"e51811cd1864f4d8"},"oaisearch":{"status":200,"content_type":"text/html","bytes":5354,"sha256":"c19bfa66629ed01d92a6d83fe53cc30a502e42031b313b90f7f15d4bbea0000b","final_host":"domains.google","ms":293,"identity":"OAI-SearchBot","html":true,"fp":"e51811cd1864f4d8"},"perplexity":{"status":200,"content_type":"text/html","bytes":5354,"sha256":"7bed58800644adcb90c19452e22f03b0dd2f6455a2a3852a334ccc0356882121","final_host":"domains.google","ms":282,"identity":"PerplexityBot","html":true,"fp":"e51811cd1864f4d8"}},"robots":{"status":200,"content_type":"text/html","bytes":5354,"sha256":"5eb731bb858f1e3377520e643a7be143dc388f2d6bc28c4f10e6b99fd4f3ed9f","final_host":"domains.google","ms":263,"html":true,"text":null,"agents":null},"sitemap":{"status":200,"content_type":"text/html","bytes":5354,"sha256":"effe0fc40f8d2000be32b333c6515d9db0caed0d38fed730ff75d0db0beca9a5","final_host":"domains.google","ms":59,"url":"https://googledomains.com/sitemap.xml","declared":false,"xml":false},"files":{"llms_txt":{"status":200,"content_type":"text/html","bytes":5354,"sha256":"12800a1ef80601b872c6e586417afc12d0cc125eadb8f2b4fa26b9cd0b17bf3f","final_host":"domains.google","ms":61,"path":"/llms.txt","html":true,"json":null,"present":false},"agents_md":{"status":200,"content_type":"text/html","bytes":5354,"sha256":"b541322bca6ce646bd40654911e0ceea3b8a553decdf25c7d2d2db2259839735","final_host":"domains.google","ms":60,"path":"/agents.md","html":true,"json":null,"present":false},"agent_skills":{"status":200,"content_type":"text/html","bytes":5354,"sha256":"72d6ae520eb7d4c4fa7b5fd92dd7526bad47243708edfe595c381ec944c5a572","final_host":"domains.google","ms":56,"path":"/.well-known/agent-skills/index.json","html":true,"json":null,"present":false},"mcp_card":{"status":200,"content_type":"text/html","bytes":5354,"sha256":"2053f85684fb78324aa052a099cdf71f791cfb1fbdbbc4efe9cf3c6838acc60f","final_host":"domains.google","ms":61,"path":"/.well-known/mcp/server-card.json","html":true,"json":null,"present":false},"api_catalog":{"status":200,"content_type":"text/html","bytes":5354,"sha256":"ef61e23ea6053e27453b7b988b50077e4d20d1cf71e916f5a13e5928f36ed2e5","final_host":"domains.google","ms":63,"path":"/.well-known/api-catalog","html":true,"json":null,"present":false},"a2a_card":{"status":200,"content_type":"text/html","bytes":5354,"sha256":"872b07693816c8882adc859d58507a81276fb23386627f1692fb3b99939b6def","final_host":"domains.google","ms":55,"path":"/.well-known/agent-card.json","html":true,"json":null,"present":false}},"rings":{"v":1,"candidates":[],"mcp":[],"a2a":null,"webmcp":{"model_context":false,"bridge":false,"mcp_json":false,"link_rel":false,"any":false},"handshakes":0,"get_path":{"status":200,"content_type":"text/html","ms":63,"error":null},"get_host":{"status":530,"content_type":"text/plain","ms":88,"error":null}},"derived":{"reachable":true,"robots_served":false,"robots_sitemap":false,"content_signal":false,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":[],"blocks_any_ai":false,"files":{"llms_txt":false,"agents_md":false,"agent_skills":false,"mcp_card":false,"api_catalog":false,"a2a_card":false},"any_capability":false,"ai_refused":[],"ai_thin":[],"rings":{"mcp_declared":false,"mcp_live":false,"mcp_auth":false,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":2,"fh":"domains.google","fp_other_host":[],"fp":"e51811cd1864f4d8","fp_bits":{"gptbot":0,"claudebot":0,"oaisearch":0,"perplexity":0}},"derive_v":3,"frame":"top1k","shard":7},{"v":1,"kind":"crawlcheck-census-record","run":14,"domain":"cloudflare.net","rank":57,"measured_at":"2026-10-15T05:26:03.595Z","home":{"browser":{"status":200,"content_type":"text/html","bytes":64639,"sha256":"8635f3def2f75788242166560093d3470cf04e16e7dd32d84d4bea2c11b2d295","final_host":"cloudflare.net","ms":285,"identity":"Mobile browser","html":true,"fp":"c598113e86d046fd"},"gptbot":{"status":403,"content_type":"text/html","bytes":5585,"sha256":"f675b910e7d64ba698221efd7c2869347da2ab4da9970b308e2dba9a45e9cd81","final_host":"cloudflare.net","ms":5,"identity":"GPTBot","html":true},"claudebot":{"status":200,"content_type":"text/html","bytes":64639,"sha256":"3a3486a682fd5e4d88827056fa8176f8a768274e527de2926be3f2ac360dc7c9","final_host":"cloudflare.net","ms":187,"identity":"ClaudeBot","html":true,"fp":"c598113e86d046fd"},"oaisearch":{"status":200,"content_type":"text/html","bytes":64639,"sha256":"e5f3b51ff926cea35472e669722bebf6df8c6f614624b166fc6de8b15fdf86f4","final_host":"cloudflare.net","ms":135,"identity":"OAI-SearchBot","html":true,"fp":"c598113e86d046fd"},"perplexity":{"status":200,"content_type":"text/html","bytes":64639,"sha256":"1cef20f95e27879ab9f5a7e6683cc39b5bbe778443e5a36005b051eed87140f1","final_host":"cloudflare.net","ms":27,"identity":"PerplexityBot","html":true,"fp":"c598113e86d046fd"}},"robots":{"status":200,"content_type":"text/plain","bytes":207,"sha256":"de73d32142f114b8d18bbfd7fa5779ccee5f7a1753873402cfdab59444834ccd","final_host":"cloudflare.net","ms":138,"html":false,"text":"User-agent: *\r\nCrawl-delay: 10\r\nAllow: /\r\n\r\nUser-agent: Eluta.ca\r\nCrawl-delay: 60\r\nAllow: /\r\n\r\nUser-agent: MS Search 6.0 Robot\r\nCrawl-delay: 60\r\nAllow: /\r\n\r\nUser-agent: gsa-crawler\r\nCrawl-delay: 60\r\nAllow: /","agents":{"GPTBot":true,"OAI-SearchBot":true,"ChatGPT-User":true,"ClaudeBot":true,"Claude-SearchBot":true,"PerplexityBot":true,"Google-Extended":true,"Applebot-Extended":true,"CCBot":true,"Bytespider":true,"Meta-ExternalAgent":true,"Amazonbot":true}},"sitemap":{"status":404,"content_type":"text/html","bytes":1286,"sha256":"049e1b638af15c4f3525aed1b4be5d3fd6fe361d999601cbc978dfb34a8c3285","final_host":"cloudflare.net","ms":147,"url":"https://cloudflare.net/sitemap.xml","declared":false,"xml":false},"files":{"llms_txt":{"status":404,"content_type":"text/html","bytes":1286,"sha256":"049e1b638af15c4f3525aed1b4be5d3fd6fe361d999601cbc978dfb34a8c3285","final_host":"cloudflare.net","ms":128,"path":"/llms.txt","html":true,"json":null,"present":false},"agents_md":{"status":404,"content_type":"text/html","bytes":1286,"sha256":"049e1b638af15c4f3525aed1b4be5d3fd6fe361d999601cbc978dfb34a8c3285","final_host":"cloudflare.net","ms":199,"path":"/agents.md","html":true,"json":null,"present":false},"agent_skills":{"status":200,"content_type":"text/html","bytes":11,"sha256":"ee172faa9c17e8e5f4b63ef56d6d6863a48c223de152be43781305c99e143200","final_host":"cloudflare.net","ms":176,"path":"/.well-known/agent-skills/index.json","html":true,"json":null,"present":false},"mcp_card":{"status":200,"content_type":"text/html","bytes":11,"sha256":"ee172faa9c17e8e5f4b63ef56d6d6863a48c223de152be43781305c99e143200","final_host":"cloudflare.net","ms":156,"path":"/.well-known/mcp/server-card.json","html":true,"json":null,"present":false},"api_catalog":{"status":200,"content_type":"text/html","bytes":11,"sha256":"ee172faa9c17e8e5f4b63ef56d6d6863a48c223de152be43781305c99e143200","final_host":"cloudflare.net","ms":286,"path":"/.well-known/api-catalog","html":true,"json":null,"present":false},"a2a_card":{"status":200,"content_type":"text/html","bytes":11,"sha256":"ee172faa9c17e8e5f4b63ef56d6d6863a48c223de152be43781305c99e143200","final_host":"cloudflare.net","ms":158,"path":"/.well-known/agent-card.json","html":true,"json":null,"present":false}},"rings":{"v":1,"candidates":[],"mcp":[],"a2a":null,"webmcp":{"model_context":false,"bridge":false,"mcp_json":false,"link_rel":false,"any":false},"handshakes":0,"get_path":{"status":404,"content_type":"text/html","ms":884,"error":null},"get_host":{"status":530,"content_type":"text/plain","ms":8,"error":null}},"derived":{"reachable":true,"robots_served":true,"robots_sitemap":false,"content_signal":false,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":[],"blocks_any_ai":false,"files":{"llms_txt":false,"agents_md":false,"agent_skills":false,"mcp_card":false,"api_catalog":false,"a2a_card":false},"any_capability":false,"ai_refused":["gptbot"],"ai_thin":[],"rings":{"mcp_declared":false,"mcp_live":false,"mcp_auth":false,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":2,"fh":"cloudflare.net","fp_other_host":[],"fp":"c598113e86d046fd","fp_bits":{"claudebot":0,"oaisearch":0,"perplexity":0}},"derive_v":3,"frame":"top1k","shard":0},{"v":1,"kind":"crawlcheck-census-record","run":14,"domain":"goo.gl","rank":58,"measured_at":"2026-10-15T05:27:03.729Z","home":{"browser":{"status":400,"content_type":"text/html","bytes":9233,"sha256":"467f85f08ed95e4d207f06ae2f39969cbf6c9c25a18a0929a0835ee367404d81","final_host":"goo.gl","ms":333,"identity":"Mobile browser","html":true},"gptbot":{"status":400,"content_type":"text/html","bytes":9233,"sha256":"467f85f08ed95e4d207f06ae2f39969cbf6c9c25a18a0929a0835ee367404d81","final_host":"goo.gl","ms":419,"identity":"GPTBot","html":true},"claudebot":{"status":400,"content_type":"text/html","bytes":9233,"sha256":"467f85f08ed95e4d207f06ae2f39969cbf6c9c25a18a0929a0835ee367404d81","final_host":"goo.gl","ms":316,"identity":"ClaudeBot","html":true},"oaisearch":{"status":400,"content_type":"text/html","bytes":9233,"sha256":"467f85f08ed95e4d207f06ae2f39969cbf6c9c25a18a0929a0835ee367404d81","final_host":"goo.gl","ms":412,"identity":"OAI-SearchBot","html":true},"perplexity":{"status":400,"content_type":"text/html","bytes":9233,"sha256":"467f85f08ed95e4d207f06ae2f39969cbf6c9c25a18a0929a0835ee367404d81","final_host":"goo.gl","ms":311,"identity":"PerplexityBot","html":true}},"robots":{"status":400,"content_type":"text/html","bytes":9258,"sha256":"bae5d66fbe371f66a934935346906d25bb35df6fb4cd5504a359cd324e89ea11","final_host":"goo.gl","ms":161,"html":true,"text":null,"agents":null},"sitemap":null,"files":null,"derived":{"reachable":false,"robots_served":false,"robots_sitemap":false,"content_signal":false,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":[],"blocks_any_ai":false,"files":{"llms_txt":false,"agents_md":false,"agent_skills":false,"mcp_card":false,"api_catalog":false,"a2a_card":false},"any_capability":false,"ai_refused":[],"ai_thin":[],"rings":{"mcp_declared":false,"mcp_live":false,"mcp_auth":false,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":2,"fh":"goo.gl","fp_other_host":[]},"derive_v":3,"frame":"top1k","shard":1},{"v":1,"kind":"crawlcheck-census-record","run":14,"domain":"yahoo.com","rank":59,"measured_at":"2026-10-15T05:28:03.923Z","home":{"browser":{"status":200,"content_type":"text/html","bytes":556529,"sha256":"e634c47d89b4ea246ef3db45c5b28141c69e63add22eebc224875acd8075d66b","final_host":"www.yahoo.com","ms":895,"identity":"Mobile browser","html":true,"fp":"28c39f15165fa98c"},"gptbot":{"status":429,"content_type":"text/html","bytes":23,"sha256":"0d24c98db98d3b4a87b9626a026e7d72c99754ab6bd88211a17bc372350b7d89","final_host":"www.yahoo.com","ms":563,"identity":"GPTBot","html":true},"claudebot":{"status":429,"content_type":"text/html","bytes":23,"sha256":"0d24c98db98d3b4a87b9626a026e7d72c99754ab6bd88211a17bc372350b7d89","final_host":"www.yahoo.com","ms":458,"identity":"ClaudeBot","html":true},"oaisearch":{"status":404,"content_type":"text/html","bytes":1616,"sha256":"17ea8d8c4f070d0b6fd4c08935fcaae7553667f43acfdd8cb587a090d867e581","final_host":"yahoo.com","ms":533,"identity":"OAI-SearchBot","html":true},"perplexity":{"status":404,"content_type":"text/html","bytes":1616,"sha256":"17ea8d8c4f070d0b6fd4c08935fcaae7553667f43acfdd8cb587a090d867e581","final_host":"yahoo.com","ms":428,"identity":"PerplexityBot","html":true}},"robots":{"status":200,"content_type":"text/plain","bytes":1464,"sha256":"dfcd18f101783bde40c3bdca97bb90dbea1653321fd8b57f81cf5a6a6dc00b4b","final_host":"www.yahoo.com","ms":617,"html":false,"text":"User-agent: *\nDisallow: /info/p.gif\nDisallow: /p/\nDisallow: /r/\nDisallow: /bin/\nDisallow: /caas/\nDisallow: /blank.html\nDisallow: /includes/\nDisallow: /_td_api\nDisallow: /tdv2_fp\nDisallow: /nel_ms\nDisallow: /fp_ms\nDisallow: /sports_fp_ms\nDisallow: /search_ms\nDisallow: /_tdpp_api\nDisallow: /_remote\nDisallow: /_multiremote\nDisallow: /_tdhl_api\nDisallow: /digest\nDisallow: /fpjs\nDisallow: /myjs\nDisallow: /news/m/\n\nUser-agent: ADmantX\nUser-agent: AlphaBot\nUser-agent: anthropic-ai\nUser-agent: AwarioRssBot\nUser-agent: AwarioSmartBot\nUser-agent: BLEXBot\nUser-agent: Buzzbot\nUser-agent: Bytespider\nUser-agent: CCBot\nUser-agent: ChatGPT-User\nUser-agent: claritybot\nUser-agent: Claude-Web\nUser-agent: ClaudeBot\nUser-agent: cohere-ai\nUser-agent: Diffbot\nUser-agent: FacebookBot\nUser-agent: FriendlyCrawler\nUser-agent: Google-Extended\nUser-agent: GPTBot\nUser-agent: huggingface\nUser-agent: ImagesiftBot\nUser-agent: img2dataset\nUser-agent: magpie-crawler\nUser-agent: Meltwater\nUser-agent: Neevabot\nUser-agent: news-please\nUser-agent: NewsNow\nUser-agent: Nutch\nUser-agent: omgili\nUser-agent: omgilibot\nUser-agent: panscient.com\nUser-agent: Perplexity-ai\nUser-agent: PerplexityBot\nUser-agent: PetalBot\nUser-agent: PiplBot\nUser-agent: scoop.it\nUser-agent: Scrapy\nUser-agent: Seekr\nUser-agent: SentiBot\nUser-agent: SeznamBot\nUser-agent: TurnitinBot\nUser-agent: YouBot\nUser-agent: ZumBot\nDisallow: /\n\nUser-agent: Claude-SearchBot\nUser-agent: OAI-SearchBot\nDisallow: */articles/\n","agents":{"GPTBot":false,"OAI-SearchBot":true,"ChatGPT-User":false,"ClaudeBot":false,"Claude-SearchBot":true,"PerplexityBot":false,"Google-Extended":false,"Applebot-Extended":true,"CCBot":false,"Bytespider":false,"Meta-ExternalAgent":true,"Amazonbot":true}},"sitemap":{"status":404,"content_type":"text/html","bytes":4592,"sha256":"ba8cfc5afe9de0a71c2846bda68a1ae6e5fdcd2046101334bf2e2867ac38d277","final_host":"www.yahoo.com","ms":183,"url":"https://yahoo.com/sitemap.xml","declared":false,"xml":false},"files":{"llms_txt":{"status":404,"content_type":"text/html","bytes":4589,"sha256":"76aefe102edbdf82b6689c476ae26c10ae26ad718391f3012391aa46d2798d00","final_host":"www.yahoo.com","ms":209,"path":"/llms.txt","html":true,"json":null,"present":false},"agents_md":{"status":404,"content_type":"text/html","bytes":4590,"sha256":"2890bd397d1d1e842e1a432e3ec3a3444df30847571f6f64b385f49072279b4c","final_host":"www.yahoo.com","ms":209,"path":"/agents.md","html":true,"json":null,"present":false},"agent_skills":{"status":404,"content_type":"text/html","bytes":4618,"sha256":"7d73d69d13a4e747694f121b798b539fe843a6507a14a362200ead7bfa9452b6","final_host":"www.yahoo.com","ms":189,"path":"/.well-known/agent-skills/index.json","html":true,"json":null,"present":false},"mcp_card":{"status":404,"content_type":"text/html","bytes":4615,"sha256":"5732f5311b614ded0c37e19f9d4a8f49533a7d5024d81e04a41397eb2c1313fd","final_host":"www.yahoo.com","ms":251,"path":"/.well-known/mcp/server-card.json","html":true,"json":null,"present":false},"api_catalog":{"status":404,"content_type":"text/html","bytes":4606,"sha256":"97017f6bb073e56a65db22aeceeb531fc769e6dcf9d2e0295094e1256c35eeb1","final_host":"www.yahoo.com","ms":239,"path":"/.well-known/api-catalog","html":true,"json":null,"present":false},"a2a_card":{"status":404,"content_type":"text/html","bytes":4610,"sha256":"777fb51b565a7d61f063e88413f2d5999c6a15b15a35e6b11b53bca8275d3345","final_host":"www.yahoo.com","ms":243,"path":"/.well-known/agent-card.json","html":true,"json":null,"present":false}},"rings":{"v":1,"candidates":[],"mcp":[],"a2a":null,"webmcp":{"model_context":false,"bridge":false,"mcp_json":false,"link_rel":false,"any":false},"handshakes":0,"get_path":{"status":404,"content_type":"text/html","ms":206,"error":null},"get_host":{"status":530,"content_type":"text/plain","ms":53,"error":null}},"derived":{"reachable":true,"robots_served":true,"robots_sitemap":false,"content_signal":false,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":["GPTBot","ChatGPT-User","ClaudeBot","PerplexityBot","Google-Extended","CCBot","Bytespider"],"blocks_any_ai":true,"files":{"llms_txt":false,"agents_md":false,"agent_skills":false,"mcp_card":false,"api_catalog":false,"a2a_card":false},"any_capability":false,"ai_refused":["gptbot","claudebot","oaisearch","perplexity"],"ai_thin":[],"rings":{"mcp_declared":false,"mcp_live":false,"mcp_auth":false,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":2,"fh":"www.yahoo.com","fp_other_host":[],"fp":"28c39f15165fa98c","fp_bits":{}},"derive_v":3,"frame":"top1k","shard":2},{"v":1,"kind":"crawlcheck-census-record","run":14,"domain":"googlesyndication.com","rank":60,"measured_at":"2026-10-15T05:29:03.596Z","home":{"browser":{"status":429,"content_type":"text/html","bytes":3140,"sha256":"ccb9ec7e1972f6fc19689fd21117ce397c39ae23b68d26e2a776b435b7d0fd4c","final_host":"www.google.com","ms":524,"identity":"Mobile browser","html":true},"gptbot":{"status":429,"content_type":"text/html","bytes":3140,"sha256":"d8394a32b8e118c523dcb200c54d436d26a698937beb46af6dc38924cd70f281","final_host":"www.google.com","ms":349,"identity":"GPTBot","html":true},"claudebot":{"status":429,"content_type":"text/html","bytes":3140,"sha256":"65e14b5a6ccb3107d1d4b584e73e678004c9b949515207fee570cbf644a33c09","final_host":"www.google.com","ms":586,"identity":"ClaudeBot","html":true},"oaisearch":{"status":429,"content_type":"text/html","bytes":3140,"sha256":"317a048f73b8edc265e306a6f0e8ee661c36f3722766f7a1644095ef69ef8fda","final_host":"www.google.com","ms":536,"identity":"OAI-SearchBot","html":true},"perplexity":{"status":429,"content_type":"text/html","bytes":3140,"sha256":"138dc97fa89ea43c985c5cde4ed709bd8bdfb8c86d550984795538a5bc4759f7","final_host":"www.google.com","ms":359,"identity":"PerplexityBot","html":true}},"robots":{"status":404,"content_type":"text/html","bytes":1571,"sha256":"6adee83a3eba87d30fdca3997d27097697f2767805f222a0f78c3f04392004d2","final_host":"googlesyndication.com","ms":159,"html":true,"text":null,"agents":null},"sitemap":null,"files":null,"derived":{"reachable":false,"robots_served":false,"robots_sitemap":false,"content_signal":false,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":[],"blocks_any_ai":false,"files":{"llms_txt":false,"agents_md":false,"agent_skills":false,"mcp_card":false,"api_catalog":false,"a2a_card":false},"any_capability":false,"ai_refused":[],"ai_thin":[],"rings":{"mcp_declared":false,"mcp_live":false,"mcp_auth":false,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":2,"fh":"www.google.com","fp_other_host":[]},"derive_v":3,"frame":"top1k","shard":3},{"v":1,"kind":"crawlcheck-census-record","run":14,"domain":"gvt1.com","rank":61,"measured_at":"2026-10-15T05:30:03.842Z","home":{"browser":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"gvt1.com","ms":75,"identity":"Mobile browser","html":false},"gptbot":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"gvt1.com","ms":75,"identity":"GPTBot","html":false},"claudebot":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"gvt1.com","ms":32,"identity":"ClaudeBot","html":false},"oaisearch":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"gvt1.com","ms":76,"identity":"OAI-SearchBot","html":false},"perplexity":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"gvt1.com","ms":73,"identity":"PerplexityBot","html":false}},"robots":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"gvt1.com","ms":73,"html":false,"text":null,"agents":null},"sitemap":null,"files":null,"derived":{"reachable":false,"robots_served":false,"robots_sitemap":false,"content_signal":false,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":[],"blocks_any_ai":false,"files":{"llms_txt":false,"agents_md":false,"agent_skills":false,"mcp_card":false,"api_catalog":false,"a2a_card":false},"any_capability":false,"ai_refused":[],"ai_thin":[],"rings":{"mcp_declared":false,"mcp_live":false,"mcp_auth":false,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":2,"fh":"gvt1.com","fp_other_host":[]},"derive_v":3,"frame":"top1k","shard":4},{"v":1,"kind":"crawlcheck-census-record","run":14,"domain":"roblox.com","rank":62,"measured_at":"2026-10-15T05:31:08.958Z","home":{"browser":{"status":200,"content_type":"text/html","bytes":60254,"sha256":"32f6cb3e792fae3d203b55fbf74aa1ed78e5f732687be9ee8529d1d55e087dcd","final_host":"www.roblox.com","ms":438,"identity":"Mobile browser","html":true,"fp":"17b8511efc7419ed"},"gptbot":{"status":200,"content_type":"text/html","bytes":63856,"sha256":"5c95c8a8032724cd3476a72deeb0b312ff0de2394b91fb25c1e135f100a72184","final_host":"www.roblox.com","ms":348,"identity":"GPTBot","html":true},"claudebot":{"status":200,"content_type":"text/html","bytes":63856,"sha256":"20022041e82671089995d807c397945d6cd7458086ddaf2965a1e11276e52fc4","final_host":"www.roblox.com","ms":336,"identity":"ClaudeBot","html":true},"oaisearch":{"status":200,"content_type":"text/html","bytes":63856,"sha256":"9389cfda3981ff22d9579a75370313fe70558632661a499998b6a91da508e0c0","final_host":"www.roblox.com","ms":339,"identity":"OAI-SearchBot","html":true},"perplexity":{"status":200,"content_type":"text/html","bytes":63856,"sha256":"630427dc526d3df959b350227facbb93abf46c62b820e26b0f60533472aaec1c","final_host":"www.roblox.com","ms":277,"identity":"PerplexityBot","html":true}},"robots":{"status":200,"content_type":"text/plain","bytes":2181,"sha256":"a4a9450540cd8263a373527026c92df49b7289076e87a3bb5bb72a7c28bc5213","final_host":"www.roblox.com","ms":504,"html":false,"text":"user-agent: *\n\ndisallow: *.ashx\ndisallow: /abusereport/\ndisallow: /*/abusereport/\ndisallow: /admi/\ndisallow: /ads/\ndisallow: /catalog/contents\ndisallow: /catalog/html\ndisallow: /client-status\ndisallow: /data/\ndisallow: /error/\ndisallow: /forum/\ndisallow: /Forum/\ndisallow: /game/report-event\ndisallow: /game/report-stats\ndisallow: /ide/clienttoolbox\ndisallow: /javascript/\ndisallow: /reports/\ndisallow: /thumbnail/resolve-hash\ndisallow: /thumbnail_holder/g\ndisallow: /viewapp/\ndisallow: /error.html\ndisallow: /offline.html\ndisallow: /badges/\ndisallow: /*/badges/\ndisallow: /*.aspx$\ndisallow: /upgrades/checkout/success?giftPurchaseId=*\ndisallow: /report-abuse\ndisallow: /*/report-abuse\n\nSitemap: https://www.roblox.com/sitemap.xml\nSitemap: https://www.roblox.com/sitemap-games.xml\nSitemap: https://www.roblox.com/sitemap-ar.xml\nSitemap: https://www.roblox.com/sitemap-games-ar.xml\nSitemap: https://www.roblox.com/sitemap-de.xml\nSitemap: https://www.roblox.com/sitemap-games-de.xml\nSitemap: https://www.roblox.com/sitemap-es.xml\nSitemap: https://www.roblox.com/sitemap-games-es.xml\nSitemap: https://www.roblox.com/sitemap-fr.xml\nSitemap: https://www.roblox.com/sitemap-games-fr.xml\nSitemap: https://www.roblox.com/sitemap-hi.xml\nSitemap: https://www.roblox.com/sitemap-games-hi.xml\nSitemap: https://www.roblox.com/sitemap-id.xml\nSitemap: https://www.roblox.com/sitemap-games-id.xml\nSitemap: https://www.roblox.com/sitemap-it.xml\nSitemap: https://www.roblox.com/sitemap-games-it.xml\nSitemap: https://www.roblox.com/sitemap-ja.xml\nSitemap: https://www.roblox.com/sitemap-games-ja.xml\nSitemap: https://www.roblox.com/sitemap-ko.xml\nSitemap: https://www.roblox.com/sitemap-games-ko.xml\nSitemap: https://www.roblox.com/sitemap-pl.xml\nSitemap: https://www.roblox.com/sitemap-games-pl.xml\nSitemap: https://www.roblox.com/sitemap-pt.xml\nSitemap: https://www.roblox.com/sitemap-games-pt.xml\nSitemap: https://www.roblox.com/sitemap-th.xml\nSitemap: https://www.roblox.com/sitemap-games-th.xml\nSitemap: https://www.roblox.com/sitemap-tr.xml\nSitemap: https://www.roblox.com/sitemap-games-tr.xml\nSitemap: https://www.roblox.com/sitemap-vi.xml\nSitemap: https://www.roblox.com/sitemap-games-vi.xml\n","agents":{"GPTBot":true,"OAI-SearchBot":true,"ChatGPT-User":true,"ClaudeBot":true,"Claude-SearchBot":true,"PerplexityBot":true,"Google-Extended":true,"Applebot-Extended":true,"CCBot":true,"Bytespider":true,"Meta-ExternalAgent":true,"Amazonbot":true}},"sitemap":{"status":200,"content_type":"application/xml","bytes":63238,"sha256":"8afe97be0c0960c7648277041ce67c94e669d7e0f955be3ef98230d9ec7c7eaf","final_host":"www.roblox.com","ms":284,"url":"https://www.roblox.com/sitemap.xml","declared":true,"xml":true},"files":{"llms_txt":{"status":404,"content_type":"text/html","bytes":67345,"sha256":"2ce10a91fc26cf7a40ea9b973060cd32a3c9ee7aa59aa8e9771173c0ad975de6","final_host":"www.roblox.com","ms":191,"path":"/llms.txt","html":true,"json":null,"present":false},"agents_md":{"status":404,"content_type":"text/html","bytes":67345,"sha256":"3614d31284e1a632b1863ac14bc592c8ed9e98db22d74576263a12bac9571f35","final_host":"www.roblox.com","ms":190,"path":"/agents.md","html":true,"json":null,"present":false},"agent_skills":{"status":404,"content_type":"application/json","bytes":40,"sha256":"6ed55387e8fd77ee283840993d928721a8c3779847c65e57e058082802c75eff","final_host":"www.roblox.com","ms":163,"path":"/.well-known/agent-skills/index.json","html":false,"json":null,"present":false},"mcp_card":{"status":404,"content_type":"application/json","bytes":40,"sha256":"6ed55387e8fd77ee283840993d928721a8c3779847c65e57e058082802c75eff","final_host":"www.roblox.com","ms":174,"path":"/.well-known/mcp/server-card.json","html":false,"json":null,"present":false},"api_catalog":{"status":404,"content_type":"application/json","bytes":40,"sha256":"6ed55387e8fd77ee283840993d928721a8c3779847c65e57e058082802c75eff","final_host":"www.roblox.com","ms":162,"path":"/.well-known/api-catalog","html":false,"json":null,"present":false},"a2a_card":{"status":404,"content_type":"application/json","bytes":40,"sha256":"6ed55387e8fd77ee283840993d928721a8c3779847c65e57e058082802c75eff","final_host":"www.roblox.com","ms":297,"path":"/.well-known/agent-card.json","html":false,"json":null,"present":false}},"rings":{"v":1,"candidates":[],"mcp":[],"a2a":null,"webmcp":{"model_context":false,"bridge":false,"mcp_json":false,"link_rel":false,"any":false},"handshakes":0,"get_path":{"status":404,"content_type":"application/json","ms":177,"error":null},"get_host":{"status":530,"content_type":"text/plain","ms":103,"error":null}},"derived":{"reachable":true,"robots_served":true,"robots_sitemap":true,"content_signal":false,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":[],"blocks_any_ai":false,"files":{"llms_txt":false,"agents_md":false,"agent_skills":false,"mcp_card":false,"api_catalog":false,"a2a_card":false},"any_capability":false,"ai_refused":[],"ai_thin":[],"rings":{"mcp_declared":false,"mcp_live":false,"mcp_auth":false,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":2,"fh":"www.roblox.com","fp_other_host":[],"fp":"17b8511efc7419ed","fp_bits":{}},"derive_v":3,"frame":"top1k","shard":5},{"v":1,"kind":"crawlcheck-census-record","run":14,"domain":"endpoints.news","rank":63,"measured_at":"2026-10-15T05:33:03.639Z","home":{"browser":{"status":200,"content_type":"text/html","bytes":117186,"sha256":"dace25c1d286d0b04dbf902d856b542fc8cd640702c5ee6ffaee35d4e568f154","final_host":"endpoints.news","ms":350,"identity":"Mobile browser","html":true,"fp":"6fc95f64b786d32c"},"gptbot":{"status":403,"content_type":"text/html","bytes":919,"sha256":"05f861adc0a96a14fa4f2c54b7333bafb29fed9cdc85479cb2db0fde2e34ad1e","final_host":"endpoints.news","ms":237,"identity":"GPTBot","html":true},"claudebot":{"status":403,"content_type":"text/html","bytes":919,"sha256":"c6f2f8e9d3fb1dd7201a406e840ebafb6e88e1559a948df8819a541776fafc93","final_host":"endpoints.news","ms":63,"identity":"ClaudeBot","html":true},"oaisearch":{"status":403,"content_type":"text/html","bytes":919,"sha256":"fc056295ca1fd5c16d55a6c4cb273a28aa6caeaf606e46b312727acbcb93b20d","final_host":"endpoints.news","ms":59,"identity":"OAI-SearchBot","html":true},"perplexity":{"status":403,"content_type":"text/html","bytes":919,"sha256":"03819785372a19e54afb19ff1ef028790fb078c7052a144a5c48f766609d8c9c","final_host":"endpoints.news","ms":59,"identity":"PerplexityBot","html":true}},"robots":{"status":403,"content_type":"text/html","bytes":919,"sha256":"c301787b33f755c20e07df854e0f2ce3d346b29fae70f35c771006c0e640e68e","final_host":"endpoints.news","ms":53,"html":true,"text":null,"agents":null},"sitemap":{"status":403,"content_type":"text/html","bytes":919,"sha256":"540825f3f3de5b4f28218888a2dc49d429aff849517510b376724da9f3785b50","final_host":"endpoints.news","ms":11,"url":"https://endpoints.news/sitemap.xml","declared":false,"xml":false},"files":{"llms_txt":{"status":403,"content_type":"text/html","bytes":919,"sha256":"936169d15089f50ac99ea6e59fff3541e6b9800a34c18b3435be72473ad424a0","final_host":"endpoints.news","ms":39,"path":"/llms.txt","html":true,"json":null,"present":false},"agents_md":{"status":403,"content_type":"text/html","bytes":919,"sha256":"72340e3266b520c7a3f0ad2e691df3ba5eb9dab0bd8dcd57e3357b20aecf769e","final_host":"endpoints.news","ms":7,"path":"/agents.md","html":true,"json":null,"present":false},"agent_skills":{"status":403,"content_type":"text/html","bytes":919,"sha256":"237a276480d6d968c666190e5c646eb2231b8a64d1000282c874d14eeaec0b6e","final_host":"endpoints.news","ms":7,"path":"/.well-known/agent-skills/index.json","html":true,"json":null,"present":false},"mcp_card":{"status":403,"content_type":"text/html","bytes":919,"sha256":"b66536f92dba045e97b0bcefd279f44e292414148bac7c973503dd1f4f668b2c","final_host":"endpoints.news","ms":22,"path":"/.well-known/mcp/server-card.json","html":true,"json":null,"present":false},"api_catalog":{"status":403,"content_type":"text/html","bytes":919,"sha256":"337bd8ec2f59775f7eb430357b711f6dc0ac3649c0fe33f411afff7f13efb352","final_host":"endpoints.news","ms":6,"path":"/.well-known/api-catalog","html":true,"json":null,"present":false},"a2a_card":{"status":403,"content_type":"text/html","bytes":919,"sha256":"b7849e7772cd577a21f4bc15f4917b9bcc769be32ae9c296ddfa8c0b1bde6616","final_host":"endpoints.news","ms":16,"path":"/.well-known/agent-card.json","html":true,"json":null,"present":false}},"rings":{"v":1,"candidates":[],"mcp":[],"a2a":null,"webmcp":{"model_context":false,"bridge":false,"mcp_json":false,"link_rel":false,"any":false},"handshakes":0,"get_path":{"status":403,"content_type":"text/html","ms":8,"error":null},"get_host":{"status":530,"content_type":"text/plain","ms":40,"error":null}},"derived":{"reachable":true,"robots_served":false,"robots_sitemap":false,"content_signal":false,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":[],"blocks_any_ai":false,"files":{"llms_txt":false,"agents_md":false,"agent_skills":false,"mcp_card":false,"api_catalog":false,"a2a_card":false},"any_capability":false,"ai_refused":["gptbot","claudebot","oaisearch","perplexity"],"ai_thin":[],"rings":{"mcp_declared":false,"mcp_live":false,"mcp_auth":false,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":2,"fh":"endpoints.news","fp_other_host":[],"fp":"6fc95f64b786d32c","fp_bits":{}},"derive_v":3,"frame":"top1k","shard":6},{"v":1,"kind":"crawlcheck-census-record","run":14,"domain":"windows.net","rank":64,"measured_at":"2026-10-15T05:34:03.814Z","home":{"browser":{"status":526,"content_type":"text/plain","bytes":16,"sha256":"4c543fe8a4d739c6f622b04303ac9e9d9fafbf00da950fc916ad826da2c0cdac","final_host":"windows.net","ms":392,"identity":"Mobile browser","html":false},"gptbot":{"status":526,"content_type":"text/plain","bytes":16,"sha256":"4c543fe8a4d739c6f622b04303ac9e9d9fafbf00da950fc916ad826da2c0cdac","final_host":"windows.net","ms":444,"identity":"GPTBot","html":false},"claudebot":{"status":526,"content_type":"text/plain","bytes":16,"sha256":"4c543fe8a4d739c6f622b04303ac9e9d9fafbf00da950fc916ad826da2c0cdac","final_host":"windows.net","ms":888,"identity":"ClaudeBot","html":false},"oaisearch":{"status":526,"content_type":"text/plain","bytes":16,"sha256":"4c543fe8a4d739c6f622b04303ac9e9d9fafbf00da950fc916ad826da2c0cdac","final_host":"windows.net","ms":337,"identity":"OAI-SearchBot","html":false},"perplexity":{"status":null,"error":"timeout","ms":8000,"identity":"PerplexityBot","html":null}},"robots":{"status":526,"content_type":"text/plain","bytes":16,"sha256":"4c543fe8a4d739c6f622b04303ac9e9d9fafbf00da950fc916ad826da2c0cdac","final_host":"windows.net","ms":851,"html":false,"text":null,"agents":null},"sitemap":null,"files":null,"derived":{"reachable":false,"robots_served":false,"robots_sitemap":false,"content_signal":false,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":[],"blocks_any_ai":false,"files":{"llms_txt":false,"agents_md":false,"agent_skills":false,"mcp_card":false,"api_catalog":false,"a2a_card":false},"any_capability":false,"ai_refused":[],"ai_thin":[],"rings":{"mcp_declared":false,"mcp_live":false,"mcp_auth":false,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":2,"fh":"windows.net","fp_other_host":[]},"derive_v":3,"frame":"top1k","shard":7},{"v":1,"kind":"crawlcheck-census-record","run":14,"domain":"adobe.com","rank":65,"measured_at":"2026-10-15T05:26:03.595Z","home":{"browser":{"status":200,"content_type":"text/html","bytes":50037,"sha256":"baf1311608dd35d0ac356ae8a62a32ef1f2e8c41960e7cad838fdef67feef3cf","final_host":"www.adobe.com","ms":123,"identity":"Mobile browser","html":true,"fp":"dd0a412194ba14c2"},"gptbot":{"status":403,"content_type":"text/html","bytes":367,"sha256":"4571ace09e6dfd4e961bdc7904b4b29731c41bed0e5cac9296bdd6feb0c318b7","final_host":"www.adobe.com","ms":120,"identity":"GPTBot","html":true},"claudebot":{"status":403,"content_type":"text/html","bytes":367,"sha256":"8bbd8cff37cc9252b76f083e667c61603e29ad1a523c8fe65324a3c0fa3498bf","final_host":"www.adobe.com","ms":159,"identity":"ClaudeBot","html":true},"oaisearch":{"status":403,"content_type":"text/html","bytes":367,"sha256":"1bb6aab3305f9cbd1898b85d3235a2ebcb9f43aff5b64b578e26c27488cd92a4","final_host":"www.adobe.com","ms":87,"identity":"OAI-SearchBot","html":true},"perplexity":{"status":403,"content_type":"text/html","bytes":367,"sha256":"6ccd7be2d39769e5ae2b8de9f2b43715fd282766d67d71290128099c240b2c55","final_host":"www.adobe.com","ms":132,"identity":"PerplexityBot","html":true}},"robots":{"status":200,"content_type":"text/plain","bytes":10199,"sha256":"41081faa0fd323774e225ab51e8a73262055b24ecc7732557229fb93938f9d84","final_host":"www.adobe.com","ms":228,"html":false,"text":"# The use of robots or other automated means to access the Adobe site\n# without the express permission of Adobe is strictly prohibited.\n# Notwithstanding the foregoing, Adobe may permit automated access to\n# access certain Adobe pages but solely for the limited purpose of\n# including content in publicly available search engines. Any other\n# use of robots or failure to obey the robots exclusion standards set\n# forth at http://www.robotstxt.org/ is strictly prohibited.\n\n# Details about Googlebot available at: http://www.google.com/bot.html\n# The Google search engine can see everything\nUser-agent: gsa-crawler-www \nAllow: /assets/globalnav/v1.1.4/css/globalnav.css\nAllow: /assets/globalnav/v1.1.4/*/globalnav.js\nAllow: /au/special/symposium\nAllow: /sea/special/symposium\nAllow: /*special/products/marketing-cloud/*.css$\nAllow: /*special/products/marketing-cloud/*.png$\nAllow: /*special/products/marketing-cloud/*.js$\nAllow: /www.adobe.com/special/ri/prod/assets/*.css$\nAllow: /www.adobe.com/special/ri/prod/assets/*.jpg$\nAllow: /www.adobe.com/special/ri/prod/assets/*.js$\nAllow: /www.adobe.com/special/ri/prod/assets/*.png$\nAllow: /www.adobe.com/special/retail-shopping-insights/\nAllow: /uk/special/landing/student/digitaledgeaward/\nAllow: /uk/special/landing/student/digitaledgeaward/\nAllow: /uk/special/landing/student/lovethejourney\nAllow: /fr/special/landing/student/lovethejourney\nDisallow: /assets/globalnav/\nDisallow: /*creativesuite/bridge\nDisallow: /cfusion/gpr/\nDisallow: /cfusion/search/\nDisallow: /cfusion/partnerportal/\nDisallow: /globalsearch\nDisallow: /cgi-local/\nDisallow: /de/products/index-notabs.html\nDisallow: /de/products/index-tabs.html\nDisallow: /devnet-archive/\nDisallow: /devnet/archive\nDisallow: /*devnet/tagsearch\nDisallow: /emea/eemeachannel/\nDisallow: /events/ciocouncil/\nDisallow: /events/executivecouncil/\nDisallow: /google_indexing/\nDisallow: /*.html_\nDisallow: /licensing/distribution/strategies/\nDisallow: /*limited/\nDisallow: /livedocs/\nDisallow: /*Ljava.lang.Object*\nDisallow: /products/acrobat/readstep2.html;JMICROSITE2SESSION*\nDisallow: /*products/flashplayer/distribution3.html\nDisallow: /*products/flashplayer/distribution4.html\nDisallow: /*special/\nDisallow: /*.sql$\nDisallow: /*supportservice/\nDisallow: /*.translate.html\nDisallow: /*products/adobeconnect/enterprise/trial.html\nDisallow: /*products/adobeconnect/enterprise/trial-check.html\nDisallow: /*products/adobeconnect/enterprise/trial-thankyou.html\nDisallow: /*products/adobeconnect/enterprise/trial-error.html\nDisallow: /*products/photoshop/selector\nDisallow: /*unsubscribe.html\nDisallow: /content/dam/acom/en/marketing-cloud/offers/pdf\nDisallow: /content/dam/www/us/en/adi/2020/pdfs/*\nDisallow: /content/dotcom/*.html$\nDisallow: /content/dam/www/us/en/offer/digital-trends-2020/digital-trends-2020-full-report.pdf\nDisallow: /plans-fragments/\nDisallow: /plans.modals*\n\n# The Omniture search engine can see everything\nUser-agent: Atomz/1.0\nDisallow: /devnet-archive/\nDisallow: /devnet/archive\nDisallow: /events/ciocouncil/\nDisallow: /events/executivecouncil/\nDisallow: /licensing/distribution/strategies/\nDisallow: /*limited/\nDisallow: /livedocs/\nDisallow: /products/flashplayer/distribution3.html\nDisallow: /products/flashplayer/distribution4.html\nDisallow: /*special/\nDisallow: /*.sql$\nDisallow: /*supportservice/\n\nUser-agent: *\nAllow: /assets/globalnav/v1.1.4/css/globalnav.css\nAllow: /assets/globalnav/v1.1.4/*globalnav.js\nAllow: /au/special/symposium\nAllow: /sea/special/symposium\nAllow: /*special/products/marketing-cloud/*.css$\nAllow: /*special/products/marketing-cloud/*.js$\nAllow: /*special/products/marketing-cloud/*.png$\nAllow: /www.adobe.com/special/ri/prod/assets/*.css$\nAllow: /www.adobe.com/special/ri/prod/assets/*.jpg$\nAllow: /www.adobe.com/special/ri/prod/assets/*.js$\nAllow: /www.adobe.com/special/ri/prod/assets/*.png$\nAllow: /www.adobe.com/special/retail-shopping-insights/\nAllow: /uk/special/landing/student/digitaledgeaward/\nAllow: /uk/special/landing/student/lovethejourney\nAllow: /fr/special/landing/student/lovethejourney\nAllow: /community/gallery$\nAllow: /community/gallery/assets$\nAllow: /community/gallery/challenges$\nDisallow: /assets/globalnav/\nDisallow: /*creativesuite/bridge\nDisallow: /cfusion/gpr/\nDisallow: /cfusion/search/\nDisallow: /cfusion/partnerportal/\nDisallow: /globalsearch\nDisallow: /cgi-local/\nDisallow: /de/products/index-notabs.html\nDisallow: /de/products/index-tabs.html\nDisallow: /devnet-archive/\nDisallow: /devnet/archive\nDisallow: /*devnet/tagsearch\nDisallow: /emea/eemeachannel/\nDisallow: /events/ciocouncil/\nDisallow: /events/executivecouncil/\nDisallow: /google_indexing/\nDisallow: /*.html_\nDisallow: /licensing/distribution/strategies/\nDisallow: /*limited/\nDisallow: /livedocs/\nDisallow: /*Ljava.lang.Object*\nDisallow: /products/acrobat/readstep2.html;JMICROSITE2SESSION*\nDisallow: /*products/flashplayer/distribution3.html\nDisallow: /*products/flashplayer/distribution4.html\nDisallow: /*special/\nDisallow: /*.sql$\nDisallow: /*supportservice/\nDisallow: /*.translate.html\nDisallow: /*products/adobeconnect/enterprise/trial.html\nDisallow: /*products/adobeconnect/enterprise/trial-check.html\nDisallow: /*products/adobeconnect/enterprise/trial-thankyou.html\nDisallow: /*products/adobeconnect/enterprise/trial-error.html\nDisallow: /*products/photoshop/selector\nDisallow: /*unsubscribe.html\nDisallow: /content/dam/acom/en/marketing-cloud/offers/pdf\nDisallow: /content/dotcom/*.html$\nDisallow: /plans-fragments/\nDisallow: /content/dam/www/us/en/offer/digital-trends-2020-in-media-and-entertainment-html/digital-trends-2020-in-media-and-entertainment.pdf\nDisallow: /content/dam/www/us/en/offer/digital-trends-2020-in-media-and-entertainment-html/digital-trends-2020-in-media-and-entertainment-DE.pdf\nDisallow: /content/dam/www/us/en/offer/digital-trends-2020-in-media-and-entertainment-html/digital-trends-2020-in-media-and-entertainment-EN-US.pdf\nDisallow: /content/dam/www/us/en/offer/digital-trends-2020-in-media-and-entertainment-html/digital-trends-2020-in-media-and-entertainment-EN.pdf\nDisallow: /content/dam/www/us/en/offer/digital-trends-2020-in-media-and-entertainment-html/digital-trends-2020-in-media-and-entertainment-ES.pdf\nDisallow: /content/dam/www/us/en/offer/digital-trends-2020-in-media-and-entertainment-html/digital-trends-2020-in-media-and-entertainment-FR.pdf\nDisallow: /content/dam/www/us/en/offer/digital-trends-2020-in-media-and-entertainment-html/digital-trends-2020-in-media-and-entertainment-IT.pdf\nDisallow: /content/dam/www/us/en/offer/digital-trends-2020-in-media-and-entertainment-html/digital-trends-2020-in-media-and-entertainment-JA.pdf\nDisallow: /content/dam/www/us/en/offer/digital-trends-2020-in-media-and-entertainment-html/digital-trends-2020-in-media-and-entertainment-KR.pdf\nDisallow: /content/dam/www/us/en/offer/digital-trends-2020-in-media-and-entertainment-html/digital-trends-2020-in-media-and-entertainment-zh-Hans.pdf\nDisallow: /content/dam/www/us/en/offer/digital-trends-2020-in-media-and-entertainment-html/digital-trends-2020-in-media-and-entertainment-zh-Hant.pdf\nDisallow: /content/dam/www/us/en/offer/digital-trends-2020-in-media-and-entertainment-html/digital_trends_21_core_report_TC_FA.pdf\nDisallow: /content/dam/dx/us/en/resources/reports/digital-trends-2021-in-high-tech/2021_Digital_Trends_High_Tech_in_Focus.pdf\nDisallow: /content/dam/dx/us/en/resources/digital-trends-2021-in-telecommunications-media-and-entertainment/Digital_Trends_2021_Media_Entertainment_and_Telecoms_Report.pdf\nDisallow: /content/dam/dx/us/en/resources/digital-trends-2021-in-retail/2021_Digital_Trends_Retail_in_Focus.pdf\nDisallow: /content/dam/dx/us/en/resources/digital-trends-2021-in-manufacturing/2021_Digital_Trends_Manufacturing_in_Focus.pdf\nDisallow: /content/dam/www/us/en/offer/technology-trends-2021/eConsultancy_Digital_Trends_2021_IT_Report.pdf\nDisallow: /content/dam/www/us/en/offer/pdf/digital-trends-2020-in-financial-services.pdf\nDisallow: /content/dam/www/us/en/offer/pdf/digital-trends-2020-in-retail.pdf\nDisallow: /content/dam/www/us/en/offer/pdf/digital-trends-2020-in-media-and-entertainment.pdf\nDisallow: /content/dam/www/us/en/offer/pdf/digital-trends-2020-in-travel-and-hospitality.pdf\nDisallow: /content/dam/www/us/en/offer/pdf/digital-trends-2020-in-telecommunications.pdf\nDisallow: /content/dam/www/us/en/offer/pdf/digital-trends-2020-in-advertising.pdf\nDisallow: /content/dam/www/us/en/offer/pdf/digital-trends-2020-in-high-tech.pdf\nDisallow: /content/dam/www/us/en/offer/pdf/digital-trends-2020-in-IT.pdf\nDisallow: /content/dam/www/us/en/adi/2020/pdfs/*\nDisallow: /content/dam/www/us/en/offer/digital-trends-2021-core/digital-trends-2021-full-report-*.pdf\nDisallow: /*langmaster/\nDisallow: /*express/*.mp4$\nDisallow: /en-AE/learn/\nDisallow: /en-BE/learn/\nDisallow: /en-CY/learn/\nDisallow: /en-GR/learn/\nDisallow: /en-HK/learn/\nDisallow: /en-ID/learn/\nDisallow: /en-IL/learn/\nDisallow: /en-LU/learn/\nDisallow: /en-MT/learn/\nDisallow: /en-MY/learn/\nDisallow: /en-PH/learn/\nDisallow: /en-SA/learn/\nDisallow: /en-SG/learn/\nDisallow: /en-TH/learn/\nDisallow: /en-VN/learn/\nDisallow: /en-x-MenA/learn/\nDisallow: /en-ZA/learn/\nDisallow: /*?searchId=\nDisallow: /*?tasks=\nDisallow: /*?campaignid=\nDisallow: /*?skwcid=\nDisallow: /products/photoshop/photoshop-brushes.h*\nDisallow: /community/gallery\n\nUser-agent: Googlebot\nAllow: /?promoid=RTQCN3LX\nDisallow: /*promoid=\nDisallow: /*trackingid=\nDisallow: /*?searchId=\nDisallow: /*?tasks=\nDisallow: /*?campaignid=\nDisallow: /*?skwcid=\nDisallow: /products/photoshop/photoshop-brushes.h*\n\nUser-agent: AdsBot-Google\nAllow: /\n\nUser-agent: AdsBot-Google-Mobile\nAllow: /\n\nUser-agent: SearchmetricsBot\nDisallow: /\n\n# XML sitemaps\nSitemap: https://www.adobe.com/cc-product.index.xml\nSitemap: https://www.adobe.com/creativecloud/sitemap-index.xml\nSitemap: https://www.adobe.com/acrobat/sitemap-index.xml\nSitemap: https://www.adobe.com/home-sitemap.xml\nSitemap: https://www.adobe.com/plans-catalog-sitemap.xml\nSitemap: https://www.adobe.com/learn/sitemap_index.xml\n\n# CC Express \nSitemap: https://www.adobe.com/express/sitemap-index.xml\n\n#Naver\nSitemap: https://www.adobe.com/naver.index.xml\n\n#Education\nSitemap: https://www.adobe.com/edu/sitemap-index.xml\n","agents":{"GPTBot":true,"OAI-SearchBot":true,"ChatGPT-User":true,"ClaudeBot":true,"Claude-SearchBot":true,"PerplexityBot":true,"Google-Extended":true,"Applebot-Extended":true,"CCBot":true,"Bytespider":true,"Meta-ExternalAgent":true,"Amazonbot":true}},"sitemap":{"status":200,"content_type":"application/xml","bytes":8649,"sha256":"8f5710016d9dce1daf3d0d0cbe81f62c1154d28961484d0fb8c518a22a5d638d","final_host":"www.adobe.com","ms":57,"url":"https://www.adobe.com/cc-product.index.xml","declared":true,"xml":true},"files":{"llms_txt":{"status":200,"content_type":"text/plain","bytes":10499,"sha256":"811353cb603ff604ecd257b7e24643f1c7e157837015221a951d9d2db0c494d8","final_host":"www.adobe.com","ms":100,"path":"/llms.txt","html":false,"json":null,"present":true},"agents_md":{"status":404,"content_type":"text/html","bytes":889,"sha256":"4cea9068d315a6b931c26a44bd0234f614fdd0155777173b1c93d51729c5f744","final_host":"www.adobe.com","ms":116,"path":"/agents.md","html":true,"json":null,"present":false},"agent_skills":{"status":404,"content_type":"text/html","bytes":889,"sha256":"4cea9068d315a6b931c26a44bd0234f614fdd0155777173b1c93d51729c5f744","final_host":"www.adobe.com","ms":96,"path":"/.well-known/agent-skills/index.json","html":true,"json":null,"present":false},"mcp_card":{"status":404,"content_type":"text/html","bytes":889,"sha256":"4cea9068d315a6b931c26a44bd0234f614fdd0155777173b1c93d51729c5f744","final_host":"www.adobe.com","ms":87,"path":"/.well-known/mcp/server-card.json","html":true,"json":null,"present":false},"api_catalog":{"status":404,"content_type":"text/html","bytes":889,"sha256":"4cea9068d315a6b931c26a44bd0234f614fdd0155777173b1c93d51729c5f744","final_host":"www.adobe.com","ms":114,"path":"/.well-known/api-catalog","html":true,"json":null,"present":false},"a2a_card":{"status":404,"content_type":"text/html","bytes":889,"sha256":"4cea9068d315a6b931c26a44bd0234f614fdd0155777173b1c93d51729c5f744","final_host":"www.adobe.com","ms":134,"path":"/.well-known/agent-card.json","html":true,"json":null,"present":false}},"rings":{"v":1,"candidates":[],"mcp":[],"a2a":null,"webmcp":{"model_context":false,"bridge":false,"mcp_json":false,"link_rel":false,"any":false},"handshakes":0,"get_path":{"status":404,"content_type":"text/html","ms":86,"error":null},"get_host":{"status":530,"content_type":"text/plain","ms":5,"error":null}},"derived":{"reachable":true,"robots_served":true,"robots_sitemap":true,"content_signal":false,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":[],"blocks_any_ai":false,"files":{"llms_txt":true,"agents_md":false,"agent_skills":false,"mcp_card":false,"api_catalog":false,"a2a_card":false},"any_capability":false,"ai_refused":["gptbot","claudebot","oaisearch","perplexity"],"ai_thin":[],"rings":{"mcp_declared":false,"mcp_live":false,"mcp_auth":false,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":2,"fh":"www.adobe.com","fp_other_host":[],"fp":"dd0a412194ba14c2","fp_bits":{}},"derive_v":3,"frame":"top1k","shard":0},{"v":1,"kind":"crawlcheck-census-record","run":14,"domain":"pv-cdn.net","rank":66,"measured_at":"2026-10-15T05:27:03.729Z","home":{"browser":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"pv-cdn.net","ms":84,"identity":"Mobile browser","html":false},"gptbot":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"pv-cdn.net","ms":82,"identity":"GPTBot","html":false},"claudebot":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"pv-cdn.net","ms":1,"identity":"ClaudeBot","html":false},"oaisearch":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"pv-cdn.net","ms":1,"identity":"OAI-SearchBot","html":false},"perplexity":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"pv-cdn.net","ms":1,"identity":"PerplexityBot","html":false}},"robots":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"pv-cdn.net","ms":1,"html":false,"text":null,"agents":null},"sitemap":null,"files":null,"derived":{"reachable":false,"robots_served":false,"robots_sitemap":false,"content_signal":false,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":[],"blocks_any_ai":false,"files":{"llms_txt":false,"agents_md":false,"agent_skills":false,"mcp_card":false,"api_catalog":false,"a2a_card":false},"any_capability":false,"ai_refused":[],"ai_thin":[],"rings":{"mcp_declared":false,"mcp_live":false,"mcp_auth":false,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":2,"fh":"pv-cdn.net","fp_other_host":[]},"derive_v":3,"frame":"top1k","shard":1},{"v":1,"kind":"crawlcheck-census-record","run":14,"domain":"windowsupdate.com","rank":67,"measured_at":"2026-10-15T05:28:03.923Z","home":{"browser":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"windowsupdate.com","ms":15,"identity":"Mobile browser","html":false},"gptbot":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"windowsupdate.com","ms":2,"identity":"GPTBot","html":false},"claudebot":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"windowsupdate.com","ms":2,"identity":"ClaudeBot","html":false},"oaisearch":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"windowsupdate.com","ms":2,"identity":"OAI-SearchBot","html":false},"perplexity":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"windowsupdate.com","ms":2,"identity":"PerplexityBot","html":false}},"robots":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"windowsupdate.com","ms":2,"html":false,"text":null,"agents":null},"sitemap":null,"files":null,"derived":{"reachable":false,"robots_served":false,"robots_sitemap":false,"content_signal":false,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":[],"blocks_any_ai":false,"files":{"llms_txt":false,"agents_md":false,"agent_skills":false,"mcp_card":false,"api_catalog":false,"a2a_card":false},"any_capability":false,"ai_refused":[],"ai_thin":[],"rings":{"mcp_declared":false,"mcp_live":false,"mcp_auth":false,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":2,"fh":"windowsupdate.com","fp_other_host":[]},"derive_v":3,"frame":"top1k","shard":2},{"v":1,"kind":"crawlcheck-census-record","run":14,"domain":"spotify.com","rank":68,"measured_at":"2026-10-15T05:29:03.596Z","home":{"browser":{"status":200,"content_type":"text/html","bytes":309458,"sha256":"df79873f87657ecc73d2b3b35032cc71b7ca664da33e0ef3bb15023c3c4415fd","final_host":"open.spotify.com","ms":287,"identity":"Mobile browser","html":true,"fp":"0af8da10f40931d9"},"gptbot":{"status":200,"content_type":"text/html","bytes":169014,"sha256":"c68302e30dc812ff5ea5d0779629d3db97dce6e33a6a6ca2518f9426b0c992df","final_host":"open.spotify.com","ms":208,"identity":"GPTBot","html":true},"claudebot":{"status":200,"content_type":"text/html","bytes":169014,"sha256":"ba7194e09d5dfd5222bcad24b68b6084e423a3531bd006acc1204d93c22fc0d9","final_host":"open.spotify.com","ms":179,"identity":"ClaudeBot","html":true},"oaisearch":{"status":200,"content_type":"text/html","bytes":169014,"sha256":"88661d168904c0ef6da8b58fc67cfa09e9a3624800f3a423f2f209e465b64fb1","final_host":"open.spotify.com","ms":154,"identity":"OAI-SearchBot","html":true},"perplexity":{"status":200,"content_type":"text/html","bytes":169014,"sha256":"7c91e3a1d0b09d8c1c9e0217b9588d1afbabc5ed37b458e81da917dcef5cfe1d","final_host":"open.spotify.com","ms":196,"identity":"PerplexityBot","html":true}},"robots":{"status":200,"content_type":"text/plain","bytes":635,"sha256":"b68c89548f813d5c465c13051353b9452dba066f14146cafabb46dafd239b738","final_host":"www.spotify.com","ms":287,"html":false,"text":"User-agent: *\nDisallow: /*/about-us/contact/contact-spotify-password/\nDisallow: /*/about-us/contact/contact-spotify-account/\nDisallow: /*/get-spotify/*\nDisallow: /*/xhr/*\nDisallow: /*/external/*\nDisallow: /*/legal/*?ets=\nDisallow: /*/legal/advertiser-terms-and-conditions/\nDisallow: /*/legal/gdpr-article-15-information/\nDisallow: /*/legal/spotify-controller-data-processing-terms/\nDisallow: /*/legal/podcast-api-terms/\nDisallow: /*/legal/distribution-api-terms/\nDisallow: /*/account/cls/*\nDisallow: /*/starbuckspartners\nDisallow: /starbuckspartners\nDisallow: /ppt/*?\nDisallow: /partner/*?\nSitemap: https://www.spotify.com/sitemap.xml\n","agents":{"GPTBot":true,"OAI-SearchBot":true,"ChatGPT-User":true,"ClaudeBot":true,"Claude-SearchBot":true,"PerplexityBot":true,"Google-Extended":true,"Applebot-Extended":true,"CCBot":true,"Bytespider":true,"Meta-ExternalAgent":true,"Amazonbot":true}},"sitemap":{"status":200,"content_type":"application/xml","bytes":1114,"sha256":"2ae8c8fbf612a2977db25fba637d68380b5855470af21c83312aed14e1339258","final_host":"www.spotify.com","ms":44,"url":"https://www.spotify.com/sitemap.xml","declared":true,"xml":true},"files":{"llms_txt":{"status":200,"content_type":"text/html","bytes":46117,"sha256":"e104055da1c66cb19dc75bf4b6c4ba0760c1243afd48103c7adec939ea32daea","final_host":"www.spotify.com","ms":229,"path":"/llms.txt","html":true,"json":null,"present":false},"agents_md":{"status":200,"content_type":"text/html","bytes":46117,"sha256":"e104055da1c66cb19dc75bf4b6c4ba0760c1243afd48103c7adec939ea32daea","final_host":"www.spotify.com","ms":264,"path":"/agents.md","html":true,"json":null,"present":false},"agent_skills":{"status":404,"content_type":"text/html","bytes":3310,"sha256":"c54bef043bab693894ccf620d4200ee671eb063d854d6ea3923d268af674d347","final_host":"www.spotify.com","ms":111,"path":"/.well-known/agent-skills/index.json","html":true,"json":null,"present":false},"mcp_card":{"status":404,"content_type":"text/html","bytes":3310,"sha256":"c54bef043bab693894ccf620d4200ee671eb063d854d6ea3923d268af674d347","final_host":"www.spotify.com","ms":76,"path":"/.well-known/mcp/server-card.json","html":true,"json":null,"present":false},"api_catalog":{"status":404,"content_type":"text/html","bytes":3310,"sha256":"c54bef043bab693894ccf620d4200ee671eb063d854d6ea3923d268af674d347","final_host":"www.spotify.com","ms":108,"path":"/.well-known/api-catalog","html":true,"json":null,"present":false},"a2a_card":{"status":404,"content_type":"text/html","bytes":3310,"sha256":"c54bef043bab693894ccf620d4200ee671eb063d854d6ea3923d268af674d347","final_host":"www.spotify.com","ms":76,"path":"/.well-known/agent-card.json","html":true,"json":null,"present":false}},"rings":{"v":1,"candidates":[],"mcp":[],"a2a":null,"webmcp":{"model_context":false,"bridge":false,"mcp_json":false,"link_rel":false,"any":false},"handshakes":0,"get_path":{"status":200,"content_type":"text/html","ms":158,"error":null},"get_host":{"status":530,"content_type":"text/plain","ms":67,"error":null}},"derived":{"reachable":true,"robots_served":true,"robots_sitemap":true,"content_signal":false,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":[],"blocks_any_ai":false,"files":{"llms_txt":false,"agents_md":false,"agent_skills":false,"mcp_card":false,"api_catalog":false,"a2a_card":false},"any_capability":false,"ai_refused":[],"ai_thin":["gptbot","claudebot","oaisearch","perplexity"],"rings":{"mcp_declared":false,"mcp_live":false,"mcp_auth":false,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":2,"fh":"open.spotify.com","fp_other_host":[],"fp":"0af8da10f40931d9","fp_bits":{}},"derive_v":3,"frame":"top1k","shard":3},{"v":1,"kind":"crawlcheck-census-record","run":14,"domain":"wa.me","rank":69,"measured_at":"2026-10-15T05:30:03.842Z","home":{"browser":{"status":400,"content_type":"text/html","bytes":6278,"sha256":"9a8864fd8e8a71007dd2252c62ab4743e082c8220d62f68c90f5422cded0285f","final_host":"api.whatsapp.com","ms":781,"identity":"Mobile browser","html":true},"gptbot":{"status":200,"content_type":"text/html","bytes":191485,"sha256":"9cdff9759d49dbae855d639dbe8d6608f1fa7a2efa469ab6552a79fc5eb87e56","final_host":"api.whatsapp.com","ms":981,"identity":"GPTBot","html":true,"fp":"61868d36b9075b36"},"claudebot":{"status":200,"content_type":"text/html","bytes":191430,"sha256":"f9a5344abd15ed6f8f4221c5cbbb7a02d73197ec12933572276cd39358b4ec60","final_host":"api.whatsapp.com","ms":788,"identity":"ClaudeBot","html":true,"fp":"61868d36b9075b36"},"oaisearch":{"status":200,"content_type":"text/html","bytes":191484,"sha256":"e0e961b2e73ee13c7da6e2b3aacbba8e47ff118e766734f09aca354e33c534e9","final_host":"api.whatsapp.com","ms":770,"identity":"OAI-SearchBot","html":true,"fp":"61868d36b9075b36"},"perplexity":{"status":200,"content_type":"text/html","bytes":201562,"sha256":"44e6f80b9038030a707fdb8d05b90de5128df35bd5338aa63e5e754443caedec","final_host":"api.whatsapp.com","ms":779,"identity":"PerplexityBot","html":true,"fp":"65068126b907533e"}},"robots":{"status":200,"content_type":"text/plain","bytes":1525,"sha256":"6ceebd7d44206c973321a46e46b94102a8edcc37eea87394b2dd94ffe713df8d","final_host":"wa.me","ms":360,"html":false,"text":"# Notice: Collection of data on WhatsApp through automated means is prohibited\n# See: https://www.whatsapp.com/legal/terms-of-service\nuser-agent: *\ndisallow: /usephone/\ndisallow: /message/\ndisallow: /qr/\ndisallow: /pay/\ndisallow: /stickerpack/\ndisallow: /biztools/\ndisallow: /settings/\ndisallow: /disappearing_messages/\ndisallow: /download/\ndisallow: /archive_settings/\ndisallow: /tos/\ndisallow: /support/\ndisallow: /bizsearch/\ndisallow: /directory/\ndisallow: /guia/\ndisallow: /profile/\ndisallow: /status/\ndisallow: /premium/\ndisallow: /community/\ndisallow: /advertise/\ndisallow: /biztab/\ndisallow: /biz-edit-profile/\ndisallow: /biz-edit-description/\ndisallow: /biz-edit-catalog/\ndisallow: /biz-location/\ndisallow: /biz-hours/\ndisallow: /biz-linked-accounts/\ndisallow: /privacy/\ndisallow: /biz-website/\ndisallow: /proxy/\ndisallow: /biz-price-tier/\ndisallow: /chat-transfer/\ndisallow: /calling/\ndisallow: /ais/\ndisallow: /marketing-message/\ndisallow: /invite/\ndisallow: /contacts/\ndisallow: /edit-profile-picture/\ndisallow: /wabai/\ndisallow: /notifications/\ndisallow: /favorites/\ndisallow: /meta_verified/\ndisallow: /biz-agents-onboarding/\ndisallow: /subscription_hub/\ndisallow: /share-whatsapp-web/\ndisallow: /biz-ai-learning-summary/\ndisallow: /waffle/\ndisallow: /family/\ndisallow: /daily-ads-summary/\ndisallow: /tools/\ndisallow: /join_nova_waitlist/\ndisallow: /man/\ndisallow: /hatch/\ndisallow: /subscriptions/\ndisallow: /biz-broadcast-home/\ndisallow: /business-broadcast/\nsitemap: https://wa.me/smp/wa_smb_custom_url.xml.gz","agents":{"GPTBot":true,"OAI-SearchBot":true,"ChatGPT-User":true,"ClaudeBot":true,"Claude-SearchBot":true,"PerplexityBot":true,"Google-Extended":true,"Applebot-Extended":true,"CCBot":true,"Bytespider":true,"Meta-ExternalAgent":true,"Amazonbot":true}},"sitemap":null,"files":null,"derived":{"reachable":false,"robots_served":false,"robots_sitemap":false,"content_signal":false,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":[],"blocks_any_ai":false,"files":{"llms_txt":false,"agents_md":false,"agent_skills":false,"mcp_card":false,"api_catalog":false,"a2a_card":false},"any_capability":false,"ai_refused":[],"ai_thin":[],"rings":{"mcp_declared":false,"mcp_live":false,"mcp_auth":false,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":2,"fh":"api.whatsapp.com","fp_other_host":[]},"derive_v":3,"frame":"top1k","shard":4},{"v":1,"kind":"crawlcheck-census-record","run":14,"domain":"myfritz.net","rank":70,"measured_at":"2026-10-15T05:31:08.958Z","home":{"browser":{"status":200,"content_type":"text/html","bytes":6885,"sha256":"a4fee43f758a0dc58b5411d8385bc0f83a43869593f3250559ed28bb31872284","final_host":"www.myfritz.net","ms":330,"identity":"Mobile browser","html":true},"gptbot":{"status":200,"content_type":"text/html","bytes":6885,"sha256":"a4fee43f758a0dc58b5411d8385bc0f83a43869593f3250559ed28bb31872284","final_host":"www.myfritz.net","ms":329,"identity":"GPTBot","html":true},"claudebot":{"status":200,"content_type":"text/html","bytes":6885,"sha256":"a4fee43f758a0dc58b5411d8385bc0f83a43869593f3250559ed28bb31872284","final_host":"www.myfritz.net","ms":332,"identity":"ClaudeBot","html":true},"oaisearch":{"status":200,"content_type":"text/html","bytes":6885,"sha256":"a4fee43f758a0dc58b5411d8385bc0f83a43869593f3250559ed28bb31872284","final_host":"www.myfritz.net","ms":329,"identity":"OAI-SearchBot","html":true},"perplexity":{"status":200,"content_type":"text/html","bytes":6885,"sha256":"a4fee43f758a0dc58b5411d8385bc0f83a43869593f3250559ed28bb31872284","final_host":"www.myfritz.net","ms":289,"identity":"PerplexityBot","html":true}},"robots":{"status":200,"content_type":"text/plain","bytes":269,"sha256":"0267852a7136ff65cf570bc69e024a5198e7d31ef564f90165cd5333250a62ac","final_host":"myfritz.net","ms":118,"html":false,"text":"User-agent: Yandex \nDisallow: /\n\nUser-agent: *\nDisallow: /\nAllow: /$\nAllow: /devices$\nAllow: /devices/$\nAllow: /devices/index.html$\nAllow: /sitemap.xml\nAllow: /legal-notice\nAllow: /terms\nAllow: /about\nHost: www.myfritz.net\n\nSitemap: https://www.myfritz.net/sitemap.xml\n","agents":{"GPTBot":true,"OAI-SearchBot":true,"ChatGPT-User":true,"ClaudeBot":true,"Claude-SearchBot":true,"PerplexityBot":true,"Google-Extended":true,"Applebot-Extended":true,"CCBot":true,"Bytespider":true,"Meta-ExternalAgent":true,"Amazonbot":true}},"sitemap":{"status":200,"content_type":"text/xml","bytes":2903,"sha256":"b83c75a3e8aa820ed2f4297239776d397858323c41e1edc4fcb339d16e973040","final_host":"www.myfritz.net","ms":19,"url":"https://www.myfritz.net/sitemap.xml","declared":true,"xml":true},"files":{"llms_txt":{"status":200,"content_type":"text/html","bytes":6885,"sha256":"a4fee43f758a0dc58b5411d8385bc0f83a43869593f3250559ed28bb31872284","final_host":"www.myfritz.net","ms":41,"path":"/llms.txt","html":true,"json":null,"present":false},"agents_md":{"status":200,"content_type":"text/html","bytes":6885,"sha256":"a4fee43f758a0dc58b5411d8385bc0f83a43869593f3250559ed28bb31872284","final_host":"www.myfritz.net","ms":45,"path":"/agents.md","html":true,"json":null,"present":false},"agent_skills":{"status":200,"content_type":"text/html","bytes":6885,"sha256":"a4fee43f758a0dc58b5411d8385bc0f83a43869593f3250559ed28bb31872284","final_host":"www.myfritz.net","ms":40,"path":"/.well-known/agent-skills/index.json","html":true,"json":null,"present":false},"mcp_card":{"status":200,"content_type":"text/html","bytes":6885,"sha256":"a4fee43f758a0dc58b5411d8385bc0f83a43869593f3250559ed28bb31872284","final_host":"www.myfritz.net","ms":51,"path":"/.well-known/mcp/server-card.json","html":true,"json":null,"present":false},"api_catalog":{"status":200,"content_type":"text/html","bytes":6885,"sha256":"a4fee43f758a0dc58b5411d8385bc0f83a43869593f3250559ed28bb31872284","final_host":"www.myfritz.net","ms":49,"path":"/.well-known/api-catalog","html":true,"json":null,"present":false},"a2a_card":{"status":200,"content_type":"text/html","bytes":6885,"sha256":"a4fee43f758a0dc58b5411d8385bc0f83a43869593f3250559ed28bb31872284","final_host":"www.myfritz.net","ms":41,"path":"/.well-known/agent-card.json","html":true,"json":null,"present":false}},"rings":{"v":1,"candidates":[],"mcp":[],"a2a":null,"webmcp":{"model_context":false,"bridge":false,"mcp_json":false,"link_rel":false,"any":false},"handshakes":0,"get_path":{"status":200,"content_type":"text/html","ms":34,"error":null},"get_host":{"status":530,"content_type":"text/plain","ms":41,"error":null}},"derived":{"reachable":true,"robots_served":true,"robots_sitemap":true,"content_signal":false,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":[],"blocks_any_ai":false,"files":{"llms_txt":false,"agents_md":false,"agent_skills":false,"mcp_card":false,"api_catalog":false,"a2a_card":false},"any_capability":false,"ai_refused":[],"ai_thin":[],"rings":{"mcp_declared":false,"mcp_live":false,"mcp_auth":false,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":2,"fh":"www.myfritz.net","fp_other_host":[]},"derive_v":3,"frame":"top1k","shard":5},{"v":1,"kind":"crawlcheck-census-record","run":14,"domain":"msn.com","rank":71,"measured_at":"2026-10-15T05:33:03.639Z","home":{"browser":{"status":200,"content_type":"text/html","bytes":293917,"sha256":"b3978305ebb47f9fb6ca4b1fdd6af6f08e265b62d9a511a39d481b87516d10fa","final_host":"www.msn.com","ms":432,"identity":"Mobile browser","html":true},"gptbot":{"status":200,"content_type":"text/html","bytes":48094,"sha256":"f27a081b4da4e21d97d0378dd5cdf68a4f3505346403ed49590627ac03fe4fb6","final_host":"www.msn.com","ms":284,"identity":"GPTBot","html":true},"claudebot":{"status":200,"content_type":"text/html","bytes":48176,"sha256":"da32da38d2771dd8dc0f5944ee819e5ee6654ac2d89cf7ac8e802377c2b138ca","final_host":"www.msn.com","ms":287,"identity":"ClaudeBot","html":true},"oaisearch":{"status":200,"content_type":"text/html","bytes":48109,"sha256":"34f5b89a6dfac6555b737d97b3d576e3968938c2115c884e6bbbc390fd60406f","final_host":"www.msn.com","ms":279,"identity":"OAI-SearchBot","html":true},"perplexity":{"status":200,"content_type":"text/html","bytes":47581,"sha256":"8448a9ae4e7828ec3905c7df8e43499756153baa850819065a3e505cffb82039","final_host":"www.msn.com","ms":275,"identity":"PerplexityBot","html":true}},"robots":{"status":200,"content_type":"text/plain","bytes":5245,"sha256":"aefbffa0a0a083e30637972046b42afc1569f7813c5687739a5ecdee0a6be4dd","final_host":"www.msn.com","ms":198,"html":false,"text":"User-agent: *\r\nDisallow: /*/health/search/filter\r\nDisallow: /spartan\r\nDisallow: /pt-ao\r\nDisallow: /*preview=\r\nDisallow: /*/autos/marketplace/product/\r\nDisallow: /*/cars/marketplace/product/\r\nDisallow: /*?item=\r\nDisallow: /*&item=\r\nDisallow: /codeCoverage\r\nDisallow: /vpBaselines\r\nDisallow: /msnBaselines\r\nDisallow: /*/studio\r\nDisallow: /*/indexnowdelete/\r\nAllow: /*/shopping/*/ar-\r\nAllow: /*/shopping/*/ss-\r\nAllow: /*/shopping/*/vi-\r\nDisallow: /*/shopping/\r\n\r\nUser-agent: AdsBot-Google\r\nAllow: /\r\nDisallow: /*/health/search/filter\r\n\r\nUser-agent: Verity\r\nCrawl-delay: 3\r\nDisallow: /*/health/search/filter\r\nDisallow: /spartan\r\nDisallow: /pt-ao\r\nDisallow: /*preview=\r\nDisallow: /*/autos/marketplace/product/\r\nDisallow: /*/cars/marketplace/product/\r\nDisallow: /*?item=\r\nDisallow: /*&item=\r\nDisallow: /codeCoverage\r\nDisallow: /vpBaselines\r\nDisallow: /msnBaselines\r\nDisallow: /*/studio\r\nDisallow: /*/indexnowdelete/\r\nAllow: /*/shopping/*/ar-\r\nAllow: /*/shopping/*/ss-\r\nAllow: /*/shopping/*/vi-\r\nDisallow: /*/shopping/\r\n\r\nUser-agent: ias_crawler\r\nCrawl-delay: 0.1\r\nDisallow: /*/health/search/filter\r\nDisallow: /spartan\r\nDisallow: /pt-ao\r\nDisallow: /*preview=\r\nDisallow: /*/autos/marketplace/product/\r\nDisallow: /*/cars/marketplace/product/\r\nDisallow: /*?item=\r\nDisallow: /*&item=\r\nDisallow: /codeCoverage\r\nDisallow: /vpBaselines\r\nDisallow: /msnBaselines\r\nDisallow: /*/studio\r\nDisallow: /*/indexnowdelete/\r\nAllow: /*/shopping/*/ar-\r\nAllow: /*/shopping/*/ss-\r\nAllow: /*/shopping/*/vi-\r\nDisallow: /*/shopping/\r\n\r\nUser-agent: Mediapartners-Google\r\nCrawl-delay: 0.5\r\nDisallow: /*/health/search/filter\r\nDisallow: /spartan\r\nDisallow: /pt-ao\r\nDisallow: /*preview=\r\nDisallow: /*/autos/marketplace/product/\r\nDisallow: /*/cars/marketplace/product/\r\nDisallow: /*?item=\r\nDisallow: /*&item=\r\nDisallow: /codeCoverage\r\nDisallow: /vpBaselines\r\nDisallow: /msnBaselines\r\nDisallow: /*/studio\r\nDisallow: /*/indexnowdelete/\r\nAllow: /*/shopping/*/ar-\r\nAllow: /*/shopping/*/ss-\r\nAllow: /*/shopping/*/vi-\r\nDisallow: /*/shopping/\r\n\r\nUser-agent: CriteoBot\r\nCrawl-delay: 0.5\r\nDisallow: /*/health/search/filter\r\nDisallow: /spartan\r\nDisallow: /pt-ao\r\nDisallow: /*preview=\r\nDisallow: /*/autos/marketplace/product/\r\nDisallow: /*/cars/marketplace/product/\r\nDisallow: /*?item=\r\nDisallow: /*&item=\r\nDisallow: /codeCoverage\r\nDisallow: /vpBaselines\r\nDisallow: /msnBaselines\r\nDisallow: /*/studio\r\nDisallow: /*/indexnowdelete/\r\nAllow: /*/shopping/*/ar-\r\nAllow: /*/shopping/*/ss-\r\nAllow: /*/shopping/*/vi-\r\nDisallow: /*/shopping/\r\n\r\nUser-agent: yandex.com/bot\r\nCrawl-delay: 1\r\nDisallow: /*/weather/\r\nDisallow: /*/clima/\r\nDisallow: /*/cuaca/\r\nDisallow: /*/eltiempo/\r\nDisallow: /*/el-tiempo/\r\nDisallow: /*/havadurumu/\r\nDisallow: /*/idojaras/\r\nDisallow: /*/meteo/\r\nDisallow: /*/meteorologia/\r\nDisallow: /*/pocasi/\r\nDisallow: /*/pogoda/\r\nDisallow: /*/saa/\r\nDisallow: /*/vader/\r\nDisallow: /*/vejr/\r\nDisallow: /*/weer/\r\nDisallow: /*/wetter/\r\nAllow: /*/shopping/*/ar-\r\nAllow: /*/shopping/*/ss-\r\nAllow: /*/shopping/*/vi-\r\nDisallow: /*/shopping/\r\n\r\nUser-agent: admantx-ussy04/3.2\r\nAllow: /\r\n\r\nUser-agent: TTD-Content\r\nAllow: /\r\n\r\nUser-agent: AmazonAdBot\r\nAllow: /\r\n\r\nUser-agent: AI2Bot\r\nUser-agent: Ai2Bot-Dolma\r\nUser-agent: Amazonbot\r\nUser-agent: anthropic-ai\r\nUser-agent: Applebot\r\nUser-agent: Applebot-Extended\r\nUser-agent: Bedrockbot\r\nUser-agent: Bytespider\r\nUser-agent: CCBot\r\nUser-agent: ChatGLM\r\nUser-agent: ChatGPT-User\r\nUser-agent: ClaudeBot\r\nUser-agent: Claude-User\r\nUser-agent: Claude-SearchBot \r\nUser-agent: Claude-Web\r\nUser-agent: cohere-ai\r\nUser-agent: cohere-training-data-crawler\r\nUser-agent: Cotoyogi\r\nUser-agent: DeepSeekBot\r\nUser-agent: Diffbot\r\nUser-agent: DuckAssistBot\r\nUser-agent: FacebookBot\r\nUser-agent: FriendlyCrawler\r\nUser-agent: Gensparkbot \r\nUser-agent: Google-CloudVertexBot\r\nUser-agent: Google-Extended\r\nUser-agent: GoogleOther\r\nUser-agent: GoogleOther-Image\r\nUser-agent: GoogleOther-Video\r\nUser-agent: GPTBot\r\nUser-agent: Grok\r\nUser-agent: iaskspider/2.0\r\nUser-agent: ICC-Crawler\r\nUser-agent: ImagesiftBot\r\nUser-agent: Img2dataset\r\nUser-agent: ISSCyberRiskCrawler\r\nUser-agent: Kangaroo Bot\r\nUser-agent: KunatoCrawler\r\nUser-agent: magpie-crawler\r\nUser-agent: Meltwater\r\nUser-agent: Meta-ExternalAgent\r\nUser-agent: Meta-ExternalFetcher\r\nUser-agent: MistralAI\r\nUser-agent: MistralAI-User\r\nUser-agent: Momentumbot\r\nUser-agent: OAI-SearchBot\r\nUser-agent: omgili\r\nUser-agent: Omgilibot\r\nUser-agent: Other\r\nUser-agent: PanguBot\r\nUser-agent: Perplexity-ai\r\nUser-agent: PerplexityBot\r\nUser-agent: Perplexity-User\r\nUser-agent: QualifiedBot\r\nUser-agent: Scrapy\r\nUser-agent: Seekr\r\nUser-agent: Sidetrade indexer bot\r\nUser-agent: TikTokSpider\r\nUser-agent: Timpibot\r\nUser-agent: Twitterbot\r\nUser-agent: VelenPublicWebCrawler\r\nUser-agent: WARDBot\r\nUser-agent: Webzio-Extended\r\nUser-agent: WRTNBot\r\nUser-agent: YouBot\r\nDisallow: /\r\n\r\nSitemap: https://www.msn.com/sitemaps/casualgames/sitemap-index.xml\r\nSitemap: https://www.msn.com/sitemaps/consumption/index.xml\r\nSitemap: https://www.msn.com/sitemaps/health/health-sitemap-index.xml\r\nSitemap: https://www.msn.com/staticsb/statics/latest/0/casualgames/sitemaps/sitemap-index.xml\r\nSitemap: https://www.msn.com/staticsb/statics/latest/0/finance/sitemaps/sitemap-index.xml\r\nSitemap: https://www.msn.com/sitemaps/weather/sitemap-index.xml","agents":{"GPTBot":false,"OAI-SearchBot":false,"ChatGPT-User":false,"ClaudeBot":false,"Claude-SearchBot":false,"PerplexityBot":false,"Google-Extended":false,"Applebot-Extended":false,"CCBot":false,"Bytespider":false,"Meta-ExternalAgent":false,"Amazonbot":false}},"sitemap":{"status":200,"content_type":"text/xml","bytes":5402,"sha256":"6357cf8b4653e26a07eca3573c30413dbf7e64d2e58f3b9ee45505f3f96607c8","final_host":"www.msn.com","ms":47,"url":"https://www.msn.com/sitemaps/casualgames/sitemap-index.xml","declared":true,"xml":true},"files":{"llms_txt":{"status":404,"content_type":"text/html","bytes":19143,"sha256":"10fcf682b17971154f37ba43feed59f3f054c12d46e8d9dab193696175e960ff","final_host":"www.msn.com","ms":82,"path":"/llms.txt","html":true,"json":null,"present":false},"agents_md":{"status":404,"content_type":"text/html","bytes":19143,"sha256":"10fcf682b17971154f37ba43feed59f3f054c12d46e8d9dab193696175e960ff","final_host":"www.msn.com","ms":103,"path":"/agents.md","html":true,"json":null,"present":false},"agent_skills":{"status":404,"content_type":"text/html","bytes":19143,"sha256":"10fcf682b17971154f37ba43feed59f3f054c12d46e8d9dab193696175e960ff","final_host":"www.msn.com","ms":86,"path":"/.well-known/agent-skills/index.json","html":true,"json":null,"present":false},"mcp_card":{"status":404,"content_type":"text/html","bytes":19143,"sha256":"10fcf682b17971154f37ba43feed59f3f054c12d46e8d9dab193696175e960ff","final_host":"www.msn.com","ms":90,"path":"/.well-known/mcp/server-card.json","html":true,"json":null,"present":false},"api_catalog":{"status":404,"content_type":"text/html","bytes":19143,"sha256":"10fcf682b17971154f37ba43feed59f3f054c12d46e8d9dab193696175e960ff","final_host":"www.msn.com","ms":87,"path":"/.well-known/api-catalog","html":true,"json":null,"present":false},"a2a_card":{"status":404,"content_type":"text/html","bytes":19143,"sha256":"10fcf682b17971154f37ba43feed59f3f054c12d46e8d9dab193696175e960ff","final_host":"www.msn.com","ms":90,"path":"/.well-known/agent-card.json","html":true,"json":null,"present":false}},"rings":{"v":1,"candidates":[],"mcp":[],"a2a":null,"webmcp":{"model_context":false,"bridge":false,"mcp_json":false,"link_rel":false,"any":false},"handshakes":0,"get_path":{"status":404,"content_type":"text/html","ms":89,"error":null},"get_host":{"status":404,"content_type":"text/html","ms":257,"error":null}},"derived":{"reachable":true,"robots_served":true,"robots_sitemap":true,"content_signal":false,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":["GPTBot","OAI-SearchBot","ChatGPT-User","ClaudeBot","Claude-SearchBot","PerplexityBot","Google-Extended","Applebot-Extended","CCBot","Bytespider","Meta-ExternalAgent","Amazonbot"],"blocks_any_ai":true,"files":{"llms_txt":false,"agents_md":false,"agent_skills":false,"mcp_card":false,"api_catalog":false,"a2a_card":false},"any_capability":false,"ai_refused":[],"ai_thin":["gptbot","claudebot","oaisearch","perplexity"],"rings":{"mcp_declared":false,"mcp_live":false,"mcp_auth":false,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":2,"fh":"www.msn.com","fp_other_host":[]},"derive_v":3,"frame":"top1k","shard":6},{"v":1,"kind":"crawlcheck-census-record","run":14,"domain":"zoom.us","rank":72,"measured_at":"2026-10-15T05:34:03.814Z","home":{"browser":{"status":200,"content_type":"text/html","bytes":318999,"sha256":"8674f566a8c7f6dab7959d6ac429327ae1da77e31f312fa84d1b0328188cbf9f","final_host":"www.zoom.com","ms":64,"identity":"Mobile browser","html":true,"fp":"a0beb85019e3668c"},"gptbot":{"status":200,"content_type":"text/html","bytes":318999,"sha256":"fe68a4c7b5ea7b76810ce96cb6aba7285f20db74ed9ae0b5d66d83a3b7327c55","final_host":"www.zoom.com","ms":42,"identity":"GPTBot","html":true,"fp":"a0beb85019e3668c"},"claudebot":{"status":200,"content_type":"text/html","bytes":318999,"sha256":"c136e8f6c990d46a516ee44c49ef5ab5d017a8f1699ff21189f1fcd323abf918","final_host":"www.zoom.com","ms":60,"identity":"ClaudeBot","html":true,"fp":"a0beb85019e3668c"},"oaisearch":{"status":200,"content_type":"text/html","bytes":318999,"sha256":"10fd8f98307cc39c8c6710641254b7cd70f45b667ba45cad857daa9ec2e04482","final_host":"www.zoom.com","ms":66,"identity":"OAI-SearchBot","html":true,"fp":"a0beb85019e3668c"},"perplexity":{"status":200,"content_type":"text/html","bytes":318999,"sha256":"6d9f9e2db4c1bd5b1c8787b6243b8b844685fa1b1736ee3b9414a78cd91bd63e","final_host":"www.zoom.com","ms":41,"identity":"PerplexityBot","html":true,"fp":"a0beb85019e3668c"}},"robots":{"status":200,"content_type":"text/plain","bytes":2005,"sha256":"b3ed9f53f38c480b046b4d5a648aa7a879c6720957301506dc282b32fb2f5593","final_host":"zoom.us","ms":191,"html":false,"text":"User-Agent: *\nDisallow: /download/*/Zoom_launcher.exe\nDisallow: /download/*/zoomusLauncher.zip\nDisallow: /docs/image/new/brand/outdated/\nDisallow: /docs/doc/2-Page-All-Products.pdf\nDisallow: /docs/doc/zoom-apps-security-privacy-faq.pdf\nDisallow: /docs/doc/zoom-apps-data-management.pdf\nDisallow: /wc/*\nDisallow: /web/*\nDisallow: /share*\nDisallow: /support*\nDisallow: /j/*\nDisallow: /z/*\nDisallow: /s/*\nDisallow: /u/*\nDisallow: /w/*\nDisallow: /e/*\nDisallow: /account*\nDisallow: /start*\nDisallow: /postattendee*\nDisallow: /rec/*\nDisallow: /register*\nDisallow: /webinar/register/*\nDisallow: /webinar/registerevent/*\nDisallow: /meeting/register/*\nDisallow: /*oauth_signin\nDisallow: /my/*\nDisallow: /download2*\nDisallow: /*?\n\nUser-agent: Googlebot\nAllow: /join\n\nUser-agent: LinkedInBot\nAllow: /webinar/register/*\nAllow: /webinar/registerevent/*\nAllow: /meeting/register/*\n\nUser-agent: facebookexternalhit\nAllow: /webinar/register/*\nAllow: /webinar/registerevent/*\nAllow: /meeting/register/*\n\nUser-agent: Twitterbot\nAllow: /webinar/register/*\nAllow: /webinar/registerevent/*\nAllow: /meeting/register/*\n\nUser-agent: PinterestBot\nAllow: /webinar/register/*\nAllow: /webinar/registerevent/*\nAllow: /meeting/register/*\n\nUser-agent: InstagramBot\nAllow: /webinar/register/*\nAllow: /webinar/registerevent/*\nAllow: /meeting/register/*\n\nUser-agent: Slackbot\nAllow: /webinar/register/*\nAllow: /webinar/registerevent/*\nAllow: /meeting/register/*\n\nUser-agent: WhatsApp\nAllow: /webinar/register/*\nAllow: /webinar/registerevent/*\nAllow: /meeting/register/*\n\nUser-agent: redditbot\nAllow: /webinar/register/*\nAllow: /webinar/registerevent/*\nAllow: /meeting/register/*\n\nUser-agent: Tumblr\nAllow: /webinar/register/*\nAllow: /webinar/registerevent/*\nAllow: /meeting/register/*\n\nUser-agent: TikTokBot\nAllow: /webinar/register/*\nAllow: /webinar/registerevent/*\nAllow: /meeting/register/*\n\nUser-agent: Snapchat\nAllow: /webinar/register/*\nAllow: /webinar/registerevent/*\nAllow: /meeting/register/*\n\nSitemap: https://zoom.us/sitemap.xml","agents":{"GPTBot":true,"OAI-SearchBot":true,"ChatGPT-User":true,"ClaudeBot":true,"Claude-SearchBot":true,"PerplexityBot":true,"Google-Extended":true,"Applebot-Extended":true,"CCBot":true,"Bytespider":true,"Meta-ExternalAgent":true,"Amazonbot":true}},"sitemap":{"status":200,"content_type":"text/xml","bytes":432921,"sha256":"fc46be69d0c7b7529bb15d13a177daf4e337d6ec8f0b3e0875e0706c138489b8","final_host":"zoom.us","ms":223,"url":"https://zoom.us/sitemap.xml","declared":true,"xml":true},"files":{"llms_txt":{"status":404,"content_type":"text/html","bytes":515029,"sha256":"bc62f84081f8a297743212131f3751318c644f7138e7cfe2005f4f8cb542d373","final_host":"zoom.us","ms":258,"path":"/llms.txt","html":true,"json":null,"present":false},"agents_md":{"status":404,"content_type":"text/html","bytes":515029,"sha256":"60c84e60d712ed8d4ec187199f893b483d4d07a08e59dc79ff9a70eccdcc3620","final_host":"zoom.us","ms":242,"path":"/agents.md","html":true,"json":null,"present":false},"agent_skills":{"status":200,"content_type":"application/json","bytes":27018,"sha256":"4b67b874d5fec34fcf20a6b3d8b52111a5dbc6b167c06c4718d7503099e8c829","final_host":"zoom.us","ms":303,"path":"/.well-known/agent-skills/index.json","html":false,"json":true,"present":true},"mcp_card":{"status":200,"content_type":"application/json","bytes":3459,"sha256":"a5911c1e3bb13ba1c67b907de118dec106b468f01d525a64d8d08bb9d97d6ea1","final_host":"zoom.us","ms":318,"path":"/.well-known/mcp/server-card.json","html":false,"json":true,"present":true},"api_catalog":{"status":200,"content_type":"binary/octet-stream","bytes":33427,"sha256":"2b22c7624b5ef187e6e377f8e1214ccba01d192d3c2c4a30f20172fbb1dcec5a","final_host":"zoom.us","ms":361,"path":"/.well-known/api-catalog","html":false,"json":true,"present":true},"a2a_card":{"status":404,"content_type":"text/html","bytes":515029,"sha256":"ee5ddd445ab6b91671a2ab2b1069f7f7739cac83aff7a4607416d083fee33d66","final_host":"zoom.us","ms":295,"path":"/.well-known/agent-card.json","html":true,"json":null,"present":false}},"rings":{"v":1,"candidates":[{"url":"https://mcp.zoom.us/mcp/zoom/streamable","source":"mcp_card"},{"url":"https://mcp.zoom.us/mcp/whiteboard/streamable","source":"mcp_card"},{"url":"https://mcp.zoom.us/mcp/docs/streamable","source":"mcp_card"},{"url":"https://mcp.zoom.us/mcp/team_chat/streamable","source":"mcp_card"},{"url":"https://developers.zoom.us/docs/mcp/","source":"homepage_link"}],"mcp":[{"url":"https://mcp.zoom.us/mcp/zoom/streamable","source":"mcp_card","outcome":"auth_required","status":401,"ms":131,"initialized":false,"protocol_version":null,"server":null,"tools":null,"tools_sha256":null,"auth_header":true},{"url":"https://mcp.zoom.us/mcp/whiteboard/streamable","source":"mcp_card","outcome":"auth_required","status":401,"ms":473,"initialized":false,"protocol_version":null,"server":null,"tools":null,"tools_sha256":null,"auth_header":true},{"url":"https://mcp.zoom.us/mcp/docs/streamable","source":"mcp_card","outcome":"auth_required","status":401,"ms":520,"initialized":false,"protocol_version":null,"server":null,"tools":null,"tools_sha256":null,"auth_header":true}],"a2a":null,"webmcp":{"model_context":false,"bridge":false,"mcp_json":false,"link_rel":false,"any":false},"handshakes":3,"get_path":{"status":404,"content_type":"text/html","ms":374,"error":null},"get_host":{"status":404,"content_type":"text/html","ms":822,"error":null}},"derived":{"reachable":true,"robots_served":true,"robots_sitemap":true,"content_signal":false,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":[],"blocks_any_ai":false,"files":{"llms_txt":false,"agents_md":false,"agent_skills":true,"mcp_card":true,"api_catalog":true,"a2a_card":false},"any_capability":true,"ai_refused":[],"ai_thin":[],"rings":{"mcp_declared":true,"mcp_live":false,"mcp_auth":true,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":2,"fh":"www.zoom.com","fp_other_host":[],"fp":"a0beb85019e3668c","fp_bits":{"gptbot":0,"claudebot":0,"oaisearch":0,"perplexity":0}},"derive_v":3,"frame":"top1k","shard":7},{"v":1,"kind":"crawlcheck-census-record","run":14,"domain":"ntp.org","rank":73,"measured_at":"2026-10-15T05:26:03.595Z","home":{"browser":{"status":200,"content_type":"text/html","bytes":9114,"sha256":"9812869b85080fba2b26749348b535e391dfdc861753788aefc52b86262e3c41","final_host":"www.ntp.org","ms":1827,"identity":"Mobile browser","html":true,"fp":"4d02f33496f546e0"},"gptbot":{"status":200,"content_type":"text/html","bytes":9114,"sha256":"9812869b85080fba2b26749348b535e391dfdc861753788aefc52b86262e3c41","final_host":"www.ntp.org","ms":2108,"identity":"GPTBot","html":true,"fp":"4d02f33496f546e0"},"claudebot":{"status":200,"content_type":"text/html","bytes":9114,"sha256":"9812869b85080fba2b26749348b535e391dfdc861753788aefc52b86262e3c41","final_host":"www.ntp.org","ms":2134,"identity":"ClaudeBot","html":true,"fp":"4d02f33496f546e0"},"oaisearch":{"status":200,"content_type":"text/html","bytes":9114,"sha256":"9812869b85080fba2b26749348b535e391dfdc861753788aefc52b86262e3c41","final_host":"www.ntp.org","ms":1975,"identity":"OAI-SearchBot","html":true,"fp":"4d02f33496f546e0"},"perplexity":{"status":200,"content_type":"text/html","bytes":9114,"sha256":"9812869b85080fba2b26749348b535e391dfdc861753788aefc52b86262e3c41","final_host":"www.ntp.org","ms":1967,"identity":"PerplexityBot","html":true,"fp":"4d02f33496f546e0"}},"robots":{"status":200,"content_type":"text/plain","bytes":16,"sha256":"5a5a4434f108ef034c256b9d3e169e516fb806aa6f6353e53eff9655732fbb79","final_host":"www.ntp.org","ms":1736,"html":false,"text":"\nUser-agent: *\n\n","agents":{"GPTBot":true,"OAI-SearchBot":true,"ChatGPT-User":true,"ClaudeBot":true,"Claude-SearchBot":true,"PerplexityBot":true,"Google-Extended":true,"Applebot-Extended":true,"CCBot":true,"Bytespider":true,"Meta-ExternalAgent":true,"Amazonbot":true}},"sitemap":{"status":200,"content_type":"application/xml","bytes":420,"sha256":"435f823bc1987b2bbde0e8ae89e9984fe8ef5d03d06e126455c2578cfbb94ab3","final_host":"www.ntp.org","ms":356,"url":"https://ntp.org/sitemap.xml","declared":false,"xml":true},"files":{"llms_txt":{"status":404,"content_type":"text/html","bytes":6601,"sha256":"ce78ad119af14e88e0bd97553a54a70a036626840ba00e22b9aebc2022165394","final_host":"www.ntp.org","ms":519,"path":"/llms.txt","html":true,"json":null,"present":false},"agents_md":{"status":404,"content_type":"text/html","bytes":6601,"sha256":"ce78ad119af14e88e0bd97553a54a70a036626840ba00e22b9aebc2022165394","final_host":"www.ntp.org","ms":357,"path":"/agents.md","html":true,"json":null,"present":false},"agent_skills":{"status":404,"content_type":"text/html","bytes":6601,"sha256":"ce78ad119af14e88e0bd97553a54a70a036626840ba00e22b9aebc2022165394","final_host":"www.ntp.org","ms":661,"path":"/.well-known/agent-skills/index.json","html":true,"json":null,"present":false},"mcp_card":{"status":404,"content_type":"text/html","bytes":6601,"sha256":"ce78ad119af14e88e0bd97553a54a70a036626840ba00e22b9aebc2022165394","final_host":"www.ntp.org","ms":867,"path":"/.well-known/mcp/server-card.json","html":true,"json":null,"present":false},"api_catalog":{"status":404,"content_type":"text/html","bytes":6601,"sha256":"ce78ad119af14e88e0bd97553a54a70a036626840ba00e22b9aebc2022165394","final_host":"www.ntp.org","ms":410,"path":"/.well-known/api-catalog","html":true,"json":null,"present":false},"a2a_card":{"status":404,"content_type":"text/html","bytes":6601,"sha256":"ce78ad119af14e88e0bd97553a54a70a036626840ba00e22b9aebc2022165394","final_host":"www.ntp.org","ms":792,"path":"/.well-known/agent-card.json","html":true,"json":null,"present":false}},"rings":{"v":1,"candidates":[],"mcp":[],"a2a":null,"webmcp":{"model_context":false,"bridge":false,"mcp_json":false,"link_rel":false,"any":false},"handshakes":0,"get_path":{"status":404,"content_type":"text/html","ms":362,"error":null},"get_host":{"status":530,"content_type":"text/plain","ms":28,"error":null}},"derived":{"reachable":true,"robots_served":true,"robots_sitemap":false,"content_signal":false,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":[],"blocks_any_ai":false,"files":{"llms_txt":false,"agents_md":false,"agent_skills":false,"mcp_card":false,"api_catalog":false,"a2a_card":false},"any_capability":false,"ai_refused":[],"ai_thin":[],"rings":{"mcp_declared":false,"mcp_live":false,"mcp_auth":false,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":2,"fh":"www.ntp.org","fp_other_host":[],"fp":"4d02f33496f546e0","fp_bits":{"gptbot":0,"claudebot":0,"oaisearch":0,"perplexity":0}},"derive_v":3,"frame":"top1k","shard":0},{"v":1,"kind":"crawlcheck-census-record","run":14,"domain":"qq.com","rank":74,"measured_at":"2026-10-15T05:27:03.729Z","home":{"browser":{"status":200,"content_type":"text/html","bytes":136453,"sha256":"e1934c6f27634760009edba256ee42bc366e0c312300f98e6b4924c452656e38","final_host":"www.qq.com","ms":2379,"identity":"Mobile browser","html":true,"fp":"025b646632274ac2"},"gptbot":{"status":200,"content_type":"text/html","bytes":136454,"sha256":"20d461473f31e33b534abca747920fbc767ba6d28d498308b58a531e7bc89f3f","final_host":"www.qq.com","ms":2455,"identity":"GPTBot","html":true,"fp":"025b646632274ac2"},"claudebot":{"status":200,"content_type":"text/html","bytes":136463,"sha256":"6f3a400d07375d8755a35d0e6ce45fe5333a1e772770d224acaf584d2d06b2c5","final_host":"www.qq.com","ms":2376,"identity":"ClaudeBot","html":true,"fp":"025b646632274ac2"},"oaisearch":{"status":200,"content_type":"text/html","bytes":136463,"sha256":"06a98d1624a4218c2b9c6f0cccfa40db9969f37e23718e5aa37557a95eee2f78","final_host":"www.qq.com","ms":2370,"identity":"OAI-SearchBot","html":true,"fp":"025b646632274ac2"},"perplexity":{"status":200,"content_type":"text/html","bytes":136445,"sha256":"b00fd94199a3a57383701a5072e0b9893303d66447679a33e2498b911f21826d","final_host":"www.qq.com","ms":2335,"identity":"PerplexityBot","html":true,"fp":"025b646632274ac2"}},"robots":{"status":200,"content_type":"text/plain","bytes":71,"sha256":"1f5d4ddc6513b31998acadbb56fe02e8b9eb244a3eb64d36f446f146e0bc3d5f","final_host":"www.qq.com","ms":2475,"html":false,"text":"User-agent: *\nDisallow: \nSitemap: https://news.qq.com/sitemap/index.xml","agents":{"GPTBot":true,"OAI-SearchBot":true,"ChatGPT-User":true,"ClaudeBot":true,"Claude-SearchBot":true,"PerplexityBot":true,"Google-Extended":true,"Applebot-Extended":true,"CCBot":true,"Bytespider":true,"Meta-ExternalAgent":true,"Amazonbot":true}},"sitemap":{"status":200,"content_type":"application/xml","bytes":105125,"sha256":"4d7158bc12ef8319465783f0ef0a6bc348d2bb3aae6bd0e192747959aacd4535","final_host":"news.qq.com","ms":2687,"url":"https://news.qq.com/sitemap/index.xml","declared":true,"xml":true},"files":{"llms_txt":{"status":200,"content_type":"text/html","bytes":4183,"sha256":"dd3ca8f2a7ee2f282645a07d06786a1fdb6f0caa1e090063805dc51f525c2464","final_host":"h5.news.qq.com","ms":2002,"path":"/llms.txt","html":true,"json":null,"present":false},"agents_md":{"status":200,"content_type":"text/html","bytes":4183,"sha256":"dd3ca8f2a7ee2f282645a07d06786a1fdb6f0caa1e090063805dc51f525c2464","final_host":"h5.news.qq.com","ms":1960,"path":"/agents.md","html":true,"json":null,"present":false},"agent_skills":{"status":200,"content_type":"text/html","bytes":4183,"sha256":"dd3ca8f2a7ee2f282645a07d06786a1fdb6f0caa1e090063805dc51f525c2464","final_host":"h5.news.qq.com","ms":1931,"path":"/.well-known/agent-skills/index.json","html":true,"json":null,"present":false},"mcp_card":{"status":200,"content_type":"text/html","bytes":4183,"sha256":"dd3ca8f2a7ee2f282645a07d06786a1fdb6f0caa1e090063805dc51f525c2464","final_host":"h5.news.qq.com","ms":1707,"path":"/.well-known/mcp/server-card.json","html":true,"json":null,"present":false},"api_catalog":{"status":200,"content_type":"text/html","bytes":4183,"sha256":"dd3ca8f2a7ee2f282645a07d06786a1fdb6f0caa1e090063805dc51f525c2464","final_host":"h5.news.qq.com","ms":1306,"path":"/.well-known/api-catalog","html":true,"json":null,"present":false},"a2a_card":{"status":200,"content_type":"text/html","bytes":4183,"sha256":"dd3ca8f2a7ee2f282645a07d06786a1fdb6f0caa1e090063805dc51f525c2464","final_host":"h5.news.qq.com","ms":770,"path":"/.well-known/agent-card.json","html":true,"json":null,"present":false}},"rings":{"v":1,"candidates":[],"mcp":[],"a2a":null,"webmcp":{"model_context":false,"bridge":false,"mcp_json":false,"link_rel":false,"any":false},"handshakes":0,"get_path":{"status":200,"content_type":"text/html","ms":677,"error":null},"get_host":{"status":530,"content_type":"text/plain","ms":463,"error":null}},"derived":{"reachable":true,"robots_served":true,"robots_sitemap":true,"content_signal":false,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":[],"blocks_any_ai":false,"files":{"llms_txt":false,"agents_md":false,"agent_skills":false,"mcp_card":false,"api_catalog":false,"a2a_card":false},"any_capability":false,"ai_refused":[],"ai_thin":[],"rings":{"mcp_declared":false,"mcp_live":false,"mcp_auth":false,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":2,"fh":"www.qq.com","fp_other_host":[],"fp":"025b646632274ac2","fp_bits":{"gptbot":0,"claudebot":0,"oaisearch":0,"perplexity":0}},"derive_v":3,"frame":"top1k","shard":1},{"v":1,"kind":"crawlcheck-census-record","run":14,"domain":"root-servers.net","rank":75,"measured_at":"2026-10-15T05:28:03.923Z","home":{"browser":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"root-servers.net","ms":80,"identity":"Mobile browser","html":false},"gptbot":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"root-servers.net","ms":80,"identity":"GPTBot","html":false},"claudebot":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"root-servers.net","ms":79,"identity":"ClaudeBot","html":false},"oaisearch":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"root-servers.net","ms":3,"identity":"OAI-SearchBot","html":false},"perplexity":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"root-servers.net","ms":3,"identity":"PerplexityBot","html":false}},"robots":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"root-servers.net","ms":5,"html":false,"text":null,"agents":null},"sitemap":null,"files":null,"derived":{"reachable":false,"robots_served":false,"robots_sitemap":false,"content_signal":false,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":[],"blocks_any_ai":false,"files":{"llms_txt":false,"agents_md":false,"agent_skills":false,"mcp_card":false,"api_catalog":false,"a2a_card":false},"any_capability":false,"ai_refused":[],"ai_thin":[],"rings":{"mcp_declared":false,"mcp_live":false,"mcp_auth":false,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":2,"fh":"root-servers.net","fp_other_host":[]},"derive_v":3,"frame":"top1k","shard":2},{"v":1,"kind":"crawlcheck-census-record","run":14,"domain":"nginx.org","rank":76,"measured_at":"2026-10-15T05:29:03.596Z","home":{"browser":{"status":200,"content_type":"text/html","bytes":6736,"sha256":"5fbe37e0082f934109a0df667423876337ebfe0e78ddc6ed039697affeb2ea42","final_host":"nginx.org","ms":212,"identity":"Mobile browser","html":true,"fp":"f7a06c0751c419ba"},"gptbot":{"status":200,"content_type":"text/html","bytes":6736,"sha256":"5fbe37e0082f934109a0df667423876337ebfe0e78ddc6ed039697affeb2ea42","final_host":"nginx.org","ms":153,"identity":"GPTBot","html":true,"fp":"f7a06c0751c419ba"},"claudebot":{"status":200,"content_type":"text/html","bytes":6736,"sha256":"5fbe37e0082f934109a0df667423876337ebfe0e78ddc6ed039697affeb2ea42","final_host":"nginx.org","ms":173,"identity":"ClaudeBot","html":true,"fp":"f7a06c0751c419ba"},"oaisearch":{"status":200,"content_type":"text/html","bytes":6736,"sha256":"5fbe37e0082f934109a0df667423876337ebfe0e78ddc6ed039697affeb2ea42","final_host":"nginx.org","ms":116,"identity":"OAI-SearchBot","html":true,"fp":"f7a06c0751c419ba"},"perplexity":{"status":200,"content_type":"text/html","bytes":6736,"sha256":"5fbe37e0082f934109a0df667423876337ebfe0e78ddc6ed039697affeb2ea42","final_host":"nginx.org","ms":59,"identity":"PerplexityBot","html":true,"fp":"f7a06c0751c419ba"}},"robots":{"status":200,"content_type":"text/plain","bytes":72,"sha256":"8f5b42f5d23ca623fa42f917a5c90b8677ef8d7c9b5ffc0324bbea8e7067f26e","final_host":"nginx.org","ms":193,"html":false,"text":"User-Agent: *\nDisallow: /libxslt/\nSitemap: http://nginx.org/sitemap.xml\n","agents":{"GPTBot":true,"OAI-SearchBot":true,"ChatGPT-User":true,"ClaudeBot":true,"Claude-SearchBot":true,"PerplexityBot":true,"Google-Extended":true,"Applebot-Extended":true,"CCBot":true,"Bytespider":true,"Meta-ExternalAgent":true,"Amazonbot":true}},"sitemap":{"status":200,"content_type":"text/xml","bytes":41548,"sha256":"b0cf68570ac4578746020695a2028f85a7c8f55d6c15208495e0da87888d18a9","final_host":"nginx.org","ms":97,"url":"http://nginx.org/sitemap.xml","declared":true,"xml":true},"files":{"llms_txt":{"status":404,"content_type":"text/html","bytes":2010,"sha256":"682eb26255576d9eaaeada47525a71cfbaf01f5dcd1e6398576f296c18b6aef4","final_host":"nginx.org","ms":25,"path":"/llms.txt","html":true,"json":null,"present":false},"agents_md":{"status":404,"content_type":"text/html","bytes":2010,"sha256":"682eb26255576d9eaaeada47525a71cfbaf01f5dcd1e6398576f296c18b6aef4","final_host":"nginx.org","ms":37,"path":"/agents.md","html":true,"json":null,"present":false},"agent_skills":{"status":404,"content_type":"text/html","bytes":2010,"sha256":"682eb26255576d9eaaeada47525a71cfbaf01f5dcd1e6398576f296c18b6aef4","final_host":"nginx.org","ms":42,"path":"/.well-known/agent-skills/index.json","html":true,"json":null,"present":false},"mcp_card":{"status":404,"content_type":"text/html","bytes":2010,"sha256":"682eb26255576d9eaaeada47525a71cfbaf01f5dcd1e6398576f296c18b6aef4","final_host":"nginx.org","ms":59,"path":"/.well-known/mcp/server-card.json","html":true,"json":null,"present":false},"api_catalog":{"status":404,"content_type":"text/html","bytes":2010,"sha256":"682eb26255576d9eaaeada47525a71cfbaf01f5dcd1e6398576f296c18b6aef4","final_host":"nginx.org","ms":79,"path":"/.well-known/api-catalog","html":true,"json":null,"present":false},"a2a_card":{"status":404,"content_type":"text/html","bytes":2010,"sha256":"682eb26255576d9eaaeada47525a71cfbaf01f5dcd1e6398576f296c18b6aef4","final_host":"nginx.org","ms":114,"path":"/.well-known/agent-card.json","html":true,"json":null,"present":false}},"rings":{"v":1,"candidates":[],"mcp":[],"a2a":null,"webmcp":{"model_context":false,"bridge":false,"mcp_json":false,"link_rel":false,"any":false},"handshakes":0,"get_path":{"status":404,"content_type":"text/html","ms":26,"error":null},"get_host":{"status":530,"content_type":"text/plain","ms":254,"error":null}},"derived":{"reachable":true,"robots_served":true,"robots_sitemap":true,"content_signal":false,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":[],"blocks_any_ai":false,"files":{"llms_txt":false,"agents_md":false,"agent_skills":false,"mcp_card":false,"api_catalog":false,"a2a_card":false},"any_capability":false,"ai_refused":[],"ai_thin":[],"rings":{"mcp_declared":false,"mcp_live":false,"mcp_auth":false,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":2,"fh":"nginx.org","fp_other_host":[],"fp":"f7a06c0751c419ba","fp_bits":{"gptbot":0,"claudebot":0,"oaisearch":0,"perplexity":0}},"derive_v":3,"frame":"top1k","shard":3},{"v":1,"kind":"crawlcheck-census-record","run":14,"domain":"akam.net","rank":77,"measured_at":"2026-10-15T05:30:03.842Z","home":{"browser":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"akam.net","ms":30,"identity":"Mobile browser","html":false},"gptbot":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"akam.net","ms":2,"identity":"GPTBot","html":false},"claudebot":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"akam.net","ms":1,"identity":"ClaudeBot","html":false},"oaisearch":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"akam.net","ms":1,"identity":"OAI-SearchBot","html":false},"perplexity":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"akam.net","ms":2,"identity":"PerplexityBot","html":false}},"robots":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"akam.net","ms":2,"html":false,"text":null,"agents":null},"sitemap":null,"files":null,"derived":{"reachable":false,"robots_served":false,"robots_sitemap":false,"content_signal":false,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":[],"blocks_any_ai":false,"files":{"llms_txt":false,"agents_md":false,"agent_skills":false,"mcp_card":false,"api_catalog":false,"a2a_card":false},"any_capability":false,"ai_refused":[],"ai_thin":[],"rings":{"mcp_declared":false,"mcp_live":false,"mcp_auth":false,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":2,"fh":"akam.net","fp_other_host":[]},"derive_v":3,"frame":"top1k","shard":4},{"v":1,"kind":"crawlcheck-census-record","run":14,"domain":"tiktokcdn.com","rank":78,"measured_at":"2026-10-15T05:31:08.958Z","home":{"browser":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"tiktokcdn.com","ms":17,"identity":"Mobile browser","html":false},"gptbot":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"tiktokcdn.com","ms":17,"identity":"GPTBot","html":false},"claudebot":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"tiktokcdn.com","ms":15,"identity":"ClaudeBot","html":false},"oaisearch":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"tiktokcdn.com","ms":12,"identity":"OAI-SearchBot","html":false},"perplexity":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"tiktokcdn.com","ms":3,"identity":"PerplexityBot","html":false}},"robots":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"tiktokcdn.com","ms":3,"html":false,"text":null,"agents":null},"sitemap":null,"files":null,"derived":{"reachable":false,"robots_served":false,"robots_sitemap":false,"content_signal":false,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":[],"blocks_any_ai":false,"files":{"llms_txt":false,"agents_md":false,"agent_skills":false,"mcp_card":false,"api_catalog":false,"a2a_card":false},"any_capability":false,"ai_refused":[],"ai_thin":[],"rings":{"mcp_declared":false,"mcp_live":false,"mcp_auth":false,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":2,"fh":"tiktokcdn.com","fp_other_host":[]},"derive_v":3,"frame":"top1k","shard":5},{"v":1,"kind":"crawlcheck-census-record","run":14,"domain":"edgekey.net","rank":79,"measured_at":"2026-10-15T05:33:03.639Z","home":{"browser":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"edgekey.net","ms":6,"identity":"Mobile browser","html":false},"gptbot":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"edgekey.net","ms":2,"identity":"GPTBot","html":false},"claudebot":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"edgekey.net","ms":1,"identity":"ClaudeBot","html":false},"oaisearch":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"edgekey.net","ms":2,"identity":"OAI-SearchBot","html":false},"perplexity":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"edgekey.net","ms":1,"identity":"PerplexityBot","html":false}},"robots":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"edgekey.net","ms":2,"html":false,"text":null,"agents":null},"sitemap":null,"files":null,"derived":{"reachable":false,"robots_served":false,"robots_sitemap":false,"content_signal":false,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":[],"blocks_any_ai":false,"files":{"llms_txt":false,"agents_md":false,"agent_skills":false,"mcp_card":false,"api_catalog":false,"a2a_card":false},"any_capability":false,"ai_refused":[],"ai_thin":[],"rings":{"mcp_declared":false,"mcp_live":false,"mcp_auth":false,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":2,"fh":"edgekey.net","fp_other_host":[]},"derive_v":3,"frame":"top1k","shard":6},{"v":1,"kind":"crawlcheck-census-record","run":14,"domain":"baidu.com","rank":80,"measured_at":"2026-10-15T05:34:03.814Z","home":{"browser":{"status":200,"content_type":"text/html","bytes":80651,"sha256":"59fed2bb81347cbeb9b7d8b13383e61df02d6661e14519ae9b2886e51898bacd","final_host":"www.baidu.com","ms":2254,"identity":"Mobile browser","html":true},"gptbot":{"status":200,"content_type":"text/html","bytes":736307,"sha256":"4f38d3776e6c3606abe6517fa71533b170272beb5cd5029aac71dab69109a1d0","final_host":"www.baidu.com","ms":2650,"identity":"GPTBot","html":true,"fp":"9e3e791b69811335"},"claudebot":{"status":200,"content_type":"text/html","bytes":736981,"sha256":"b3792e890061313255eb7b6705dc10f66e78d7e4e30c981035a49e76bafb4ad6","final_host":"www.baidu.com","ms":3393,"identity":"ClaudeBot","html":true,"fp":"9e3e791b61811335"},"oaisearch":{"status":200,"content_type":"text/html","bytes":736327,"sha256":"19209275b30fe5eef71caad5351ccef7a0a24c330dc7b1bd7b1d5f14d6c69a67","final_host":"www.baidu.com","ms":2286,"identity":"OAI-SearchBot","html":true,"fp":"9e3e791b69811335"},"perplexity":{"status":200,"content_type":"text/html","bytes":737040,"sha256":"5f6a1054d62814b26d69c5fdcc86f430e5c3e6d73d8d66cb6f8876ecf99f9ec3","final_host":"www.baidu.com","ms":2428,"identity":"PerplexityBot","html":true,"fp":"9e3e791b61811335"}},"robots":{"status":null,"error":"timeout","ms":8000,"html":null,"text":null,"agents":null},"sitemap":{"status":404,"content_type":"text/html","bytes":209,"sha256":"636ca4bd838c8eeffcf8eaae1a56f72b6b93bb9df4f0d6e58333361968684800","final_host":"www.baidu.com","ms":411,"url":"https://baidu.com/sitemap.xml","declared":false,"xml":false},"files":{"llms_txt":{"status":404,"content_type":"text/html","bytes":206,"sha256":"0da006306ca67b9d88367bc1a63929b073ed36fb046b9b61695f934bfa544b89","final_host":"www.baidu.com","ms":461,"path":"/llms.txt","html":true,"json":null,"present":false},"agents_md":{"status":404,"content_type":"text/html","bytes":207,"sha256":"a50408f4a5a4d457e4f46fe099a1286627188827df0a21eac4e3aea2d8f471d6","final_host":"www.baidu.com","ms":948,"path":"/agents.md","html":true,"json":null,"present":false},"agent_skills":{"status":404,"content_type":"text/html","bytes":233,"sha256":"532bb81bdb57c440d614fcb44cbbcac627252208828f307c7943796457dfc49c","final_host":"www.baidu.com","ms":483,"path":"/.well-known/agent-skills/index.json","html":true,"json":null,"present":false},"mcp_card":{"status":404,"content_type":"text/html","bytes":230,"sha256":"ff2106cf9a2975014654029d3a652abeccb0b8c216730f442ddabc48f763a431","final_host":"www.baidu.com","ms":1107,"path":"/.well-known/mcp/server-card.json","html":true,"json":null,"present":false},"api_catalog":{"status":404,"content_type":"text/html","bytes":221,"sha256":"923d0d4fc2d80a5fe8f23656d1f52a907c9fe33c7380164a03f29abd4b3979b6","final_host":"www.baidu.com","ms":964,"path":"/.well-known/api-catalog","html":true,"json":null,"present":false},"a2a_card":{"status":null,"error":"timeout","ms":8001,"path":"/.well-known/agent-card.json","html":null,"json":null,"present":false}},"rings":{"v":1,"candidates":[],"mcp":[],"a2a":null,"webmcp":{"model_context":false,"bridge":false,"mcp_json":false,"link_rel":false,"any":false},"handshakes":0,"get_path":{"status":404,"content_type":"text/html","ms":453,"error":null},"get_host":{"status":404,"content_type":"text/html","ms":4967,"error":null}},"derived":{"reachable":true,"robots_served":false,"robots_sitemap":false,"content_signal":false,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":[],"blocks_any_ai":false,"files":{"llms_txt":false,"agents_md":false,"agent_skills":false,"mcp_card":false,"api_catalog":false,"a2a_card":false},"any_capability":false,"ai_refused":[],"ai_thin":[],"rings":{"mcp_declared":false,"mcp_live":false,"mcp_auth":false,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":2,"fh":"www.baidu.com","fp_other_host":[]},"derive_v":3,"frame":"top1k","shard":7},{"v":1,"kind":"crawlcheck-census-record","run":14,"domain":"workers.dev","rank":81,"measured_at":"2026-10-15T05:26:23.049Z","home":{"browser":{"status":200,"content_type":"text/html","bytes":757356,"sha256":"c6af9430c5ee2efe665c661b6171fab3bd18c1a1b9fb61ca24894e5af71ddd17","final_host":"www.cloudflare.com","ms":1036,"identity":"Mobile browser","html":true,"fp":"53623ffd8652bde3"},"gptbot":{"status":200,"content_type":"text/html","bytes":757356,"sha256":"c6af9430c5ee2efe665c661b6171fab3bd18c1a1b9fb61ca24894e5af71ddd17","final_host":"www.cloudflare.com","ms":1042,"identity":"GPTBot","html":true,"fp":"53623ffd8652bde3"},"claudebot":{"status":200,"content_type":"text/html","bytes":757356,"sha256":"04e4bb8a20e03934e549e9cb5bf40024f57e013c10367d1f2feb22d9377e8aef","final_host":"www.cloudflare.com","ms":1004,"identity":"ClaudeBot","html":true,"fp":"53623ffd8652bde3"},"oaisearch":{"status":200,"content_type":"text/html","bytes":757358,"sha256":"c6af9430c5ee2efe665c661b6171fab3bd18c1a1b9fb61ca24894e5af71ddd17","final_host":"www.cloudflare.com","ms":1042,"identity":"OAI-SearchBot","html":true,"fp":"53623ffd8652bde3"},"perplexity":{"status":200,"content_type":"text/html","bytes":757356,"sha256":"c6af9430c5ee2efe665c661b6171fab3bd18c1a1b9fb61ca24894e5af71ddd17","final_host":"www.cloudflare.com","ms":1011,"identity":"PerplexityBot","html":true,"fp":"53623ffd8652bde3"}},"robots":{"status":200,"content_type":"text/html","bytes":756921,"sha256":"e4c78f8b7657aca33dd654faefb8f4de4f98637352c167188d364e467c6e85f5","final_host":"www.cloudflare.com","ms":175,"html":true,"text":null,"agents":null},"sitemap":{"status":200,"content_type":"text/html","bytes":756921,"sha256":"6f7297d68e79fe7179135b342f0a4689d39d9f01e7bd1e95b073108744d3864c","final_host":"www.cloudflare.com","ms":152,"url":"https://workers.dev/sitemap.xml","declared":false,"xml":false},"files":{"llms_txt":{"status":200,"content_type":"text/markdown","bytes":3314,"sha256":"096e84c5fb5f1b086aca292560dab76d533bee7e5c5daa55cabe2d720f36054e","final_host":"www.cloudflare.com","ms":82,"path":"/llms.txt","html":false,"json":null,"present":true},"agents_md":{"status":200,"content_type":"text/markdown","bytes":3314,"sha256":"096e84c5fb5f1b086aca292560dab76d533bee7e5c5daa55cabe2d720f36054e","final_host":"www.cloudflare.com","ms":31,"path":"/agents.md","html":false,"json":null,"present":true},"agent_skills":{"status":200,"content_type":"text/html","bytes":756921,"sha256":"6f7297d68e79fe7179135b342f0a4689d39d9f01e7bd1e95b073108744d3864c","final_host":"www.cloudflare.com","ms":165,"path":"/.well-known/agent-skills/index.json","html":true,"json":null,"present":false},"mcp_card":{"status":200,"content_type":"text/html","bytes":756921,"sha256":"6f7297d68e79fe7179135b342f0a4689d39d9f01e7bd1e95b073108744d3864c","final_host":"www.cloudflare.com","ms":140,"path":"/.well-known/mcp/server-card.json","html":true,"json":null,"present":false},"api_catalog":{"status":200,"content_type":"text/html","bytes":756921,"sha256":"6f7297d68e79fe7179135b342f0a4689d39d9f01e7bd1e95b073108744d3864c","final_host":"www.cloudflare.com","ms":143,"path":"/.well-known/api-catalog","html":true,"json":null,"present":false},"a2a_card":{"status":200,"content_type":"text/html","bytes":756921,"sha256":"6f7297d68e79fe7179135b342f0a4689d39d9f01e7bd1e95b073108744d3864c","final_host":"www.cloudflare.com","ms":133,"path":"/.well-known/agent-card.json","html":true,"json":null,"present":false}},"rings":{"v":1,"candidates":[],"mcp":[],"a2a":null,"webmcp":{"model_context":false,"bridge":false,"mcp_json":false,"link_rel":false,"any":false},"handshakes":0,"get_path":{"status":200,"content_type":"text/html","ms":145,"error":null},"get_host":{"status":530,"content_type":"text/plain","ms":13,"error":null}},"derived":{"reachable":true,"robots_served":false,"robots_sitemap":false,"content_signal":false,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":[],"blocks_any_ai":false,"files":{"llms_txt":true,"agents_md":true,"agent_skills":false,"mcp_card":false,"api_catalog":false,"a2a_card":false},"any_capability":false,"ai_refused":[],"ai_thin":[],"rings":{"mcp_declared":false,"mcp_live":false,"mcp_auth":false,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":2,"fh":"www.cloudflare.com","fp_other_host":[],"fp":"53623ffd8652bde3","fp_bits":{"gptbot":0,"claudebot":0,"oaisearch":0,"perplexity":0}},"derive_v":3,"frame":"top1k","shard":0},{"v":1,"kind":"crawlcheck-census-record","run":14,"domain":"tiktokv.com","rank":82,"measured_at":"2026-10-15T05:27:24.896Z","home":{"browser":{"status":404,"content_type":"text/html","bytes":546,"sha256":"5dd23ed6464f587908166860364362088446bf508f2793c4df0ad2119675c33e","final_host":"tiktokv.com","ms":218,"identity":"Mobile browser","html":true},"gptbot":{"status":404,"content_type":"text/html","bytes":144,"sha256":"b282ac8512c5e029959282e6fd450dc119f5db4a072869eabf45e31440596248","final_host":"tiktokv.com","ms":554,"identity":"GPTBot","html":true},"claudebot":{"status":404,"content_type":"text/html","bytes":144,"sha256":"b282ac8512c5e029959282e6fd450dc119f5db4a072869eabf45e31440596248","final_host":"tiktokv.com","ms":553,"identity":"ClaudeBot","html":true},"oaisearch":{"status":404,"content_type":"text/html","bytes":144,"sha256":"b282ac8512c5e029959282e6fd450dc119f5db4a072869eabf45e31440596248","final_host":"tiktokv.com","ms":569,"identity":"OAI-SearchBot","html":true},"perplexity":{"status":404,"content_type":"text/html","bytes":144,"sha256":"b282ac8512c5e029959282e6fd450dc119f5db4a072869eabf45e31440596248","final_host":"tiktokv.com","ms":568,"identity":"PerplexityBot","html":true}},"robots":{"status":404,"content_type":"text/html","bytes":144,"sha256":"b282ac8512c5e029959282e6fd450dc119f5db4a072869eabf45e31440596248","final_host":"tiktokv.com","ms":250,"html":true,"text":null,"agents":null},"sitemap":null,"files":null,"derived":{"reachable":false,"robots_served":false,"robots_sitemap":false,"content_signal":false,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":[],"blocks_any_ai":false,"files":{"llms_txt":false,"agents_md":false,"agent_skills":false,"mcp_card":false,"api_catalog":false,"a2a_card":false},"any_capability":false,"ai_refused":[],"ai_thin":[],"rings":{"mcp_declared":false,"mcp_live":false,"mcp_auth":false,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":2,"fh":"tiktokv.com","fp_other_host":[]},"derive_v":3,"frame":"top1k","shard":1},{"v":1,"kind":"crawlcheck-census-record","run":14,"domain":"chatgpt.com","rank":83,"measured_at":"2026-10-15T05:28:26.341Z","home":{"browser":{"status":403,"content_type":"text/html","bytes":8686,"sha256":"ece97874e43023aae2341059d928024adbc1af2eb28207402ba701db4710cbdd","final_host":"chatgpt.com","ms":6,"identity":"Mobile browser","html":true},"gptbot":{"status":403,"content_type":"text/html","bytes":8665,"sha256":"e9ff3e99ecac5b63b7f8e92b46a0cc56583b7c9dd16d465215972c2d8469f9bd","final_host":"chatgpt.com","ms":8,"identity":"GPTBot","html":true},"claudebot":{"status":403,"content_type":"text/html","bytes":8579,"sha256":"a116f2811a2adf739185c2259e174feaa483591c76a1fdd27f8d7497a7406b4f","final_host":"chatgpt.com","ms":8,"identity":"ClaudeBot","html":true},"oaisearch":{"status":403,"content_type":"text/html","bytes":8707,"sha256":"4ca20b773b55541ca0d633cc2c921702ca4c7cedd9720174ea344c80cfdbdeb6","final_host":"chatgpt.com","ms":8,"identity":"OAI-SearchBot","html":true},"perplexity":{"status":403,"content_type":"text/html","bytes":8772,"sha256":"5f32b7c13ff449ce823b9059d9d6aac2feabc2942d12a0f71ec4655a65fd7b1a","final_host":"chatgpt.com","ms":8,"identity":"PerplexityBot","html":true}},"robots":{"status":200,"content_type":"text/plain","bytes":4390,"sha256":"a6628f6c9c3b373e41c38ce40a18a15f65179e19241e63e78bcfef74dcaee21a","final_host":"chatgpt.com","ms":11,"html":false,"text":"# https://www.robotstxt.org/robotstxt.html\n\nUser-agent: CCBot\nDisallow: /\n\nUser-agent: img2dataset\nDisallow: /\n\nUser-agent: Google-Extended\nDisallow: /\n\n# Prevent shared pages and GPTs from being used as Google Ads destinations\nUser-agent: AdsBot-Google\nUser-agent: AdsBot-Google-Mobile\nDisallow: /s/\nDisallow: /g/\n\nUser-agent: anthropic-ai\nDisallow: /\n\nUser-agent: Claude-Web\nDisallow: /\n\nUser-agent: Omgilibot\nDisallow: /\n\nUser-agent: Omgili\nDisallow: /\n\nUser-agent: FacebookBot\nDisallow: /\n\nUser-agent: Bytespider\nDisallow: /\n\nUser-agent: magpie-crawler\nDisallow: /\n\nUser-agent: PerplexityBot\nDisallow: /\n\nUser-agent: Perplexity‑User\nDisallow: /\n\n# General rules for all other bots\nUser-agent: *\n\n# Place allows first to avoid bots skipping after Disallow: /\n# Allow exactly the homepage\nAllow: /$\n# Allow the homepage with any query parameters\nAllow: /?*\nAllow: /api/share/og/\nAllow: /g/\nAllow: /s/\nAllow: /gg/v/\nAllow: /m/\nAllow: /share/\nAllow: /maps$\nAllow: /maps/\nAllow: /*/maps$\nAllow: /*/maps/\nAllow: /canvas/shared/\nAllow: /*/images\nAllow: /images\nAllow: /*/library\nAllow: /library\nAllow: /favicon.ico\nAllow: /assets/favicon\nAllow: /cdn/assets/favicon\nAllow: /cdn/assets/\nAllow: /auth/\nAllow: /gpts$\nAllow: /codex\nAllow: /*/codex\nAllow: /search$\nAllow: /backend-anon/\nAllow: /public-api/\nAllow: /sitemap.xml\nAllow: /marketing-sitemap.xml\nAllow: /images-sitemap.xml\nAllow: /writing-tools-sitemap.xml\nAllow: /football-sitemap.xml\nAllow: /acquisition-landing-pages-sitemap.xml\nAllow: /100chats\nAllow: /api/public_content/\nAllow: /backend-api/public_content/\nAllow: /?ref=dotcom\n# Static Landing Pages\nAllow: /overview\nAllow: /*/overview\nAllow: /features\nAllow: /*/features\nAllow: /health$\nAllow: /health/$\nAllow: /*/health$\nAllow: /*/health/$\nAllow: /voice$\nAllow: /voice/$\nAllow: /*/voice$\nAllow: /*/voice/$\nAllow: /apps\nAllow: /*/apps\nAllow: /finances\nAllow: /*/finances\nAllow: /shopping\nAllow: /*/shopping\nAllow: /use-cases\nAllow: /*/use-cases\nAllow: /learn\nAllow: /*/learn\nAllow: /business\nAllow: /*/business\nAllow: /pricing\nAllow: /*/pricing\nAllow: /download\nAllow: /*/download\nAllow: /students\nAllow: /students/2026\nAllow: /*/college-students\nAllow: /college-students\nAllow: /contact-sales\nAllow: /*/contact-sales\nAllow: /100chats-project\nAllow: /*/100chats-project\nAllow: /merchants\nAllow: /*/merchants\nAllow: /parent-resources\nAllow: /*/parent-resources\nAllow: /plans\nAllow: /*/plans\nAllow: /plugins\nAllow: /translate\nAllow: /*/translate\nAllow: /writing\nAllow: /*/writing\nAllow: /ask/\nAllow: /futures\nAllow: /*/futures\nAllow: /football\nAllow: /*/football\nAllow: /remote\nAllow: /*/remote\nAllow: /work\nAllow: /*/work\nAllow: /dots\nAllow: /*/dots\nAllow: /sites\nAllow: /*/sites\nAllow: /space\nAllow: /*/space\nAllow: /import-to-chatgpt\nAllow: /*/import-to-chatgpt\n# Exact locale specific homepages\nAllow: /am/$\nAllow: /ar/$\nAllow: /bg-BG/$\nAllow: /bn-BD/$\nAllow: /bs-BA/$\nAllow: /ca-ES/$\nAllow: /cs-CZ/$\nAllow: /da-DK/$\nAllow: /de-DE/$\nAllow: /el-GR/$\nAllow: /es-ES/$\nAllow: /es-419/$\nAllow: /et-EE/$\nAllow: /fi-FI/$\nAllow: /fr-FR/$\nAllow: /fr-CA/$\nAllow: /gu-IN/$\nAllow: /hi-IN/$\nAllow: /hr-HR/$\nAllow: /hu-HU/$\nAllow: /hy-AM/$\nAllow: /id-ID/$\nAllow: /is-IS/$\nAllow: /it-IT/$\nAllow: /ja-JP/$\nAllow: /ka-GE/$\nAllow: /kk/$\nAllow: /kn-IN/$\nAllow: /ko-KR/$\nAllow: /lt/$\nAllow: /lv-LV/$\nAllow: /mk-MK/$\nAllow: /ml/$\nAllow: /mn/$\nAllow: /mr-IN/$\nAllow: /ms-MY/$\nAllow: /my-MM/$\nAllow: /nb-NO/$\nAllow: /nl-NL/$\nAllow: /pa/$\nAllow: /pl-PL/$\nAllow: /pt-BR/$\nAllow: /pt-PT/$\nAllow: /ro-RO/$\nAllow: /ru-RU/$\nAllow: /sk-SK/$\nAllow: /sl-SI/$\nAllow: /so-SO/$\nAllow: /sq-AL/$\nAllow: /sr-RS/$\nAllow: /sv-SE/$\nAllow: /sw-TZ/$\nAllow: /ta-IN/$\nAllow: /te-IN/$\nAllow: /th-TH/$\nAllow: /tl/$\nAllow: /tr-TR/$\nAllow: /uk-UA/$\nAllow: /ur/$\nAllow: /vi-VN/$\nAllow: /zh-CN/$\nAllow: /zh-TW/$\nAllow: /zh-HK/$\n\n# Now block everything else\nDisallow: /\n\n# Specific disallows (redundant for some bots, but still useful for those that respect precedence)\nDisallow: /auth/logout\nDisallow: /auth/login?*\nDisallow: /backend-anon/sentinel/*\nDisallow: /backend-anon/conversation$\nDisallow: /account-link/*\n\nSitemap: https://chatgpt.com/sitemap.xml\nSitemap: https://chatgpt.com/marketing-sitemap.xml\nSitemap: https://chatgpt.com/images-sitemap.xml\nSitemap: https://chatgpt.com/writing-tools-sitemap.xml\nSitemap: https://chatgpt.com/football-sitemap.xml\nSitemap: https://chatgpt.com/acquisition-landing-pages-sitemap.xml\n","agents":{"GPTBot":true,"OAI-SearchBot":true,"ChatGPT-User":true,"ClaudeBot":true,"Claude-SearchBot":true,"PerplexityBot":false,"Google-Extended":false,"Applebot-Extended":true,"CCBot":false,"Bytespider":false,"Meta-ExternalAgent":true,"Amazonbot":true}},"sitemap":null,"files":null,"derived":{"reachable":false,"robots_served":false,"robots_sitemap":false,"content_signal":false,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":[],"blocks_any_ai":false,"files":{"llms_txt":false,"agents_md":false,"agent_skills":false,"mcp_card":false,"api_catalog":false,"a2a_card":false},"any_capability":false,"ai_refused":[],"ai_thin":[],"rings":{"mcp_declared":false,"mcp_live":false,"mcp_auth":false,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":2,"fh":"chatgpt.com","fp_other_host":[]},"derive_v":3,"frame":"top1k","shard":2},{"v":1,"kind":"crawlcheck-census-record","run":14,"domain":"yandex.net","rank":84,"measured_at":"2026-10-15T05:29:25.755Z","home":{"browser":{"status":200,"content_type":"text/html","bytes":415661,"sha256":"3d8bc29446342e76b0cf6fa629c9c7fe89f4b1e03a1a407632ba4e277dfa09d6","final_host":"ya.ru","ms":907,"identity":"Mobile browser","html":true},"gptbot":{"status":200,"content_type":"text/html","bytes":497694,"sha256":"01e77063766b7be310d6bc68ace12bf4f3646ef1a7944f3d147c6a237c082e87","final_host":"ya.ru","ms":771,"identity":"GPTBot","html":true},"claudebot":{"status":200,"content_type":"text/html","bytes":497668,"sha256":"38b70d331b63ad92df4bd48a88999b1ac56a7550dd939f7c45fa4b9a48fd7a37","final_host":"ya.ru","ms":760,"identity":"ClaudeBot","html":true},"oaisearch":{"status":200,"content_type":"text/html","bytes":497683,"sha256":"80180f84495d6f199d250cca806d3e71f36343f8446fb84f42ad84dcfd4ebc53","final_host":"ya.ru","ms":771,"identity":"OAI-SearchBot","html":true},"perplexity":{"status":200,"content_type":"text/html","bytes":497709,"sha256":"d598b1af82987e779cecf0f064ae0237dcf8d6f8809b8a2a9ef1c169926f3c11","final_host":"ya.ru","ms":780,"identity":"PerplexityBot","html":true}},"robots":{"status":404,"content_type":null,"bytes":9,"sha256":"0019dfc4b32d63c1392aa264aed2253c1e0c2fb09216f8e2cc269bbfb8bb49b5","final_host":"yandex.net","ms":279,"html":false,"text":null,"agents":null},"sitemap":{"status":404,"content_type":"text/html","bytes":11776,"sha256":"19ac139fe464f77e63d69e0d2978b79d48959130409a5f0c6ce3d9cd3c09cdd8","final_host":"ya.ru","ms":118,"url":"https://yandex.net/sitemap.xml","declared":false,"xml":false},"files":{"llms_txt":{"status":404,"content_type":"text/html","bytes":11773,"sha256":"a492825af0c554aedd0ffedb1a3ef72df1fc5f538a9d3ba6176772adc633f501","final_host":"ya.ru","ms":127,"path":"/llms.txt","html":true,"json":null,"present":false},"agents_md":{"status":404,"content_type":"text/html","bytes":11774,"sha256":"f57f5f735018baa333bca2e9d172a09796606f9cb090992251b449f7def60aac","final_host":"ya.ru","ms":122,"path":"/agents.md","html":true,"json":null,"present":false},"agent_skills":{"status":404,"content_type":null,"bytes":9,"sha256":"0019dfc4b32d63c1392aa264aed2253c1e0c2fb09216f8e2cc269bbfb8bb49b5","final_host":"yandex.net","ms":63,"path":"/.well-known/agent-skills/index.json","html":false,"json":null,"present":false},"mcp_card":{"status":404,"content_type":null,"bytes":9,"sha256":"0019dfc4b32d63c1392aa264aed2253c1e0c2fb09216f8e2cc269bbfb8bb49b5","final_host":"yandex.net","ms":60,"path":"/.well-known/mcp/server-card.json","html":false,"json":null,"present":false},"api_catalog":{"status":404,"content_type":null,"bytes":9,"sha256":"0019dfc4b32d63c1392aa264aed2253c1e0c2fb09216f8e2cc269bbfb8bb49b5","final_host":"yandex.net","ms":64,"path":"/.well-known/api-catalog","html":false,"json":null,"present":false},"a2a_card":{"status":404,"content_type":null,"bytes":9,"sha256":"0019dfc4b32d63c1392aa264aed2253c1e0c2fb09216f8e2cc269bbfb8bb49b5","final_host":"yandex.net","ms":62,"path":"/.well-known/agent-card.json","html":false,"json":null,"present":false}},"rings":{"v":1,"candidates":[],"mcp":[],"a2a":null,"webmcp":{"model_context":false,"bridge":false,"mcp_json":false,"link_rel":false,"any":false},"handshakes":0,"get_path":{"status":404,"content_type":"text/html","ms":126,"error":null},"get_host":{"status":null,"content_type":null,"ms":8001,"error":"timeout"}},"derived":{"reachable":true,"robots_served":false,"robots_sitemap":false,"content_signal":false,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":[],"blocks_any_ai":false,"files":{"llms_txt":false,"agents_md":false,"agent_skills":false,"mcp_card":false,"api_catalog":false,"a2a_card":false},"any_capability":false,"ai_refused":[],"ai_thin":[],"rings":{"mcp_declared":false,"mcp_live":false,"mcp_auth":false,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":2,"fh":"ya.ru","fp_other_host":[]},"derive_v":3,"frame":"top1k","shard":3},{"v":1,"kind":"crawlcheck-census-record","run":14,"domain":"cloudflare-dns.com","rank":85,"measured_at":"2026-10-15T05:30:22.136Z","home":{"browser":{"status":200,"content_type":"text/html","bytes":229418,"sha256":"cb173d7e74602e093bb23f2f179313958328583f302d5623b9259c3ac1fb7e38","final_host":"one.one.one.one","ms":34,"identity":"Mobile browser","html":true,"fp":"1d5b1d00036cdd02"},"gptbot":{"status":200,"content_type":"text/html","bytes":229418,"sha256":"cb173d7e74602e093bb23f2f179313958328583f302d5623b9259c3ac1fb7e38","final_host":"one.one.one.one","ms":29,"identity":"GPTBot","html":true,"fp":"1d5b1d00036cdd02"},"claudebot":{"status":200,"content_type":"text/html","bytes":229418,"sha256":"cb173d7e74602e093bb23f2f179313958328583f302d5623b9259c3ac1fb7e38","final_host":"one.one.one.one","ms":34,"identity":"ClaudeBot","html":true,"fp":"1d5b1d00036cdd02"},"oaisearch":{"status":200,"content_type":"text/html","bytes":229418,"sha256":"cb173d7e74602e093bb23f2f179313958328583f302d5623b9259c3ac1fb7e38","final_host":"one.one.one.one","ms":34,"identity":"OAI-SearchBot","html":true,"fp":"1d5b1d00036cdd02"},"perplexity":{"status":200,"content_type":"text/html","bytes":229418,"sha256":"cb173d7e74602e093bb23f2f179313958328583f302d5623b9259c3ac1fb7e38","final_host":"one.one.one.one","ms":34,"identity":"PerplexityBot","html":true,"fp":"1d5b1d00036cdd02"}},"robots":{"status":200,"content_type":"text/plain","bytes":1248,"sha256":"8fa3036c68bfcbd32365f6225d24333264093b7cd38d306e106b4dbdc934fd5b","final_host":"one.one.one.one","ms":34,"html":false,"text":"# As a condition of accessing this website, you agree to abide by the following\n# content signals:\n\n# (a)  If a content-signal = yes, you may collect content for the corresponding\n#      use.\n# (b)  If a content-signal = no, you may not collect content for the\n#      corresponding use.\n# (c)  If the website operator does not include a content signal for a\n#      corresponding use, the website operator neither grants nor restricts\n#      permission via content signal with respect to the corresponding use.\n\n# The content signals and their meanings are:\n\n# search:   building a search index and providing search results (e.g., returning\n#           hyperlinks and short excerpts from your website's contents). Search does not\n#           include providing AI-generated search summaries.\n# ai-input: inputting content into one or more AI models (e.g., retrieval\n#           augmented generation, grounding, or other real-time taking of content for\n#           generative AI search answers).\n# ai-train: training or fine-tuning AI models.\n\n# ANY RESTRICTIONS EXPRESSED VIA CONTENT SIGNALS ARE EXPRESS RESERVATIONS OF\n# RIGHTS UNDER ARTICLE 4 OF THE EUROPEAN UNION DIRECTIVE 2019/790 ON COPYRIGHT\n# AND RELATED RIGHTS IN THE DIGITAL SINGLE MARKET.\n","agents":{"GPTBot":true,"OAI-SearchBot":true,"ChatGPT-User":true,"ClaudeBot":true,"Claude-SearchBot":true,"PerplexityBot":true,"Google-Extended":true,"Applebot-Extended":true,"CCBot":true,"Bytespider":true,"Meta-ExternalAgent":true,"Amazonbot":true}},"sitemap":{"status":404,"content_type":"text/html","bytes":1285,"sha256":"8017fa1f7764a80571cc0a99c7dbdfaa33dad3ec31f19239d56d38b0a6f421fe","final_host":"one.one.one.one","ms":32,"url":"https://cloudflare-dns.com/sitemap.xml","declared":false,"xml":false},"files":{"llms_txt":{"status":404,"content_type":"text/html","bytes":1285,"sha256":"8017fa1f7764a80571cc0a99c7dbdfaa33dad3ec31f19239d56d38b0a6f421fe","final_host":"one.one.one.one","ms":31,"path":"/llms.txt","html":true,"json":null,"present":false},"agents_md":{"status":404,"content_type":"text/html","bytes":1285,"sha256":"8017fa1f7764a80571cc0a99c7dbdfaa33dad3ec31f19239d56d38b0a6f421fe","final_host":"one.one.one.one","ms":26,"path":"/agents.md","html":true,"json":null,"present":false},"agent_skills":{"status":404,"content_type":"text/html","bytes":1285,"sha256":"8017fa1f7764a80571cc0a99c7dbdfaa33dad3ec31f19239d56d38b0a6f421fe","final_host":"one.one.one.one","ms":33,"path":"/.well-known/agent-skills/index.json","html":true,"json":null,"present":false},"mcp_card":{"status":404,"content_type":"text/html","bytes":1285,"sha256":"8017fa1f7764a80571cc0a99c7dbdfaa33dad3ec31f19239d56d38b0a6f421fe","final_host":"one.one.one.one","ms":35,"path":"/.well-known/mcp/server-card.json","html":true,"json":null,"present":false},"api_catalog":{"status":404,"content_type":"text/html","bytes":1285,"sha256":"8017fa1f7764a80571cc0a99c7dbdfaa33dad3ec31f19239d56d38b0a6f421fe","final_host":"one.one.one.one","ms":31,"path":"/.well-known/api-catalog","html":true,"json":null,"present":false},"a2a_card":{"status":404,"content_type":"text/html","bytes":1285,"sha256":"8017fa1f7764a80571cc0a99c7dbdfaa33dad3ec31f19239d56d38b0a6f421fe","final_host":"one.one.one.one","ms":33,"path":"/.well-known/agent-card.json","html":true,"json":null,"present":false}},"rings":{"v":1,"candidates":[],"mcp":[],"a2a":null,"webmcp":{"model_context":false,"bridge":false,"mcp_json":false,"link_rel":false,"any":false},"handshakes":0,"get_path":{"status":404,"content_type":"text/html","ms":27,"error":null},"get_host":{"status":404,"content_type":"text/html","ms":25,"error":null}},"derived":{"reachable":true,"robots_served":true,"robots_sitemap":false,"content_signal":false,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":[],"blocks_any_ai":false,"files":{"llms_txt":false,"agents_md":false,"agent_skills":false,"mcp_card":false,"api_catalog":false,"a2a_card":false},"any_capability":false,"ai_refused":[],"ai_thin":[],"rings":{"mcp_declared":false,"mcp_live":false,"mcp_auth":false,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":2,"fh":"one.one.one.one","fp_other_host":[],"fp":"1d5b1d00036cdd02","fp_bits":{"gptbot":0,"claudebot":0,"oaisearch":0,"perplexity":0}},"derive_v":3,"frame":"top1k","shard":4},{"v":1,"kind":"crawlcheck-census-record","run":14,"domain":"opera.com","rank":86,"measured_at":"2026-10-15T05:31:25.738Z","home":{"browser":{"status":200,"content_type":"text/html","bytes":193947,"sha256":"ba60e8153eeef65a9aae5d69b687c408e48f778f6fb5c0cd26564d3e10d117f3","final_host":"www.opera.com","ms":542,"identity":"Mobile browser","html":true,"fp":"aa3601ca47315efc"},"gptbot":{"status":200,"content_type":"text/html","bytes":190248,"sha256":"b056838f15cc957a3b4f96ae823890b18b7896ae4bee5af707f2db29561cba82","final_host":"www.opera.com","ms":536,"identity":"GPTBot","html":true,"fp":"eab601ca55117efc"},"claudebot":{"status":200,"content_type":"text/html","bytes":190248,"sha256":"b056838f15cc957a3b4f96ae823890b18b7896ae4bee5af707f2db29561cba82","final_host":"www.opera.com","ms":542,"identity":"ClaudeBot","html":true,"fp":"eab601ca55117efc"},"oaisearch":{"status":200,"content_type":"text/html","bytes":190248,"sha256":"b056838f15cc957a3b4f96ae823890b18b7896ae4bee5af707f2db29561cba82","final_host":"www.opera.com","ms":542,"identity":"OAI-SearchBot","html":true,"fp":"eab601ca55117efc"},"perplexity":{"status":200,"content_type":"text/html","bytes":190224,"sha256":"898c33608026c7f3f14f75d1598d1b38e0da2dcfd6ebd503a9bb7545a2817513","final_host":"www.opera.com","ms":542,"identity":"PerplexityBot","html":true,"fp":"eab601ca55117efc"}},"robots":{"status":200,"content_type":"text/plain","bytes":994,"sha256":"0c6b878d90d9783f0a0972e38888f8dfef8393e317be80a266a82f3da548f20f","final_host":"www.opera.com","ms":758,"html":false,"text":"User-agent: *\nAllow: /help/tutorials/\nAllow: /browser/tutorials/\nAllow: /support/tutorials/\nAllow: /lp/br-themes\nAllow: /pt-br/lp/br-themes\nAllow: /lp/video-streaming\nAllow: /*/lp/video-streaming\nDisallow: /o/\nDisallow: /*/o/\nDisallow: /abtest/\nDisallow: /*/abtest/\nDisallow: /api/\nDisallow: /client/\nDisallow: /downloadassets/\nDisallow: /*/downloadassets/\nDisallow: /download/get*\nDisallow: /get$\nDisallow: /get/\nDisallow: /promoassets/\nDisallow: /*/promoassets/\nDisallow: /campaign/\nDisallow: /*/campaign/\nDisallow: /suite/\nDisallow: /*/suite/\nDisallow: /info/\nDisallow: /*/info/\nDisallow: /computer/thanks\nDisallow: /*/computer/thanks\nDisallow: /computer/thanks-partner\nDisallow: /*/computer/thanks-partner\nDisallow: /ofa/\nDisallow: /*/ofa/\nDisallow: /oma/\nDisallow: /*/oma/\nDisallow: */tlp/*\nDisallow: */lp/*\nDisallow: /tutorials\nDisallow: /*/tutorials\nDisallow: /opera/features\nDisallow: /*/opera/features\nDisallow: /opera-coupons-uninstall\nSitemap: https://www.opera.com/sitemap-index.xml","agents":{"GPTBot":true,"OAI-SearchBot":true,"ChatGPT-User":true,"ClaudeBot":true,"Claude-SearchBot":true,"PerplexityBot":true,"Google-Extended":true,"Applebot-Extended":true,"CCBot":true,"Bytespider":true,"Meta-ExternalAgent":true,"Amazonbot":true}},"sitemap":{"status":200,"content_type":"application/xml","bytes":314,"sha256":"8eb2cdb83e21bab426f66f7b493ba60caa5b818a2c1a4f6e8b6fbbc1992dc718","final_host":"www.opera.com","ms":32,"url":"https://www.opera.com/sitemap-index.xml","declared":true,"xml":true},"files":{"llms_txt":{"status":200,"content_type":"text/plain","bytes":11613,"sha256":"abefbde9fb5c1ef40e063c0d21aeff38c71e1f9899b3b7cc8d924a89deffb013","final_host":"www.opera.com","ms":61,"path":"/llms.txt","html":false,"json":null,"present":true},"agents_md":{"status":404,"content_type":"text/html","bytes":119803,"sha256":"5845dba5419cfbf2b760b15d2960fa140fed90beeee7be84800a48fe6f6469b2","final_host":"www.opera.com","ms":104,"path":"/agents.md","html":true,"json":null,"present":false},"agent_skills":{"status":404,"content_type":"text/html","bytes":119850,"sha256":"a46356598a4436f7ad825e2f02c6e46b0704453c0c84e20b9da28c09ebc58d3c","final_host":"www.opera.com","ms":97,"path":"/.well-known/agent-skills/index.json","html":true,"json":null,"present":false},"mcp_card":{"status":404,"content_type":"text/html","bytes":115901,"sha256":"df9d6784f4f6120d2d3cc8786b60b1849f5677dcb355f5933abf14c22591ebb1","final_host":"www.opera.com","ms":57,"path":"/.well-known/mcp/server-card.json","html":true,"json":null,"present":false},"api_catalog":{"status":404,"content_type":"text/html","bytes":119831,"sha256":"d9896c878b5e163d3eec7fb0bfa111fe70b25ca8c389c5645d1875c4cd0b36ec","final_host":"www.opera.com","ms":120,"path":"/.well-known/api-catalog","html":true,"json":null,"present":false},"a2a_card":{"status":404,"content_type":"text/html","bytes":119834,"sha256":"e8e86d1fa2dc5554f491016b8407fdc6be80948300bf7c4785d795da29cd179b","final_host":"www.opera.com","ms":59,"path":"/.well-known/agent-card.json","html":true,"json":null,"present":false}},"rings":{"v":1,"candidates":[],"mcp":[],"a2a":null,"webmcp":{"model_context":false,"bridge":false,"mcp_json":false,"link_rel":false,"any":false},"handshakes":0,"get_path":{"status":404,"content_type":"text/html","ms":111,"error":null},"get_host":{"status":530,"content_type":"text/plain","ms":156,"error":null}},"derived":{"reachable":true,"robots_served":true,"robots_sitemap":true,"content_signal":false,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":[],"blocks_any_ai":false,"files":{"llms_txt":true,"agents_md":false,"agent_skills":false,"mcp_card":false,"api_catalog":false,"a2a_card":false},"any_capability":false,"ai_refused":[],"ai_thin":[],"rings":{"mcp_declared":false,"mcp_live":false,"mcp_auth":false,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":2,"fh":"www.opera.com","fp_other_host":[],"fp":"aa3601ca47315efc","fp_bits":{"gptbot":6,"claudebot":6,"oaisearch":6,"perplexity":6}},"derive_v":3,"frame":"top1k","shard":5},{"v":1,"kind":"crawlcheck-census-record","run":14,"domain":"vimeo.com","rank":87,"measured_at":"2026-10-15T05:33:24.843Z","home":{"browser":{"status":200,"content_type":"text/html","bytes":1749264,"sha256":"5206ed970ec02f63b66c3c43a11a7c2d1571da3cab6adea6a38844c423b6be7f","final_host":"vimeo.com","ms":135,"identity":"Mobile browser","html":true,"fp":"e681ff7cadc4b34a"},"gptbot":{"status":200,"content_type":"text/html","bytes":1749264,"sha256":"5206ed970ec02f63b66c3c43a11a7c2d1571da3cab6adea6a38844c423b6be7f","final_host":"vimeo.com","ms":135,"identity":"GPTBot","html":true,"fp":"e681ff7cadc4b34a"},"claudebot":{"status":200,"content_type":"text/html","bytes":1749264,"sha256":"5206ed970ec02f63b66c3c43a11a7c2d1571da3cab6adea6a38844c423b6be7f","final_host":"vimeo.com","ms":135,"identity":"ClaudeBot","html":true,"fp":"e681ff7cadc4b34a"},"oaisearch":{"status":200,"content_type":"text/html","bytes":1749264,"sha256":"5206ed970ec02f63b66c3c43a11a7c2d1571da3cab6adea6a38844c423b6be7f","final_host":"vimeo.com","ms":135,"identity":"OAI-SearchBot","html":true,"fp":"e681ff7cadc4b34a"},"perplexity":{"status":200,"content_type":"text/html","bytes":1749264,"sha256":"5206ed970ec02f63b66c3c43a11a7c2d1571da3cab6adea6a38844c423b6be7f","final_host":"vimeo.com","ms":25,"identity":"PerplexityBot","html":true,"fp":"e681ff7cadc4b34a"}},"robots":{"status":200,"content_type":"text/plain","bytes":22615,"sha256":"021ea0af6b29208b5ad678715c9e6671373ae3a937b8e879a366e5f497f88b10","final_host":"vimeo.com","ms":135,"html":false,"text":"# Notice: The use of robots, scrapers, or other automated means to access Vimeo \n# without the express written permission of Vimeo is strictly prohibited.\n# See https://vimeo.com/terms#acceptable_use_policy\n# Vimeo permits limited automated access to certain public-facing pages by approved services,\n# including search engines and select AI assistants, in accordance with this robots.txt file.\n# Access to user-generated content, including videos, user profiles, comments, and metadata, \n# is not permitted for scraping, harvesting, or use in training machine learning models.\n# For more on Vimeo's position on AI and content usage, see https://vimeo.com/blog/post/vimeos-position-on-ai\n\nUser-agent: *\nDisallow: /*/format:thumbnail\nDisallow: /download/\nDisallow: /*/download?\nDisallow: /couchmode/\nDisallow: /groups/*/forum/*\nDisallow: /musicstore/preview\nDisallow: /musicstore/download\nDisallow: /forgot_password\nDisallow: /help/forgot_login\nDisallow: /help/verify_email\nDisallow: /help/zendesk_sso\nDisallow: /help/sso\nDisallow: /user/verify\nDisallow: /search/\nDisallow: /search?\nDisallow: /search\\?\nDisallow: /_\nDisallow: /*mkc=*&\nDisallow: /*&mkc=*\nDisallow: /*gl=*&\nDisallow: /*&gl=*\nDisallow: /_global_notifications\nDisallow: /*?*referrer_vuid=\nDisallow: /*?*vimeo_search\nDisallow: /store/\nDisallow: /*?*_gl=\nDisallow: /*/collections/channels\nDisallow: /manage/videos/\nDisallow: /flarepoint/\nDisallow: /*/likes \nDisallow: /reviews/\nDisallow: /*/sort:*/*:\nDisallow: /ablincoln/\nDisallow: /*tag:\nDisallow: /*/videos/\nDisallow: /legal/~gitbook/\nDisallow: /manage/showcase\nAllow: /api/*/likes \nAllow: /_rv/viewer\nAllow: /_next/viewer\nAllow: /_rv/gtm\nAllow: /_rv/jwt\nAllow: /_next/jwt\nAllow: /_rv/title\nAllow: /_next/title\n# Clip Page Parameters\nDisallow: /*?_=\nDisallow: /*?_hsenc=\nDisallow: /*?_hsmi=\nDisallow: /*?*&_=\nDisallow: /*?*&_hsenc=\nDisallow: /*?*&_hsmi=\nDisallow: /*?*&1=\nDisallow: /*?*&activityReferer=\nDisallow: /*?*&adcPathname=\nDisallow: /*?*&ajax=\nDisallow: /*?*&allowfullscreen=\nDisallow: /*?*&autohide=\nDisallow: /*?*&autopause=\nDisallow: /*?*&background=\nDisallow: /*?*&badge=\nDisallow: /*?*&brid=\nDisallow: /*?*&byline=\nDisallow: /*?*&callback=\nDisallow: /*?*&cb=\nDisallow: /*?*&cc_load_policy=\nDisallow: /*?*&ceid=\nDisallow: /*?*&cjdata=\nDisallow: /*?*&cjevent=\nDisallow: /*?*&clickid=\nDisallow: /*?*&clipId=\nDisallow: /*?*&color=\nDisallow: /*?*&consentEnabled=\nDisallow: /*?*&content=\nDisallow: /*?*&context=\nDisallow: /*?*&controls=\nDisallow: /*?*&d=\nDisallow: /*?*&data=\nDisallow: /*?*&dnt=\nDisallow: /*?*&ei=\nDisallow: /*?*&ej=\nDisallow: /*?*&embed_email_provider=\nDisallow: /*?*&emci=\nDisallow: /*?*&emdi=\nDisallow: /*?*&enablejsapi=\nDisallow: /*?*&f1=\nDisallow: /*?*&filter_location=\nDisallow: /*?*&filter_skills=\nDisallow: /*?*&fit=\nDisallow: /*?*&flashvars=\nDisallow: /*?*&foe=\nDisallow: /*?*&format=\nDisallow: /*?*&from=\nDisallow: /*?*&fromApp=\nDisallow: /*?*&fullscreen=\nDisallow: /*?*&h=\nDisallow: /*?*&hashKey=\nDisallow: /*?*&hd=\nDisallow: /*?*&height=\nDisallow: /*?*&hss_meta=\nDisallow: /*?*&id=\nDisallow: /*?*&iframe=\nDisallow: /*?*&itok=\nDisallow: /*?*&keepEmbedCode=\nDisallow: /*?*&keyboard=\nDisallow: /*?1=\nDisallow: /*?activityReferer=\nDisallow: /*?adcPathname=\nDisallow: /*?ajax=\nDisallow: /*?allowfullscreen=\nDisallow: /*?autohide=\nDisallow: /*?autopause=\nDisallow: /*?background=\nDisallow: /*?badge=\nDisallow: /*?brid=\nDisallow: /*?byline=\nDisallow: /*?callback=\nDisallow: /*?cb=\nDisallow: /*?cc_load_policy=\nDisallow: /*?ceid=\nDisallow: /*?cjdata=\nDisallow: /*?cjevent=\nDisallow: /*?clickid=\nDisallow: /*?clipId=\nDisallow: /*?color=\nDisallow: /*?consentEnabled=\nDisallow: /*?content=\nDisallow: /*?context=\nDisallow: /*?controls=\nDisallow: /*?d=\nDisallow: /*?data=\nDisallow: /*?dnt=\nDisallow: /*?ei=\nDisallow: /*?ej=\nDisallow: /*?embed_email_provider=\nDisallow: /*?emci=\nDisallow: /*?emdi=\nDisallow: /*?enablejsapi=\nDisallow: /*?f1=\nDisallow: /*?filter_location=\nDisallow: /*?filter_skills=\nDisallow: /*?fit=\nDisallow: /*?flashvars=\nDisallow: /*?foe=\nDisallow: /*?format=\nDisallow: /*?from=\nDisallow: /*?fromApp=\nDisallow: /*?fullscreen=\nDisallow: /*?h=\nDisallow: /*?hashKey=\nDisallow: /*?hd=\nDisallow: /*?height=\nDisallow: /*?hss_meta=\nDisallow: /*?id=\nDisallow: /*?iframe=\nDisallow: /*?itok=\nDisallow: /*?keepEmbedCode=\nDisallow: /*?keyboard=\nDisallow: /*?*&%3Bfe=\nDisallow: /*?*&%3Bfl=\nDisallow: /*?*&kref=\nDisallow: /*?*&kuid=\nDisallow: /*?*&lang=\nDisallow: /*?*&lightbox[modal]=\nDisallow: /*?*&linkId=\nDisallow: /*?*&lite=\nDisallow: /*?*&login=\nDisallow: /*?*&loop=\nDisallow: /*?*&maxheight=\nDisallow: /*?*&maxw=\nDisallow: /*?*&maxwidth=\nDisallow: /*?*&mc_cid=\nDisallow: /*?*&method=\nDisallow: /*?*&mibextid=\nDisallow: /*?*&modestbranding=\nDisallow: /*?*&ms=\nDisallow: /*?*&msockid=\nDisallow: /*?*&mute=\nDisallow: /*?*&odnBg=\nDisallow: /*?*&odnHeight=\nDisallow: /*?*&odnWidth=\nDisallow: /*?*&output=\nDisallow: /*?*&outro=\nDisallow: /*?*&per_page=\nDisallow: /*?*&pgroup=\nDisallow: /*?*&platform=\nDisallow: /*?*&play=\nDisallow: /*?*&player_id=\nDisallow: /*?*&playerapiid=\nDisallow: /*?*&playlist=\nDisallow: /*?*&playsinline=\nDisallow: /*?*&portrait=\nDisallow: /*?*&progress_bar=\nDisallow: /*?*&quality_selector=\nDisallow: /*?*&quality=\nDisallow: /*?*&ra=\nDisallow: /*?*&random=\nDisallow: /*?*&redirect=\nDisallow: /*?*&rel=\nDisallow: /*?*&resize=\nDisallow: /*?*&responsive=\nDisallow: /*?*&rmxv=\nDisallow: /*?*&rt=\nDisallow: /*?*&rv=\nDisallow: /*?*&s=\nDisallow: /*?*&sa=\nDisallow: /*?*&scrlybrkr=\nDisallow: /*?*&sd=\nDisallow: /*?*&server=\nDisallow: /*?*&sfmc_id=\nDisallow: /*?*&show_byline=\nDisallow: /*?*&show_portrait=\nDisallow: /*?*&show_title=\nDisallow: /*?*&showinfo=\nDisallow: /*?*&sig=\nDisallow: /*?*&signup=\nDisallow: /*?*&sizes=\nDisallow: /*?*&sl_key=\nDisallow: /*?*&sort=\nDisallow: /*?*&speed=\nDisallow: /*?*&sr=\nDisallow: /*?*&ssl=\nDisallow: /*?*&ssp_iabi=\nDisallow: /*?*&start=\nDisallow: /*?*&streamid=\nDisallow: /*?*&sv=\nDisallow: /*?*&syclid=\nDisallow: /*?*&tblci=\nDisallow: /*?*&test=\nDisallow: /*?*&texttrack=\nDisallow: /*?*&title=\nDisallow: /*?*&tq=\nDisallow: /*?*&transparent=\nDisallow: /*?*&trk=\nDisallow: /*?*&type=\nDisallow: /*?*&url=\nDisallow: /*?*&user_id=\nDisallow: /*?*&usg=\nDisallow: /*?*&utm_date=\nDisallow: /*?*&utm_id=\nDisallow: /*?*&v=\nDisallow: /*?*&version=\nDisallow: /*?*&volumeOff=\nDisallow: /*?*&vq=\nDisallow: /*?*&w=\nDisallow: /*?*&ysclid=\nDisallow: /*?%3Bfe=\nDisallow: /*?%3Bfl=\nDisallow: /*?kref=\nDisallow: /*?kuid=\nDisallow: /*?lang=\nDisallow: /*?lightbox[modal]=\nDisallow: /*?linkId=\nDisallow: /*?lite=\nDisallow: /*?login=\nDisallow: /*?loop=\nDisallow: /*?maxheight=\nDisallow: /*?maxw=\nDisallow: /*?maxwidth=\nDisallow: /*?mc_cid=\nDisallow: /*?method=\nDisallow: /*?mibextid=\nDisallow: /*?modestbranding=\nDisallow: /*?ms=\nDisallow: /*?msockid=\nDisallow: /*?mute=\nDisallow: /*?odnBg=\nDisallow: /*?odnHeight=\nDisallow: /*?odnWidth=\nDisallow: /*?output=\nDisallow: /*?outro=\nDisallow: /*?per_page=\nDisallow: /*?pgroup=\nDisallow: /*?platform=\nDisallow: /*?play=\nDisallow: /*?player_id=\nDisallow: /*?playerapiid=\nDisallow: /*?playlist=\nDisallow: /*?playsinline=\nDisallow: /*?portrait=\nDisallow: /*?progress_bar=\nDisallow: /*?quality_selector=\nDisallow: /*?quality=\nDisallow: /*?ra=\nDisallow: /*?random=\nDisallow: /*?redirect=\nDisallow: /*?rel=\nDisallow: /*?resize=\nDisallow: /*?responsive=\nDisallow: /*?rmxv=\nDisallow: /*?rt=\nDisallow: /*?rv=\nDisallow: /*?s=\nDisallow: /*?sa=\nDisallow: /*?scrlybrkr=\nDisallow: /*?sd=\nDisallow: /*?server=\nDisallow: /*?sfmc_id=\nDisallow: /*?show_byline=\nDisallow: /*?show_portrait=\nDisallow: /*?show_title=\nDisallow: /*?showinfo=\nDisallow: /*?sig=\nDisallow: /*?signup=\nDisallow: /*?sizes=\nDisallow: /*?sl_key=\nDisallow: /*?sort=\nDisallow: /*?speed=\nDisallow: /*?sr=\nDisallow: /*?ssl=\nDisallow: /*?ssp_iabi=\nDisallow: /*?start=\nDisallow: /*?streamid=\nDisallow: /*?sv=\nDisallow: /*?syclid=\nDisallow: /*?tblci=\nDisallow: /*?test=\nDisallow: /*?texttrack=\nDisallow: /*?title=\nDisallow: /*?tq=\nDisallow: /*?transparent=\nDisallow: /*?trk=\nDisallow: /*?type=\nDisallow: /*?url=\nDisallow: /*?user_id=\nDisallow: /*?usg=\nDisallow: /*?utm_date=\nDisallow: /*?utm_id=\nDisallow: /*?v=\nDisallow: /*?version=\nDisallow: /*?volumeOff=\nDisallow: /*?vq=\nDisallow: /*?w=\nDisallow: /*?ysclid=\nAllow: /*/*?_=\nAllow: /*/*?_hsenc=\nAllow: /*/*?_hsmi=\nAllow: /*/*?*&_=\nAllow: /*/*?*&_hsenc=\nAllow: /*/*?*&_hsmi=\nAllow: /*/*?*&1=\nAllow: /*/*?*&activityReferer=\nAllow: /*/*?*&adcPathname=\nAllow: /*/*?*&ajax=\nAllow: /*/*?*&allowfullscreen=\nAllow: /*/*?*&autohide=\nAllow: /*/*?*&autopause=\nAllow: /*/*?*&background=\nAllow: /*/*?*&badge=\nAllow: /*/*?*&brid=\nAllow: /*/*?*&byline=\nAllow: /*/*?*&callback=\nAllow: /*/*?*&cb=\nAllow: /*/*?*&cc_load_policy=\nAllow: /*/*?*&ceid=\nAllow: /*/*?*&cjdata=\nAllow: /*/*?*&cjevent=\nAllow: /*/*?*&clickid=\nAllow: /*/*?*&clipId=\nAllow: /*/*?*&color=\nAllow: /*/*?*&consentEnabled=\nAllow: /*/*?*&content=\nAllow: /*/*?*&context=\nAllow: /*/*?*&controls=\nAllow: /*/*?*&d=\nAllow: /*/*?*&data=\nAllow: /*/*?*&dnt=\nAllow: /*/*?*&ei=\nAllow: /*/*?*&ej=\nAllow: /*/*?*&embed_email_provider=\nAllow: /*/*?*&emci=\nAllow: /*/*?*&emdi=\nAllow: /*/*?*&enablejsapi=\nAllow: /*/*?*&f1=\nAllow: /*/*?*&filter_location=\nAllow: /*/*?*&filter_skills=\nAllow: /*/*?*&fit=\nAllow: /*/*?*&flashvars=\nAllow: /*/*?*&foe=\nAllow: /*/*?*&format=\nAllow: /*/*?*&from=\nAllow: /*/*?*&fromApp=\nAllow: /*/*?*&fullscreen=\nAllow: /*/*?*&h=\nAllow: /*/*?*&hashKey=\nAllow: /*/*?*&hd=\nAllow: /*/*?*&height=\nAllow: /*/*?*&hss_meta=\nAllow: /*/*?*&id=\nAllow: /*/*?*&iframe=\nAllow: /*/*?*&itok=\nAllow: /*/*?*&keepEmbedCode=\nAllow: /*/*?*&keyboard=\nAllow: /*/*?1=\nAllow: /*/*?activityReferer=\nAllow: /*/*?adcPathname=\nAllow: /*/*?ajax=\nAllow: /*/*?allowfullscreen=\nAllow: /*/*?autohide=\nAllow: /*/*?autopause=\nAllow: /*/*?background=\nAllow: /*/*?badge=\nAllow: /*/*?brid=\nAllow: /*/*?byline=\nAllow: /*/*?callback=\nAllow: /*/*?cb=\nAllow: /*/*?cc_load_policy=\nAllow: /*/*?ceid=\nAllow: /*/*?cjdata=\nAllow: /*/*?cjevent=\nAllow: /*/*?clickid=\nAllow: /*/*?clipId=\nAllow: /*/*?color=\nAllow: /*/*?consentEnabled=\nAllow: /*/*?content=\nAllow: /*/*?context=\nAllow: /*/*?controls=\nAllow: /*/*?d=\nAllow: /*/*?data=\nAllow: /*/*?dnt=\nAllow: /*/*?ei=\nAllow: /*/*?ej=\nAllow: /*/*?embed_email_provider=\nAllow: /*/*?emci=\nAllow: /*/*?emdi=\nAllow: /*/*?enablejsapi=\nAllow: /*/*?f1=\nAllow: /*/*?filter_location=\nAllow: /*/*?filter_skills=\nAllow: /*/*?fit=\nAllow: /*/*?flashvars=\nAllow: /*/*?foe=\nAllow: /*/*?format=\nAllow: /*/*?from=\nAllow: /*/*?fromApp=\nAllow: /*/*?fullscreen=\nAllow: /*/*?h=\nAllow: /*/*?hashKey=\nAllow: /*/*?hd=\nAllow: /*/*?height=\nAllow: /*/*?hss_meta=\nAllow: /*/*?id=\nAllow: /*/*?iframe=\nAllow: /*/*?itok=\nAllow: /*/*?keepEmbedCode=\nAllow: /*/*?keyboard=\nAllow: /*/*?*&%3Bfe=\nAllow: /*/*?*&%3Bfl=\nAllow: /*/*?*&kref=\nAllow: /*/*?*&kuid=\nAllow: /*/*?*&lang=\nAllow: /*/*?*&lightbox[modal]=\nAllow: /*/*?*&linkId=\nAllow: /*/*?*&lite=\nAllow: /*/*?*&login=\nAllow: /*/*?*&loop=\nAllow: /*/*?*&maxheight=\nAllow: /*/*?*&maxw=\nAllow: /*/*?*&maxwidth=\nAllow: /*/*?*&mc_cid=\nAllow: /*/*?*&method=\nAllow: /*/*?*&mibextid=\nAllow: /*/*?*&modestbranding=\nAllow: /*/*?*&ms=\nAllow: /*/*?*&msockid=\nAllow: /*/*?*&mute=\nAllow: /*/*?*&odnBg=\nAllow: /*/*?*&odnHeight=\nAllow: /*/*?*&odnWidth=\nAllow: /*/*?*&output=\nAllow: /*/*?*&outro=\nAllow: /*/*?*&per_page=\nAllow: /*/*?*&pgroup=\nAllow: /*/*?*&platform=\nAllow: /*/*?*&play=\nAllow: /*/*?*&player_id=\nAllow: /*/*?*&playerapiid=\nAllow: /*/*?*&playlist=\nAllow: /*/*?*&playsinline=\nAllow: /*/*?*&portrait=\nAllow: /*/*?*&progress_bar=\nAllow: /*/*?*&quality_selector=\nAllow: /*/*?*&quality=\nAllow: /*/*?*&ra=\nAllow: /*/*?*&random=\nAllow: /*/*?*&redirect=\nAllow: /*/*?*&rel=\nAllow: /*/*?*&resize=\nAllow: /*/*?*&responsive=\nAllow: /*/*?*&rmxv=\nAllow: /*/*?*&rt=\nAllow: /*/*?*&rv=\nAllow: /*/*?*&s=\nAllow: /*/*?*&sa=\nAllow: /*/*?*&scrlybrkr=\nAllow: /*/*?*&sd=\nAllow: /*/*?*&server=\nAllow: /*/*?*&sfmc_id=\nAllow: /*/*?*&show_byline=\nAllow: /*/*?*&show_portrait=\nAllow: /*/*?*&show_title=\nAllow: /*/*?*&showinfo=\nAllow: /*/*?*&sig=\nAllow: /*/*?*&signup=\nAllow: /*/*?*&sizes=\nAllow: /*/*?*&sl_key=\nAllow: /*/*?*&sort=\nAllow: /*/*?*&speed=\nAllow: /*/*?*&sr=\nAllow: /*/*?*&ssl=\nAllow: /*/*?*&ssp_iabi=\nAllow: /*/*?*&start=\nAllow: /*/*?*&streamid=\nAllow: /*/*?*&sv=\nAllow: /*/*?*&syclid=\nAllow: /*/*?*&tblci=\nAllow: /*/*?*&test=\nAllow: /*/*?*&texttrack=\nAllow: /*/*?*&title=\nAllow: /*/*?*&tq=\nAllow: /*/*?*&transparent=\nAllow: /*/*?*&trk=\nAllow: /*/*?*&type=\nAllow: /*/*?*&url=\nAllow: /*/*?*&user_id=\nAllow: /*/*?*&usg=\nAllow: /*/*?*&utm_date=\nAllow: /*/*?*&utm_id=\nAllow: /*/*?*&v=\nAllow: /*/*?*&version=\nAllow: /*/*?*&volumeOff=\nAllow: /*/*?*&vq=\nAllow: /*/*?*&w=\nAllow: /*/*?*&ysclid=\nAllow: /*/*?%3Bfe=\nAllow: /*/*?%3Bfl=\nAllow: /*/*?kref=\nAllow: /*/*?kuid=\nAllow: /*/*?lang=\nAllow: /*/*?lightbox[modal]=\nAllow: /*/*?linkId=\nAllow: /*/*?lite=\nAllow: /*/*?login=\nAllow: /*/*?loop=\nAllow: /*/*?maxheight=\nAllow: /*/*?maxw=\nAllow: /*/*?maxwidth=\nAllow: /*/*?mc_cid=\nAllow: /*/*?method=\nAllow: /*/*?mibextid=\nAllow: /*/*?modestbranding=\nAllow: /*/*?ms=\nAllow: /*/*?msockid=\nAllow: /*/*?mute=\nAllow: /*/*?odnBg=\nAllow: /*/*?odnHeight=\nAllow: /*/*?odnWidth=\nAllow: /*/*?output=\nAllow: /*/*?outro=\nAllow: /*/*?per_page=\nAllow: /*/*?pgroup=\nAllow: /*/*?platform=\nAllow: /*/*?play=\nAllow: /*/*?player_id=\nAllow: /*/*?playerapiid=\nAllow: /*/*?playlist=\nAllow: /*/*?playsinline=\nAllow: /*/*?portrait=\nAllow: /*/*?progress_bar=\nAllow: /*/*?quality_selector=\nAllow: /*/*?quality=\nAllow: /*/*?ra=\nAllow: /*/*?random=\nAllow: /*/*?redirect=\nAllow: /*/*?rel=\nAllow: /*/*?resize=\nAllow: /*/*?responsive=\nAllow: /*/*?rmxv=\nAllow: /*/*?rt=\nAllow: /*/*?rv=\nAllow: /*/*?s=\nAllow: /*/*?sa=\nAllow: /*/*?scrlybrkr=\nAllow: /*/*?sd=\nAllow: /*/*?server=\nAllow: /*/*?sfmc_id=\nAllow: /*/*?show_byline=\nAllow: /*/*?show_portrait=\nAllow: /*/*?show_title=\nAllow: /*/*?showinfo=\nAllow: /*/*?sig=\nAllow: /*/*?signup=\nAllow: /*/*?sizes=\nAllow: /*/*?sl_key=\nAllow: /*/*?sort=\nAllow: /*/*?speed=\nAllow: /*/*?sr=\nAllow: /*/*?ssl=\nAllow: /*/*?ssp_iabi=\nAllow: /*/*?start=\nAllow: /*/*?streamid=\nAllow: /*/*?sv=\nAllow: /*/*?syclid=\nAllow: /*/*?tblci=\nAllow: /*/*?test=\nAllow: /*/*?texttrack=\nAllow: /*/*?title=\nAllow: /*/*?tq=\nAllow: /*/*?transparent=\nAllow: /*/*?trk=\nAllow: /*/*?type=\nAllow: /*/*?url=\nAllow: /*/*?user_id=\nAllow: /*/*?usg=\nAllow: /*/*?utm_date=\nAllow: /*/*?utm_id=\nAllow: /*/*?v=\nAllow: /*/*?version=\nAllow: /*/*?volumeOff=\nAllow: /*/*?vq=\nAllow: /*/*?w=\nAllow: /*/*?ysclid=\nCrawl-delay: 2\n\nUser-agent: SeznamBot\nRequest-rate: 1/10s\nDisallow: */format:thumbnail\nDisallow: /download/\nDisallow: /*/download?\nDisallow: /couchmode/\nDisallow: /_global_notifications\nDisallow: /musicstore/preview\nDisallow: /musicstore/download\nDisallow: /forgot_password\nDisallow: /help/forgot_login\nDisallow: /help/verify_email\nDisallow: /help/zendesk_sso\nDisallow: /help/sso\nDisallow: /api/\nDisallow: /flarepoint/\nDisallow: /user/verify\nDisallow: /search/\nDisallow: /search?\nDisallow: /search\\?\nDisallow: /_*\n\nUser-agent: msnbot\nCrawl-delay: 5\n\nUser-agent: Bingbot\nCrawl-delay: 5\n\nUser-agent: AdIdxBot\nCrawl-delay: 5\n\nUser-agent: BingPreview\nCrawl-delay: 5\n\nUser-agent: Yandex\nCrawl-delay: 2\n\n# Block Open AI - Allowing specific marketing/informational paths\nUser-agent: GPTBot\nDisallow: /\nAllow: /features/\nAllow: /solutions/\nAllow: /product/\nAllow: /video-uploader\nAllow: /upgrade-plan\nAllow: /enterprise/\nAllow: /integrations/\nAllow: /blog/\nAllow: /create/\nAllow: /ott/\nAllow: /customers/\nAllow: /events/\nAllow: /campaigns/\nAllow: /about/\nAllow: /vimeo-academy/\nAllow: /$\nAllow: /es/features/\nAllow: /es/solutions/\nAllow: /es/product/\nAllow: /es/video-uploader\nAllow: /es/enterprise/\nAllow: /es/integrations/\nAllow: /es/blog/\nAllow: /es/create/\nAllow: /es/ott/\nAllow: /es/customers/\nAllow: /es/events/\nAllow: /es/campaigns/\nAllow: /es/about/\nAllow: /es/vimeo-academy/\nAllow: /es/$\nAllow: /fr/features/\nAllow: /fr/solutions/\nAllow: /fr/product/\nAllow: /fr/video-uploader\nAllow: /fr/enterprise/\nAllow: /fr/integrations/\nAllow: /fr/blog/\nAllow: /fr/create/\nAllow: /fr/ott/\nAllow: /fr/customers/\nAllow: /fr/events/\nAllow: /fr/campaigns/\nAllow: /fr/about/\nAllow: /fr/vimeo-academy/\nAllow: /fr/$\nAllow: /de/features/\nAllow: /de/solutions/\nAllow: /de/product/\nAllow: /de/video-uploader\nAllow: /de/enterprise/\nAllow: /de/integrations/\nAllow: /de/blog/\nAllow: /de/create/\nAllow: /de/ott/\nAllow: /de/customers/\nAllow: /de/events/\nAllow: /de/campaigns/\nAllow: /de/about/\nAllow: /de/vimeo-academy/\nAllow: /de/$\nAllow: /pt-br/features/\nAllow: /pt-br/solutions/\nAllow: /pt-br/product/\nAllow: /pt-BR/video-uploader\nAllow: /pt-br/enterprise/\nAllow: /pt-br/integrations/\nAllow: /pt-br/blog/\nAllow: /pt-br/create/\nAllow: /pt-br/ott/\nAllow: /pt-br/customers/\nAllow: /pt-br/events/\nAllow: /pt-br/campaigns/\nAllow: /pt-br/about/\nAllow: /pt-br/vimeo-academy/\nAllow: /pt-br/$\nAllow: /ko/features/\nAllow: /ko/solutions/\nAllow: /ko/product/\nAllow: /ko/video-uploader\nAllow: /ko/enterprise/\nAllow: /ko/integrations/\nAllow: /ko/blog/\nAllow: /ko/create/\nAllow: /ko/ott/\nAllow: /ko/customers/\nAllow: /ko/events/\nAllow: /ko/campaigns/\nAllow: /ko/about/\nAllow: /ko/vimeo-academy/\nAllow: /ko/$\nAllow: /jp/features/\nAllow: /jp/solutions/\nAllow: /jp/product/\nAllow: /jp/video-uploader\nAllow: /jp/enterprise/\nAllow: /jp/integrations/\nAllow: /jp/blog/\nAllow: /jp/create/\nAllow: /jp/ott/\nAllow: /jp/customers/\nAllow: /jp/events/\nAllow: /jp/campaigns/\nAllow: /jp/about/\nAllow: /jp/vimeo-academy/\nAllow: /jp/$\n\n# Block Google (Gemini) - Allowing specific marketing/informational paths\nUser-agent: Google-Extended\nDisallow: /\nAllow: /features/\nAllow: /solutions/\nAllow: /product/\nAllow: /video-uploader\nAllow: /upgrade-plan\nAllow: /enterprise/\nAllow: /integrations/\nAllow: /blog/\nAllow: /create/\nAllow: /ott/\nAllow: /customers/\nAllow: /events/\nAllow: /campaigns/\nAllow: /about/\nAllow: /vimeo-academy/\nAllow: /$\nAllow: /es/features/\nAllow: /es/solutions/\nAllow: /es/product/\nAllow: /es/video-uploader\nAllow: /es/enterprise/\nAllow: /es/integrations/\nAllow: /es/blog/\nAllow: /es/create/\nAllow: /es/ott/\nAllow: /es/customers/\nAllow: /es/events/\nAllow: /es/campaigns/\nAllow: /es/about/\nAllow: /es/vimeo-academy/\nAllow: /es/$\nAllow: /fr/features/\nAllow: /fr/solutions/\nAllow: /fr/product/\nAllow: /fr/video-uploader\nAllow: /fr/enterprise/\nAllow: /fr/integrations/\nAllow: /fr/blog/\nAllow: /fr/create/\nAllow: /fr/ott/\nAllow: /fr/customers/\nAllow: /fr/events/\nAllow: /fr/campaigns/\nAllow: /fr/about/\nAllow: /fr/vimeo-academy/\nAllow: /fr/$\nAllow: /de/features/\nAllow: /de/solutions/\nAllow: /de/product/\nAllow: /de/video-uploader\nAllow: /de/enterprise/\nAllow: /de/integrations/\nAllow: /de/blog/\nAllow: /de/create/\nAllow: /de/ott/\nAllow: /de/customers/\nAllow: /de/events/\nAllow: /de/campaigns/\nAllow: /de/about/\nAllow: /de/vimeo-academy/\nAllow: /de/$\nAllow: /pt-br/features/\nAllow: /pt-br/solutions/\nAllow: /pt-br/product/\nAllow: /pt-BR/video-uploader\nAllow: /pt-br/enterprise/\nAllow: /pt-br/integrations/\nAllow: /pt-br/blog/\nAllow: /pt-br/create/\nAllow: /pt-br/ott/\nAllow: /pt-br/customers/\nAllow: /pt-br/events/\nAllow: /pt-br/campaigns/\nAllow: /pt-br/about/\nAllow: /pt-br/vimeo-academy/\nAllow: /pt-br/$\nAllow: /ko/features/\nAllow: /ko/solutions/\nAllow: /ko/product/\nAllow: /ko/video-uploader\nAllow: /ko/enterprise/\nAllow: /ko/integrations/\nAllow: /ko/blog/\nAllow: /ko/create/\nAllow: /ko/ott/\nAllow: /ko/customers/\nAllow: /ko/events/\nAllow: /ko/campaigns/\nAllow: /ko/about/\nAllow: /ko/vimeo-academy/\nAllow: /ko/$\nAllow: /jp/features/\nAllow: /jp/solutions/\nAllow: /jp/product/\nAllow: /jp/video-uploader\nAllow: /jp/enterprise/\nAllow: /jp/integrations/\nAllow: /jp/blog/\nAllow: /jp/create/\nAllow: /jp/ott/\nAllow: /jp/customers/\nAllow: /jp/events/\nAllow: /jp/campaigns/\nAllow: /jp/about/\nAllow: /jp/vimeo-academy/\nAllow: /jp/$\n\n# Block Claude - Allowing specific marketing/informational paths\nUser-agent: anthropic-ai\nUser-agent: Claude-Web\nUser-agent: Claudebot\nDisallow: /\nAllow: /features/\nAllow: /solutions/\nAllow: /product/\nAllow: /video-uploader\nAllow: /upgrade-plan\nAllow: /enterprise/\nAllow: /integrations/\nAllow: /blog/\nAllow: /create/\nAllow: /ott/\nAllow: /customers/\nAllow: /events/\nAllow: /campaigns/\nAllow: /about/\nAllow: /vimeo-academy/\nAllow: /$\nAllow: /es/features/\nAllow: /es/solutions/\nAllow: /es/product/\nAllow: /es/video-uploader\nAllow: /es/enterprise/\nAllow: /es/integrations/\nAllow: /es/blog/\nAllow: /es/create/\nAllow: /es/ott/\nAllow: /es/customers/\nAllow: /es/events/\nAllow: /es/campaigns/\nAllow: /es/about/\nAllow: /es/vimeo-academy/\nAllow: /es/$\nAllow: /fr/features/\nAllow: /fr/solutions/\nAllow: /fr/product/\nAllow: /fr/video-uploader\nAllow: /fr/enterprise/\nAllow: /fr/integrations/\nAllow: /fr/blog/\nAllow: /fr/create/\nAllow: /fr/ott/\nAllow: /fr/customers/\nAllow: /fr/events/\nAllow: /fr/campaigns/\nAllow: /fr/about/\nAllow: /fr/vimeo-academy/\nAllow: /fr/$\nAllow: /de/features/\nAllow: /de/solutions/\nAllow: /de/product/\nAllow: /de/video-uploader\nAllow: /de/enterprise/\nAllow: /de/integrations/\nAllow: /de/blog/\nAllow: /de/create/\nAllow: /de/ott/\nAllow: /de/customers/\nAllow: /de/events/\nAllow: /de/campaigns/\nAllow: /de/about/\nAllow: /de/vimeo-academy/\nAllow: /de/$\nAllow: /pt-br/features/\nAllow: /pt-br/solutions/\nAllow: /pt-br/product/\nAllow: /pt-BR/video-uploader\nAllow: /pt-br/enterprise/\nAllow: /pt-br/integrations/\nAllow: /pt-br/blog/\nAllow: /pt-br/create/\nAllow: /pt-br/ott/\nAllow: /pt-br/customers/\nAllow: /pt-br/events/\nAllow: /pt-br/campaigns/\nAllow: /pt-br/about/\nAllow: /pt-br/vimeo-academy/\nAllow: /pt-br/$\nAllow: /ko/features/\nAllow: /ko/solutions/\nAllow: /ko/product/\nAllow: /ko/video-uploader\nAllow: /ko/enterprise/\nAllow: /ko/integrations/\nAllow: /ko/blog/\nAllow: /ko/create/\nAllow: /ko/ott/\nAllow: /ko/customers/\nAllow: /ko/events/\nAllow: /ko/campaigns/\nAllow: /ko/about/\nAllow: /ko/vimeo-academy/\nAllow: /ko/$\nAllow: /jp/features/\nAllow: /jp/solutions/\nAllow: /jp/product/\nAllow: /jp/video-uploader\nAllow: /jp/enterprise/\nAllow: /jp/integrations/\nAllow: /jp/blog/\nAllow: /jp/create/\nAllow: /jp/ott/\nAllow: /jp/customers/\nAllow: /jp/events/\nAllow: /jp/campaigns/\nAllow: /jp/about/\nAllow: /jp/vimeo-academy/\nAllow: /jp/$\n\n# Block CommonCrawl\nUser-agent: CCBot\nUser-agent: CC-Crawler\nUser-agent: Common Crawl\nUser-agent: CommonCrawl\nDisallow: /\n\n# Block Diffbot\nUser-agent: Diffbot\nDisallow: /\n\n# Block Meta (Facebook)\nUser-agent: FacebookBot\nDisallow: /\n\n# Block ByteDance\nUser-agent: Bytespider\nDisallow: /\n\n# Block Webz.io\nUser-agent: Omgilibot\nDisallow: /\nUser-agent: Omgili\nDisallow: /\n\n# Block ImagesiftBot\nUser-agent: ImagesiftBot\nDisallow: /\n\n# Block Meltwater\nUser-agent: Meltwater\nDisallow: /\n\n# Block Seekr\nUser-agent: Seekr\nDisallow: /\n\n# Block Grok\nUser-agent: Grok\nDisallow: /\n\n# Sitemap declaration\nSitemap: https://vimeo.com/marketing.xml.gz\nSitemap: https://vimeo.com/sitemap/videos.xml.gz\n\n","agents":{"GPTBot":true,"OAI-SearchBot":true,"ChatGPT-User":true,"ClaudeBot":true,"Claude-SearchBot":true,"PerplexityBot":true,"Google-Extended":true,"Applebot-Extended":true,"CCBot":false,"Bytespider":false,"Meta-ExternalAgent":true,"Amazonbot":true}},"sitemap":{"status":200,"content_type":"application/xml","bytes":2759,"sha256":"5fd7f93b87cdc8bec0c01f4caafea206f5249b41ea2d672532661e0ffef9c326","final_host":"vimeo.com","ms":160,"url":"https://vimeo.com/marketing.xml.gz","declared":true,"xml":true},"files":{"llms_txt":{"status":404,"content_type":"text/html","bytes":163541,"sha256":"21c717b9ddb7e095d5ae2482bb03db883aec28c7d0651713c841ea8bb1a59caf","final_host":"vimeo.com","ms":248,"path":"/llms.txt","html":true,"json":null,"present":false},"agents_md":{"status":404,"content_type":"text/html","bytes":136322,"sha256":"22ff0fcc2204d6c550b5d21866955697ffc23a68de753d330e46d6388f24f3ed","final_host":"vimeo.com","ms":137,"path":"/agents.md","html":true,"json":null,"present":false},"agent_skills":{"status":404,"content_type":"text/html","bytes":163541,"sha256":"21c717b9ddb7e095d5ae2482bb03db883aec28c7d0651713c841ea8bb1a59caf","final_host":"vimeo.com","ms":403,"path":"/.well-known/agent-skills/index.json","html":true,"json":null,"present":false},"mcp_card":{"status":404,"content_type":"text/html","bytes":163541,"sha256":"21c717b9ddb7e095d5ae2482bb03db883aec28c7d0651713c841ea8bb1a59caf","final_host":"vimeo.com","ms":219,"path":"/.well-known/mcp/server-card.json","html":true,"json":null,"present":false},"api_catalog":{"status":404,"content_type":"text/html","bytes":163541,"sha256":"21c717b9ddb7e095d5ae2482bb03db883aec28c7d0651713c841ea8bb1a59caf","final_host":"vimeo.com","ms":241,"path":"/.well-known/api-catalog","html":true,"json":null,"present":false},"a2a_card":{"status":404,"content_type":"text/html","bytes":163541,"sha256":"21c717b9ddb7e095d5ae2482bb03db883aec28c7d0651713c841ea8bb1a59caf","final_host":"vimeo.com","ms":181,"path":"/.well-known/agent-card.json","html":true,"json":null,"present":false}},"rings":{"v":1,"candidates":[{"url":"https://mcp.vimeo.com/mcp","source":"conventional_host"}],"mcp":[{"url":"https://mcp.vimeo.com/mcp","source":"conventional_host","outcome":"auth_required","status":401,"ms":158,"initialized":false,"protocol_version":null,"server":null,"tools":null,"tools_sha256":null,"auth_header":true}],"a2a":null,"webmcp":{"model_context":false,"bridge":false,"mcp_json":false,"link_rel":false,"any":false},"handshakes":1,"get_path":{"status":200,"content_type":"text/html","ms":307,"error":null},"get_host":{"status":401,"content_type":"text/plain","ms":190,"error":null}},"derived":{"reachable":true,"robots_served":true,"robots_sitemap":true,"content_signal":false,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":["CCBot","Bytespider"],"blocks_any_ai":true,"files":{"llms_txt":false,"agents_md":false,"agent_skills":false,"mcp_card":false,"api_catalog":false,"a2a_card":false},"any_capability":false,"ai_refused":[],"ai_thin":[],"rings":{"mcp_declared":true,"mcp_live":false,"mcp_auth":true,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":2,"fh":"vimeo.com","fp_other_host":[],"fp":"e681ff7cadc4b34a","fp_bits":{"gptbot":0,"claudebot":0,"oaisearch":0,"perplexity":0}},"derive_v":3,"frame":"top1k","shard":6},{"v":1,"kind":"crawlcheck-census-record","run":14,"domain":"gvt2.com","rank":88,"measured_at":"2026-10-15T05:34:40.249Z","home":{"browser":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"gvt2.com","ms":67,"identity":"Mobile browser","html":false},"gptbot":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"gvt2.com","ms":68,"identity":"GPTBot","html":false},"claudebot":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"gvt2.com","ms":68,"identity":"ClaudeBot","html":false},"oaisearch":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"gvt2.com","ms":68,"identity":"OAI-SearchBot","html":false},"perplexity":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"gvt2.com","ms":68,"identity":"PerplexityBot","html":false}},"robots":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"gvt2.com","ms":68,"html":false,"text":null,"agents":null},"sitemap":null,"files":null,"derived":{"reachable":false,"robots_served":false,"robots_sitemap":false,"content_signal":false,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":[],"blocks_any_ai":false,"files":{"llms_txt":false,"agents_md":false,"agent_skills":false,"mcp_card":false,"api_catalog":false,"a2a_card":false},"any_capability":false,"ai_refused":[],"ai_thin":[],"rings":{"mcp_declared":false,"mcp_live":false,"mcp_auth":false,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":2,"fh":"gvt2.com","fp_other_host":[]},"derive_v":3,"frame":"top1k","shard":7},{"v":1,"kind":"crawlcheck-census-record","run":14,"domain":"cdninstagram.com","rank":89,"measured_at":"2026-10-15T05:26:23.049Z","home":{"browser":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"cdninstagram.com","ms":41,"identity":"Mobile browser","html":false},"gptbot":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"cdninstagram.com","ms":1,"identity":"GPTBot","html":false},"claudebot":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"cdninstagram.com","ms":1,"identity":"ClaudeBot","html":false},"oaisearch":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"cdninstagram.com","ms":1,"identity":"OAI-SearchBot","html":false},"perplexity":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"cdninstagram.com","ms":1,"identity":"PerplexityBot","html":false}},"robots":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"cdninstagram.com","ms":1,"html":false,"text":null,"agents":null},"sitemap":null,"files":null,"derived":{"reachable":false,"robots_served":false,"robots_sitemap":false,"content_signal":false,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":[],"blocks_any_ai":false,"files":{"llms_txt":false,"agents_md":false,"agent_skills":false,"mcp_card":false,"api_catalog":false,"a2a_card":false},"any_capability":false,"ai_refused":[],"ai_thin":[],"rings":{"mcp_declared":false,"mcp_live":false,"mcp_auth":false,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":2,"fh":"cdninstagram.com","fp_other_host":[]},"derive_v":3,"frame":"top1k","shard":0},{"v":1,"kind":"crawlcheck-census-record","run":14,"domain":"openai.com","rank":90,"measured_at":"2026-10-15T05:27:24.896Z","home":{"browser":{"status":403,"content_type":"text/html","bytes":9861,"sha256":"8c43b6c4551d2a08f0f8e0a0f3f2821e2cb5bf2f5cabd917ba76d7eaac58edfb","final_host":"openai.com","ms":4,"identity":"Mobile browser","html":true},"gptbot":{"status":403,"content_type":"text/html","bytes":9861,"sha256":"a7cb6c08ca4dc95a8dfd31ae00ebb4ddaaa9ec6601671e59c82c4d51a08ede58","final_host":"openai.com","ms":2,"identity":"GPTBot","html":true},"claudebot":{"status":403,"content_type":"text/html","bytes":9775,"sha256":"a95e0917712ffaa649254bed3f4e82b79e08cd417e6e6d570f60a993ec2aa58d","final_host":"openai.com","ms":2,"identity":"ClaudeBot","html":true},"oaisearch":{"status":403,"content_type":"text/html","bytes":9882,"sha256":"2743ec252a882ed572bc20c3c20990001d269844f2dfe3ff9bb1cae9b99e5d12","final_host":"openai.com","ms":2,"identity":"OAI-SearchBot","html":true},"perplexity":{"status":403,"content_type":"text/html","bytes":9968,"sha256":"1bee66961b257f3ab626490f364faa7d5c1d433ec34db7dab5e6c8a94afba284","final_host":"openai.com","ms":3,"identity":"PerplexityBot","html":true}},"robots":{"status":200,"content_type":"text/plain","bytes":98,"sha256":"c53100563b9d289efeebb6f826a20c566578e1337e3096e298eb9da5ecc2e498","final_host":"openai.com","ms":5,"html":false,"text":"User-agent: *\nAllow: /\nDisallow: /microsoft-for-startups/\n\nSitemap: https://openai.com/sitemap.xml","agents":{"GPTBot":true,"OAI-SearchBot":true,"ChatGPT-User":true,"ClaudeBot":true,"Claude-SearchBot":true,"PerplexityBot":true,"Google-Extended":true,"Applebot-Extended":true,"CCBot":true,"Bytespider":true,"Meta-ExternalAgent":true,"Amazonbot":true}},"sitemap":null,"files":null,"derived":{"reachable":false,"robots_served":false,"robots_sitemap":false,"content_signal":false,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":[],"blocks_any_ai":false,"files":{"llms_txt":false,"agents_md":false,"agent_skills":false,"mcp_card":false,"api_catalog":false,"a2a_card":false},"any_capability":false,"ai_refused":[],"ai_thin":[],"rings":{"mcp_declared":false,"mcp_live":false,"mcp_auth":false,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":2,"fh":"openai.com","fp_other_host":[]},"derive_v":3,"frame":"top1k","shard":1},{"v":1,"kind":"crawlcheck-census-record","run":14,"domain":"f5.com","rank":91,"measured_at":"2026-10-15T05:28:26.341Z","home":{"browser":{"status":200,"content_type":"text/html","bytes":495887,"sha256":"c61f3f97150857fe37772cf2e54efef29ec5c300cdb2dea0e264345ee070c896","final_host":"www.f5.com","ms":1505,"identity":"Mobile browser","html":true,"fp":"4639fb170a0bc18b"},"gptbot":{"status":200,"content_type":"text/html","bytes":495887,"sha256":"c61f3f97150857fe37772cf2e54efef29ec5c300cdb2dea0e264345ee070c896","final_host":"www.f5.com","ms":1492,"identity":"GPTBot","html":true,"fp":"4639fb170a0bc18b"},"claudebot":{"status":200,"content_type":"text/html","bytes":495887,"sha256":"c61f3f97150857fe37772cf2e54efef29ec5c300cdb2dea0e264345ee070c896","final_host":"www.f5.com","ms":1535,"identity":"ClaudeBot","html":true,"fp":"4639fb170a0bc18b"},"oaisearch":{"status":200,"content_type":"text/html","bytes":495844,"sha256":"f2da440a0268d38863f6087ae31a9bd05915642d3ae454a9903f60bc47211ad4","final_host":"www.f5.com","ms":2746,"identity":"OAI-SearchBot","html":true,"fp":"4639fb170a0bc18b"},"perplexity":{"status":200,"content_type":"text/html","bytes":495887,"sha256":"c61f3f97150857fe37772cf2e54efef29ec5c300cdb2dea0e264345ee070c896","final_host":"www.f5.com","ms":1524,"identity":"PerplexityBot","html":true,"fp":"4639fb170a0bc18b"}},"robots":{"status":200,"content_type":"text/plain","bytes":156,"sha256":"eca371645a1484bc5d7dfc8e150f188b7e70d7754e6c00c1770de364f38e1401","final_host":"www.f5.com","ms":1460,"html":false,"text":"# *\nUser-agent: *\nAllow: /\n\n# Host\nHost: https://www.f5.com\n\n# Sitemaps\nSitemap: https://www.f5.com/sitemap.xml\nSitemap: https://www.f5.com/labs/sitemap.xml","agents":{"GPTBot":true,"OAI-SearchBot":true,"ChatGPT-User":true,"ClaudeBot":true,"Claude-SearchBot":true,"PerplexityBot":true,"Google-Extended":true,"Applebot-Extended":true,"CCBot":true,"Bytespider":true,"Meta-ExternalAgent":true,"Amazonbot":true}},"sitemap":{"status":200,"content_type":"application/xml","bytes":1287,"sha256":"2705ee74dea18c947ac642a4a07d1e985e48869398a4e6bf48a56879ef38f266","final_host":"www.f5.com","ms":53,"url":"https://www.f5.com/sitemap.xml","declared":true,"xml":true},"files":{"llms_txt":{"status":404,"content_type":"text/html","bytes":103705,"sha256":"7c6cbb07d00ac9ec4f037854bce0eaa94f43f123666dffc3e823a00b88c14c90","final_host":"www.f5.com","ms":1282,"path":"/llms.txt","html":true,"json":null,"present":false},"agents_md":{"status":404,"content_type":"text/html","bytes":103708,"sha256":"2694112ca4c834387895b2a1cebc4f4e047ac4971e829a25175f5e094334286a","final_host":"www.f5.com","ms":1855,"path":"/agents.md","html":true,"json":null,"present":false},"agent_skills":{"status":404,"content_type":"text/html","bytes":103794,"sha256":"f29230eae4edad113ccb9c71e8f9c44a205213cf679f2446c8884f20c369d96b","final_host":"www.f5.com","ms":1231,"path":"/.well-known/agent-skills/index.json","html":true,"json":null,"present":false},"mcp_card":{"status":404,"content_type":"text/html","bytes":103785,"sha256":"2f8b7ff0d778c42fa6303890c8f65cee6318923388a3fbaa96f1fcc45d56dbb1","final_host":"www.f5.com","ms":1261,"path":"/.well-known/mcp/server-card.json","html":true,"json":null,"present":false},"api_catalog":{"status":404,"content_type":"text/html","bytes":103754,"sha256":"c63df1ee53a141f0c31dd5748e0be680202810a466a3e7c7cad43f5800884707","final_host":"www.f5.com","ms":1130,"path":"/.well-known/api-catalog","html":true,"json":null,"present":false},"a2a_card":{"status":404,"content_type":"text/html","bytes":103766,"sha256":"b0d7a4318345623798a40f3e84399fdf29f10c24830e4da9e42d444e6e2ad2a1","final_host":"www.f5.com","ms":1032,"path":"/.well-known/agent-card.json","html":true,"json":null,"present":false}},"rings":{"v":1,"candidates":[],"mcp":[],"a2a":null,"webmcp":{"model_context":false,"bridge":false,"mcp_json":false,"link_rel":false,"any":false},"handshakes":0,"get_path":{"status":404,"content_type":"text/html","ms":806,"error":null},"get_host":{"status":530,"content_type":"text/plain","ms":423,"error":null}},"derived":{"reachable":true,"robots_served":true,"robots_sitemap":true,"content_signal":false,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":[],"blocks_any_ai":false,"files":{"llms_txt":false,"agents_md":false,"agent_skills":false,"mcp_card":false,"api_catalog":false,"a2a_card":false},"any_capability":false,"ai_refused":[],"ai_thin":[],"rings":{"mcp_declared":false,"mcp_live":false,"mcp_auth":false,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":2,"fh":"www.f5.com","fp_other_host":[],"fp":"4639fb170a0bc18b","fp_bits":{"gptbot":0,"claudebot":0,"oaisearch":0,"perplexity":0}},"derive_v":3,"frame":"top1k","shard":2},{"v":1,"kind":"crawlcheck-census-record","run":14,"domain":"gwfb.net","rank":92,"measured_at":"2026-10-15T05:29:25.755Z","home":{"browser":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"gwfb.net","ms":582,"identity":"Mobile browser","html":false},"gptbot":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"gwfb.net","ms":101,"identity":"GPTBot","html":false},"claudebot":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"gwfb.net","ms":89,"identity":"ClaudeBot","html":false},"oaisearch":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"gwfb.net","ms":90,"identity":"OAI-SearchBot","html":false},"perplexity":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"gwfb.net","ms":81,"identity":"PerplexityBot","html":false}},"robots":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"gwfb.net","ms":5,"html":false,"text":null,"agents":null},"sitemap":null,"files":null,"derived":{"reachable":false,"robots_served":false,"robots_sitemap":false,"content_signal":false,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":[],"blocks_any_ai":false,"files":{"llms_txt":false,"agents_md":false,"agent_skills":false,"mcp_card":false,"api_catalog":false,"a2a_card":false},"any_capability":false,"ai_refused":[],"ai_thin":[],"rings":{"mcp_declared":false,"mcp_live":false,"mcp_auth":false,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":2,"fh":"gwfb.net","fp_other_host":[]},"derive_v":3,"frame":"top1k","shard":3},{"v":1,"kind":"crawlcheck-census-record","run":14,"domain":"sentry.io","rank":93,"measured_at":"2026-10-15T05:30:22.136Z","home":{"browser":{"status":200,"content_type":"text/html","bytes":628854,"sha256":"1f3a227091134ffd24bf9935e47315a08a2f85ac84a6e8a57efbe0761567101d","final_host":"sentry.io","ms":766,"identity":"Mobile browser","html":true,"fp":"4c008f584f72c757"},"gptbot":{"status":200,"content_type":"text/markdown","bytes":17779,"sha256":"10aa3dfd5aa1731db39e02e2979131553a344603b73f954d216380723abb811e","final_host":"sentry.io","ms":408,"identity":"GPTBot","html":false},"claudebot":{"status":200,"content_type":"text/markdown","bytes":17779,"sha256":"10aa3dfd5aa1731db39e02e2979131553a344603b73f954d216380723abb811e","final_host":"sentry.io","ms":427,"identity":"ClaudeBot","html":false},"oaisearch":{"status":200,"content_type":"text/markdown","bytes":17779,"sha256":"10aa3dfd5aa1731db39e02e2979131553a344603b73f954d216380723abb811e","final_host":"sentry.io","ms":428,"identity":"OAI-SearchBot","html":false},"perplexity":{"status":200,"content_type":"text/markdown","bytes":17779,"sha256":"10aa3dfd5aa1731db39e02e2979131553a344603b73f954d216380723abb811e","final_host":"sentry.io","ms":415,"identity":"PerplexityBot","html":false}},"robots":{"status":200,"content_type":"text/plain","bytes":140,"sha256":"c0f7d803d97db6c0bf75804b97eed4a533f7249c27994e253ad4fa403652da96","final_host":"sentry.io","ms":209,"html":false,"text":"User-agent: *\nContent-Signal: search=yes, ai-input=yes, ai-train=yes\nDisallow: /api/\nAllow: /\n\nSitemap: https://sentry.io/sitemap-index.xml\n","agents":{"GPTBot":true,"OAI-SearchBot":true,"ChatGPT-User":true,"ClaudeBot":true,"Claude-SearchBot":true,"PerplexityBot":true,"Google-Extended":true,"Applebot-Extended":true,"CCBot":true,"Bytespider":true,"Meta-ExternalAgent":true,"Amazonbot":true}},"sitemap":{"status":200,"content_type":"application/xml","bytes":372,"sha256":"8bfba18ac85d5dbb7f46917c1e54f491c1b2a0df8559c46e9d92307ef1d41a89","final_host":"sentry.io","ms":214,"url":"https://sentry.io/sitemap-index.xml","declared":true,"xml":true},"files":{"llms_txt":{"status":200,"content_type":"text/plain","bytes":244902,"sha256":"4f339d700f1ecc55754c49edd326b08e5371334ca1f3e076f94e1e40ae120b87","final_host":"sentry.io","ms":342,"path":"/llms.txt","html":false,"json":null,"present":true},"agents_md":{"status":200,"content_type":"text/markdown","bytes":976,"sha256":"d74022d0efda2763dde039ddcf1155c5d0a4783227ba9d88a47eed66435a7683","final_host":"sentry.io","ms":152,"path":"/agents.md","html":false,"json":null,"present":true},"agent_skills":{"status":200,"content_type":"application/json","bytes":13,"sha256":"0b513961955003886d251c9091e61f5dc648adbb93b431b1a951b08e0f6201ac","final_host":"sentry.io","ms":153,"path":"/.well-known/agent-skills/index.json","html":false,"json":true,"present":true},"mcp_card":{"status":200,"content_type":"application/json","bytes":234,"sha256":"5fb322b52608b7465c0ca90eb50ef8cea2e63a0d657a32f35cc2765023551ae2","final_host":"sentry.io","ms":151,"path":"/.well-known/mcp/server-card.json","html":false,"json":true,"present":true},"api_catalog":{"status":200,"content_type":"application/linkset+json","bytes":179,"sha256":"e906b19b2175c78512435aa44daef69d900227c4f8e2e225186a626f5d98733f","final_host":"sentry.io","ms":152,"path":"/.well-known/api-catalog","html":false,"json":true,"present":true},"a2a_card":{"status":200,"content_type":"text/html","bytes":27388,"sha256":"49e031619d9bc69c70d19ee2cf3c7c9316830902fa57b2f07c1713dc25d707db","final_host":"sentry.io","ms":460,"path":"/.well-known/agent-card.json","html":true,"json":null,"present":false}},"rings":{"v":1,"candidates":[{"url":"https://mcp.sentry.dev/mcp","source":"mcp_card"},{"url":"https://sentry.io","source":"mcp_card"},{"url":"https://mcp.sentry.dev","source":"agents_md"},{"url":"https://mcp.sentry.dev/","source":"homepage_link"}],"mcp":[{"url":"https://mcp.sentry.dev/mcp","source":"mcp_card","outcome":"auth_required","status":401,"ms":101,"initialized":false,"protocol_version":null,"server":null,"tools":null,"tools_sha256":null,"auth_header":true},{"url":"https://sentry.io","source":"mcp_card","outcome":"auth_required","status":403,"ms":145,"initialized":false,"protocol_version":null,"server":null,"tools":null,"tools_sha256":null,"auth_header":false},{"url":"https://mcp.sentry.dev","source":"agents_md","outcome":"http_error","status":405,"ms":98,"initialized":false,"protocol_version":null,"server":null,"tools":null,"tools_sha256":null,"auth_header":false}],"a2a":null,"webmcp":{"model_context":false,"bridge":false,"mcp_json":false,"link_rel":false,"any":false},"handshakes":3,"get_path":{"status":null,"content_type":null,"ms":8000,"error":"timeout"},"get_host":{"status":200,"content_type":"text/html","ms":789,"error":null}},"derived":{"reachable":true,"robots_served":true,"robots_sitemap":true,"content_signal":true,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":[],"blocks_any_ai":false,"files":{"llms_txt":true,"agents_md":true,"agent_skills":true,"mcp_card":true,"api_catalog":true,"a2a_card":false},"any_capability":true,"ai_refused":[],"ai_thin":["gptbot","claudebot","oaisearch","perplexity"],"rings":{"mcp_declared":true,"mcp_live":false,"mcp_auth":true,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":2,"fh":"sentry.io","fp_other_host":[],"fp":"4c008f584f72c757","fp_bits":{}},"derive_v":3,"frame":"top1k","shard":4},{"v":1,"kind":"crawlcheck-census-record","run":14,"domain":"a2z.com","rank":94,"measured_at":"2026-10-15T05:31:25.738Z","home":{"browser":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"a2z.com","ms":80,"identity":"Mobile browser","html":false},"gptbot":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"a2z.com","ms":74,"identity":"GPTBot","html":false},"claudebot":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"a2z.com","ms":73,"identity":"ClaudeBot","html":false},"oaisearch":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"a2z.com","ms":74,"identity":"OAI-SearchBot","html":false},"perplexity":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"a2z.com","ms":74,"identity":"PerplexityBot","html":false}},"robots":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"a2z.com","ms":3,"html":false,"text":null,"agents":null},"sitemap":null,"files":null,"derived":{"reachable":false,"robots_served":false,"robots_sitemap":false,"content_signal":false,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":[],"blocks_any_ai":false,"files":{"llms_txt":false,"agents_md":false,"agent_skills":false,"mcp_card":false,"api_catalog":false,"a2a_card":false},"any_capability":false,"ai_refused":[],"ai_thin":[],"rings":{"mcp_declared":false,"mcp_live":false,"mcp_auth":false,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":2,"fh":"a2z.com","fp_other_host":[]},"derive_v":3,"frame":"top1k","shard":5},{"v":1,"kind":"crawlcheck-census-record","run":14,"domain":"nginx.com","rank":95,"measured_at":"2026-10-15T05:33:24.843Z","home":{"browser":{"status":200,"content_type":"text/html","bytes":370661,"sha256":"3fe852260fd9a12bc6b0512bb3d113a748b36e1be249b1d9189e21cf73132460","final_host":"www.f5.com","ms":1833,"identity":"Mobile browser","html":true,"fp":"21691fde4a434812"},"gptbot":{"status":200,"content_type":"text/html","bytes":370661,"sha256":"3fe852260fd9a12bc6b0512bb3d113a748b36e1be249b1d9189e21cf73132460","final_host":"www.f5.com","ms":1770,"identity":"GPTBot","html":true,"fp":"21691fde4a434812"},"claudebot":{"status":200,"content_type":"text/html","bytes":370661,"sha256":"2c70017c1cec84ca091fbe9720e6a8065bac5e2f61c4aee0cf9668cf0b21b659","final_host":"www.f5.com","ms":2494,"identity":"ClaudeBot","html":true,"fp":"21691fde4a434812"},"oaisearch":{"status":200,"content_type":"text/html","bytes":370661,"sha256":"96828c2688eb8f7592796de6f4d1d638a3afa167aeca54f9b9df693dfe781ae1","final_host":"www.f5.com","ms":1698,"identity":"OAI-SearchBot","html":true,"fp":"21691fde4a434812"},"perplexity":{"status":200,"content_type":"text/html","bytes":370661,"sha256":"3fe852260fd9a12bc6b0512bb3d113a748b36e1be249b1d9189e21cf73132460","final_host":"www.f5.com","ms":1759,"identity":"PerplexityBot","html":true,"fp":"21691fde4a434812"}},"robots":{"status":200,"content_type":"text/html","bytes":370661,"sha256":"96828c2688eb8f7592796de6f4d1d638a3afa167aeca54f9b9df693dfe781ae1","final_host":"www.f5.com","ms":1685,"html":true,"text":null,"agents":null},"sitemap":{"status":200,"content_type":"text/html","bytes":370661,"sha256":"ff80883007ad826c63521e935f61507f2a737ce80297da0216850cfd73272901","final_host":"www.f5.com","ms":476,"url":"https://nginx.com/sitemap.xml","declared":false,"xml":false},"files":{"llms_txt":{"status":200,"content_type":"text/html","bytes":370661,"sha256":"4c8a89004a2a02d1caf3f6d4954e517ae60f85bd4548744921658c3b0e6a7be0","final_host":"www.f5.com","ms":572,"path":"/llms.txt","html":true,"json":null,"present":false},"agents_md":{"status":200,"content_type":"text/html","bytes":370661,"sha256":"ff80883007ad826c63521e935f61507f2a737ce80297da0216850cfd73272901","final_host":"www.f5.com","ms":1060,"path":"/agents.md","html":true,"json":null,"present":false},"agent_skills":{"status":200,"content_type":"text/html","bytes":370661,"sha256":"f6e7d75903d242371da69fd030d33bd5980aa21eb2af5456d88b1ac4c239a277","final_host":"www.f5.com","ms":537,"path":"/.well-known/agent-skills/index.json","html":true,"json":null,"present":false},"mcp_card":{"status":200,"content_type":"text/html","bytes":370661,"sha256":"f6e7d75903d242371da69fd030d33bd5980aa21eb2af5456d88b1ac4c239a277","final_host":"www.f5.com","ms":474,"path":"/.well-known/mcp/server-card.json","html":true,"json":null,"present":false},"api_catalog":{"status":200,"content_type":"text/html","bytes":370661,"sha256":"f6e7d75903d242371da69fd030d33bd5980aa21eb2af5456d88b1ac4c239a277","final_host":"www.f5.com","ms":490,"path":"/.well-known/api-catalog","html":true,"json":null,"present":false},"a2a_card":{"status":200,"content_type":"text/html","bytes":370661,"sha256":"4c8a89004a2a02d1caf3f6d4954e517ae60f85bd4548744921658c3b0e6a7be0","final_host":"www.f5.com","ms":315,"path":"/.well-known/agent-card.json","html":true,"json":null,"present":false}},"rings":{"v":1,"candidates":[],"mcp":[],"a2a":null,"webmcp":{"model_context":false,"bridge":false,"mcp_json":false,"link_rel":false,"any":false},"handshakes":0,"get_path":{"status":200,"content_type":"text/html","ms":1455,"error":null},"get_host":{"status":530,"content_type":"text/plain","ms":254,"error":null}},"derived":{"reachable":true,"robots_served":false,"robots_sitemap":false,"content_signal":false,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":[],"blocks_any_ai":false,"files":{"llms_txt":false,"agents_md":false,"agent_skills":false,"mcp_card":false,"api_catalog":false,"a2a_card":false},"any_capability":false,"ai_refused":[],"ai_thin":[],"rings":{"mcp_declared":false,"mcp_live":false,"mcp_auth":false,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":2,"fh":"www.f5.com","fp_other_host":[],"fp":"21691fde4a434812","fp_bits":{"gptbot":0,"claudebot":0,"oaisearch":0,"perplexity":0}},"derive_v":3,"frame":"top1k","shard":6},{"v":1,"kind":"crawlcheck-census-record","run":14,"domain":"ytimg.com","rank":96,"measured_at":"2026-10-15T05:34:40.249Z","home":{"browser":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"ytimg.com","ms":83,"identity":"Mobile browser","html":false},"gptbot":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"ytimg.com","ms":84,"identity":"GPTBot","html":false},"claudebot":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"ytimg.com","ms":84,"identity":"ClaudeBot","html":false},"oaisearch":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"ytimg.com","ms":84,"identity":"OAI-SearchBot","html":false},"perplexity":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"ytimg.com","ms":84,"identity":"PerplexityBot","html":false}},"robots":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"ytimg.com","ms":84,"html":false,"text":null,"agents":null},"sitemap":null,"files":null,"derived":{"reachable":false,"robots_served":false,"robots_sitemap":false,"content_signal":false,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":[],"blocks_any_ai":false,"files":{"llms_txt":false,"agents_md":false,"agent_skills":false,"mcp_card":false,"api_catalog":false,"a2a_card":false},"any_capability":false,"ai_refused":[],"ai_thin":[],"rings":{"mcp_declared":false,"mcp_live":false,"mcp_auth":false,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":2,"fh":"ytimg.com","fp_other_host":[]},"derive_v":3,"frame":"top1k","shard":7},{"v":1,"kind":"crawlcheck-census-record","run":14,"domain":"mozilla.org","rank":97,"measured_at":"2026-10-15T05:26:23.049Z","home":{"browser":{"status":200,"content_type":"text/html","bytes":47803,"sha256":"fd12c42560b6d95ca1b1df418224bfb83fcfb4914e09b44b15fd8da775062a2c","final_host":"www.mozilla.org","ms":270,"identity":"Mobile browser","html":true,"fp":"4bd3d43d5d3f2cc8"},"gptbot":{"status":200,"content_type":"text/html","bytes":47803,"sha256":"fd12c42560b6d95ca1b1df418224bfb83fcfb4914e09b44b15fd8da775062a2c","final_host":"www.mozilla.org","ms":226,"identity":"GPTBot","html":true,"fp":"4bd3d43d5d3f2cc8"},"claudebot":{"status":200,"content_type":"text/html","bytes":47803,"sha256":"fd12c42560b6d95ca1b1df418224bfb83fcfb4914e09b44b15fd8da775062a2c","final_host":"www.mozilla.org","ms":197,"identity":"ClaudeBot","html":true,"fp":"4bd3d43d5d3f2cc8"},"oaisearch":{"status":200,"content_type":"text/html","bytes":47803,"sha256":"fd12c42560b6d95ca1b1df418224bfb83fcfb4914e09b44b15fd8da775062a2c","final_host":"www.mozilla.org","ms":709,"identity":"OAI-SearchBot","html":true,"fp":"4bd3d43d5d3f2cc8"},"perplexity":{"status":200,"content_type":"text/html","bytes":47803,"sha256":"fd12c42560b6d95ca1b1df418224bfb83fcfb4914e09b44b15fd8da775062a2c","final_host":"www.mozilla.org","ms":723,"identity":"PerplexityBot","html":true,"fp":"4bd3d43d5d3f2cc8"}},"robots":{"status":200,"content_type":"text/plain","bytes":66,"sha256":"cf0fd2842659297b0d6fce407bed86bf424ae6666051b881dbd72e871d687fe3","final_host":"www.mozilla.org","ms":249,"html":false,"text":"user-agent: *\ndisallow: /*/newsletter/existing/\ndisallow: /*/etc/\n","agents":{"GPTBot":true,"OAI-SearchBot":true,"ChatGPT-User":true,"ClaudeBot":true,"Claude-SearchBot":true,"PerplexityBot":true,"Google-Extended":true,"Applebot-Extended":true,"CCBot":true,"Bytespider":true,"Meta-ExternalAgent":true,"Amazonbot":true}},"sitemap":{"status":404,"content_type":"text/html","bytes":8957,"sha256":"172885c656a5409595dd3232b7bf6eeef23e29f9f9349709d76f7f681ba1d34c","final_host":"www.mozilla.org","ms":217,"url":"https://mozilla.org/sitemap.xml","declared":false,"xml":false},"files":{"llms_txt":{"status":404,"content_type":"text/html","bytes":8954,"sha256":"b75a5feebe3740b635a85624198bdf69ca178b80cbe477a3f45c98f0a0be944e","final_host":"www.mozilla.org","ms":707,"path":"/llms.txt","html":true,"json":null,"present":false},"agents_md":{"status":404,"content_type":"text/html","bytes":8955,"sha256":"14d413ff4c344f1fd4cfb1025e77fa0117a49e42aa2e2b471372ce54870aaade","final_host":"www.mozilla.org","ms":1211,"path":"/agents.md","html":true,"json":null,"present":false},"agent_skills":{"status":404,"content_type":"text/html","bytes":8981,"sha256":"b0a4487593cb2f4b9bc061e4551d456a828f7d45776c21571309787ac72d2d18","final_host":"www.mozilla.org","ms":488,"path":"/.well-known/agent-skills/index.json","html":true,"json":null,"present":false},"mcp_card":{"status":404,"content_type":"text/html","bytes":8978,"sha256":"b8b6e883effe5af853c8e07e770010aa1b2cc91104940f92a48744e2bd509da1","final_host":"www.mozilla.org","ms":446,"path":"/.well-known/mcp/server-card.json","html":true,"json":null,"present":false},"api_catalog":{"status":404,"content_type":"text/html","bytes":8969,"sha256":"3af22144ca7fddf01d3b089c877540e2c65cb6f607c14b247b77e535cb465660","final_host":"www.mozilla.org","ms":958,"path":"/.well-known/api-catalog","html":true,"json":null,"present":false},"a2a_card":{"status":404,"content_type":"text/html","bytes":8973,"sha256":"dac1f20ac9137ac9355cba1eba21b8866c144689f78b3079db00dd1efbca7aa2","final_host":"www.mozilla.org","ms":964,"path":"/.well-known/agent-card.json","html":true,"json":null,"present":false}},"rings":{"v":1,"candidates":[],"mcp":[],"a2a":null,"webmcp":{"model_context":false,"bridge":false,"mcp_json":false,"link_rel":false,"any":false},"handshakes":0,"get_path":{"status":404,"content_type":"text/html","ms":950,"error":null},"get_host":{"status":530,"content_type":"text/plain","ms":6,"error":null}},"derived":{"reachable":true,"robots_served":true,"robots_sitemap":false,"content_signal":false,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":[],"blocks_any_ai":false,"files":{"llms_txt":false,"agents_md":false,"agent_skills":false,"mcp_card":false,"api_catalog":false,"a2a_card":false},"any_capability":false,"ai_refused":[],"ai_thin":[],"rings":{"mcp_declared":false,"mcp_live":false,"mcp_auth":false,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":2,"fh":"www.mozilla.org","fp_other_host":[],"fp":"4bd3d43d5d3f2cc8","fp_bits":{"gptbot":0,"claudebot":0,"oaisearch":0,"perplexity":0}},"derive_v":3,"frame":"top1k","shard":0},{"v":1,"kind":"crawlcheck-census-record","run":14,"domain":"highwebmedia.com","rank":98,"measured_at":"2026-10-15T05:27:24.896Z","home":{"browser":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"highwebmedia.com","ms":5,"identity":"Mobile browser","html":false},"gptbot":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"highwebmedia.com","ms":1,"identity":"GPTBot","html":false},"claudebot":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"highwebmedia.com","ms":1,"identity":"ClaudeBot","html":false},"oaisearch":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"highwebmedia.com","ms":1,"identity":"OAI-SearchBot","html":false},"perplexity":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"highwebmedia.com","ms":1,"identity":"PerplexityBot","html":false}},"robots":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"highwebmedia.com","ms":1,"html":false,"text":null,"agents":null},"sitemap":null,"files":null,"derived":{"reachable":false,"robots_served":false,"robots_sitemap":false,"content_signal":false,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":[],"blocks_any_ai":false,"files":{"llms_txt":false,"agents_md":false,"agent_skills":false,"mcp_card":false,"api_catalog":false,"a2a_card":false},"any_capability":false,"ai_refused":[],"ai_thin":[],"rings":{"mcp_declared":false,"mcp_live":false,"mcp_auth":false,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":2,"fh":"highwebmedia.com","fp_other_host":[]},"derive_v":3,"frame":"top1k","shard":1},{"v":1,"kind":"crawlcheck-census-record","run":14,"domain":"yandex.ru","rank":99,"measured_at":"2026-10-15T05:28:26.341Z","home":{"browser":{"status":200,"content_type":"text/html","bytes":3061,"sha256":"f981a41a32c6becba9ea4585377412d9542ffb689d4aabedadbbee4280ad9e7c","final_host":"sso.passport.yandex.ru","ms":246,"identity":"Mobile browser","html":true},"gptbot":{"status":200,"content_type":"text/html","bytes":2898,"sha256":"2b7ec0cddf724b955dbeeb42d61f0ce577312d406494f060e57c87cf53ba4350","final_host":"sso.passport.yandex.ru","ms":299,"identity":"GPTBot","html":true},"claudebot":{"status":200,"content_type":"text/html","bytes":2901,"sha256":"e417e2a1e190c9487cfec93f3ad456dd7b9487750724c9643333e3e60676da9c","final_host":"sso.passport.yandex.ru","ms":199,"identity":"ClaudeBot","html":true},"oaisearch":{"status":200,"content_type":"text/html","bytes":2901,"sha256":"4efc7ba46e82f7e6d88b67e391f5e6b4917b83fec82a852cf46e87398965096b","final_host":"sso.passport.yandex.ru","ms":268,"identity":"OAI-SearchBot","html":true},"perplexity":{"status":200,"content_type":"text/html","bytes":2901,"sha256":"b97e39ca6765caba5626adee2d599b49d61af2faf75c07c105356b86b702d51c","final_host":"sso.passport.yandex.ru","ms":290,"identity":"PerplexityBot","html":true}},"robots":{"status":200,"content_type":"text/plain","bytes":40638,"sha256":"20f09fe9e2428a94f7c71c4c059b7e1e64b366ad05c8d880c3620317f7be64c7","final_host":"yandex.ru","ms":164,"html":false,"text":"# yandex.ru\nUser-agent: *\nDisallow: /?\nDisallow: /403.html\nDisallow: /404.html\nDisallow: /500.html\nDisallow: /adddata\nDisallow: /adresa-segmentator\nDisallow: /advanced_engl.html\nDisallow: /advertising\nDisallow: /alice/term/\nDisallow: /alice/*/term/\nDisallow: /alice/old-browser\nDisallow: /alice/store/orders/\nDisallow: /alice*?\nDisallow: /alice/*&\nDisallow: /alice/chat/\nDisallow: /alice?share=\nDisallow: /all-supported-params\nDisallow: /articles\nDisallow: /aura/\nAllow: /aura/promo/\nDisallow: /blog/*?text=\nDisallow: /blog/*?tag=\nDisallow: /blog/*/*/*\nDisallow: /blog/punto\nDisallow: /MAPS/\nDisallow: /MaPS/\nDisallow: /MapS/\nDisallow: /Maps/\nDisallow: /mApS/\nDisallow: /harita/business/\nDisallow: /maps/business/\nDisallow: /maps-api/docs/js-api/reference-guide/instruction-gxs.html\nDisallow: /maps-api/docs/js-api/reference-guide/instruction-rs.html\nDisallow: /catalog/?text=\nDisallow: /chat\nDisallow: /cgi-bin/\nDisallow: /cgi/\nDisallow: /clck\nDisallow: /collections/\nDisallow: /company/*.rss\nDisallow: /company/search\nDisallow: /comments/*\nDisallow: /courier\nDisallow: /conflagexp\nDisallow: /cy\nDisallow: /cycounter\nDisallow: /dzen\nDisallow: /experiments.xml\nDisallow: /formfeedback\nDisallow: /fresh\nDisallow: /goto_issue/\nDisallow: /goto_rubric/\nDisallow: /gundem\nDisallow: /i/\nDisallow: /images-data\nDisallow: /images.html\nDisallow: /images/*\nDisallow: /images-apphost/*\nAllow: /images/$\nAllow: /images/smart/$\nAllow: /images/touch/$\nAllow: /images/touch/?\nAllow: /images/search?text=*&source=morda*\nAllow: /images/search?rpt=imageview\nAllow: /images/touch/search?text=*&source=morda*\nAllow: /images/touch/search?rpt=imageview\nAllow: /images/pad/search?text=*&source=morda*\nAllow: /images/pad/search?rpt=imageview\nDisallow: /index_m\nDisallow: /iznanka/\nDisallow: /int_search_summary\nDisallow: /delivery_tracking\nDisallow: /jobs/skill-diagnostic/private/*\nDisallow: /keyboard_qwerty.html\nDisallow: /logotypes\nDisallow: /business/widget\nDisallow: /local/\nDisallow: /lab/neurodetector\nDisallow: /lab/habr-detector\nDisallow: /more_samples\nDisallow: /msearch\nDisallow: /map-constructor/loader*\nDisallow: /maps/print/*\nAllow: /maps/*?lang=ru$\nAllow: /maps/*?lang=uk$\nAllow: /maps/*?lang=kk$\nAllow: /maps/*?lang=uz$\nDisallow: /nmaps/*?\nDisallow: /mapeditor/*?\nDisallow: /metro/*?from\nDisallow: /masters\nDisallow: /services/executor/\nDisallow: /n?\nDisallow: /n$\nDisallow: /n/\nDisallow: /neuralsearch/api\nDisallow: /norobot\nDisallow: /opensearch.xml\nDisallow: /padsearch\nDisallow: /person\nDisallow: /podpiska/login.pl\nDisallow: /pogoda/search\nDisallow: /pogoda/404\nDisallow: /pogoda/0\nDisallow: /polling\nDisallow: /poll/\nDisallow: /portal/set/\nDisallow: /portal/tune/\nDisallow: /prefetch\nDisallow: /promo/*goodbye*\nDisallow: /promo/diskelement\nDisallow: /promo/launcher/mgoodbye/*\nDisallow: /promo/*welcome*\nDisallow: /promo/yobject/changelog*\nDisallow: /project/eda/partnership_services/$\nDisallow: /q/profile/*/answers\nDisallow: /q/profile/*/questions\nDisallow: /q/profile/*/verifies\nDisallow: /q/profile/*/articles\nDisallow: /q/profile/*/subscribers\nDisallow: /q/profile/*/org_answers\nDisallow: /q/user/*/answers\nDisallow: /q/user/*/questions\nDisallow: /q/user/*/verifies\nDisallow: /q/user/*/articles\nDisallow: /q/user/*/subscribers\nDisallow: /q/user/*/org_answers\nDisallow: /soft/bm/goodbye\nDisallow: /soft/chrome/ext-install/*\nDisallow: /soft/chrome/searchline-install/*\nDisallow: /soft/*goodbye*\nDisallow: /soft/punto/mac/uninstall/*\nDisallow: /soft/punto/win/uninstall/*\nDisallow: /soft/*welcome*\nDisallow: /soft/win/?*\nDisallow: /quotes\nDisallow: /realty*?*pinned_offer_id=\nDisallow: /realty*?*pinned_nb_id=\nDisallow: /realty/search\nDisallow: /realty*?*text=\nDisallow: /realty/result\nDisallow: /realty/map_pins\nDisallow: /realty/map_dust\nDisallow: /realty/callme\nDisallow: /redir\nDisallow: /region_map\nDisallow: /regions_list.xml\nDisallow: /regions.html?\nDisallow: /s/\nDisallow: /save\nDisallow: /safety/?\nDisallow: /safety?\nDisallow: /search\nAllow: /search/entity/touch/\nDisallow: /setup\nDisallow: /showcaptcha\nDisallow: /sitesearch\nDisallow: /sl/*.html\nDisallow: /soft/extensions/goodbye\nDisallow: /sportagent\nDisallow: /storeclick\nDisallow: /storerequest\nDisallow: /subscribe/confirm.pl\nDisallow: /subscribe/view.pl\nAllow: /support/\nDisallow: /support/direct-images\nDisallow: /support/direct-tooltips\nDisallow: /support/distr\nDisallow: /support/dsp\nDisallow: /support/fe4be44a295cc679e19bf0b8f133083d\nDisallow: /support/maps-beta\nDisallow: /support/market-images\nDisallow: /support/security\nDisallow: /support/webmaster-images\nDisallow: /support/*zout_\nDisallow: /support/search-results/\nDisallow: /support/praktikum/flow.html\nDisallow: /support/praktikum-english/\nDisallow: /telsearch\nDisallow: /themes\nDisallow: /toggle-experiment\nDisallow: /touchsearch\nDisallow: /tune*retpath=\nDisallow: /turbo\nAllow: /ugcpub/\nDisallow: /ugcpub/cabinet\nDisallow: /v$\nDisallow: /viewconfig$\nDisallow: /images/v$\nDisallow: /images/viewconfig$\nAllow: /tutor\nDisallow: /tutor/admin\nDisallow: /tutor/search/problems\nDisallow: /versions\nDisallow: /video/*\nAllow: /video/$\nAllow: /video/?\nAllow: /video/sitemap\nAllow: /video/movies/entity\nAllow: /video/touch/movies/entity\nAllow: /video/touch/$\nAllow: /video/touch/?\nAllow: /video/preview\nAllow: /video/touch/preview\nAllow: /video/search?text=video$\nAllow: /video/touch/search?text=video$\nAllow: /video/search?text=%D0%B2%D0%B8%D0%B4%D0%B5%D0%BE$\nAllow: /video/touch/search?text=%D0%B2%D0%B8%D0%B4%D0%B5%D0%BE$\nAllow: /video/video_sitemap_*.xml\nAllow: /video/movies_sitemap_*.xml\nDisallow: /xmlsearch\nDisallow: /yandsearch\nDisallow: /yca/cy\nDisallow: /soft/distribution\nDisallow: /soft/?*\nDisallow: /promo/launcher/?*\nDisallow: /opera/?*\nDisallow: /firefox/?*\nDisallow: /soft/bm/?*\nDisallow: /soft/browsers/?*\nDisallow: /soft/punto/?*\nDisallow: /ie/?*\nDisallow: /soft/*goodbye*\nDisallow: /adult\nDisallow: /addappterless\nDisallow: /archive/api/image.jpg*\nAllow: /archive$\nAllow: /archive/project/sovsport$\nDisallow: /all?\nDisallow: /games/*?*reqId*\nDisallow: /games/*?*pageId*\nDisallow: /games/*?experimental=\nDisallow: /games/*?*app-id=\nDisallow: /games/*?*game-id=\nDisallow: /games/*?*div-rtx-reqid=\nDisallow: /games/*?*header=\nDisallow: /games/*?*from=\nDisallow: /games/*?*clid=\nDisallow: /games/*?*deviceid=\nDisallow: /games/*?*flags=\nDisallow: /games/*?*inapp_id=\nDisallow: /games/*?*k50id=\nDisallow: /games/*?*lang=\nDisallow: /games/*?*payload=\nDisallow: /games/*?*pfid=\nDisallow: /games/*?*promo_id=\nDisallow: /games/*?*promo_intent=\nDisallow: /games/*?*referrer=\nDisallow: /games/*?*utm_\nDisallow: /games/*?*ypr=\nDisallow: /familysearch\nDisallow: /schoolsearch\nDisallow: /yandpage\nDisallow: *maps/covid19*\nDisallow: /promo/market/mobile/app*?from=\nDisallow: /support2/*zout_\nDisallow: /activity/\nDisallow: /sport\nDisallow: /support2/sport\n\nAllow: /medicine\nDisallow: /medicine/search\nDisallow: /medicine?text=\nDisallow: /finance\nAllow: /finance/reviews\nAllow: /finance/banks/*/reviews\nDisallow: /finance/reviews*?*\nDisallow: /finance/banks/*/reviews*?\n\nSitemap: https://yandex.ru/support/sitemap.xml\nSitemap: https://yandex.ru/blog/sitemap.xml\nSitemap: https://frontend.s3.yandex.net/health/sitemap.xml\nSitemap: https://yandex.ru/uslugi/sitemap/sitemap.xml.gz\nSitemap: https://yandex.ru/turbo/public/sitemap.xml\nSitemap: https://yandex.ru/tutor/sitemap/?sitemap=sitemap.xml\nSitemap: https://yandex.ru/turbo/talk-sitemap-index.xml\nSitemap: https://yandex.ru/games/sitemaps/sitemap.index.xml\nSitemap: https://yandex.ru/q/sitemap/sitemap-index.xml\nSitemap: https://yandex.ru/images/sitemap.xml\nSitemap: https://yandex.ru/medicine/sitemaps/sitemap.xml\nSitemap: https://yandex.ru/finance/sitemaps/sitemap.xml\nSitemap: https://yandex.ru/yandexapp/ru/callerid/sitemap.xml\nSitemap: https://yandex.ru/video/sitemap_index.xml\n\nUser-agent: Bingbot\nDisallow: /?\nDisallow: /403.html\nDisallow: /404.html\nDisallow: /500.html\nDisallow: /adddata\nDisallow: /adresa-segmentator\nDisallow: /advanced_engl.html\nDisallow: /advertising\nDisallow: /alice/term/\nDisallow: /alice/*/term/\nDisallow: /alice/old-browser\nDisallow: /alice/store/orders/\nDisallow: /alice*?\nDisallow: /alice/*&\nDisallow: /alice/chat/\nDisallow: /alice?share=\nDisallow: /all-supported-params\nDisallow: /articles\nDisallow: /aura/\nAllow: /aura/promo/\nDisallow: /blog/*?text=\nDisallow: /blog/*?tag=\nDisallow: /blog/*/*/*\nDisallow: /blog/punto\nDisallow: /MAPS/\nDisallow: /MaPS/\nDisallow: /MapS/\nDisallow: /Maps/\nDisallow: /mApS/\nDisallow: /maps-api/docs/js-api/reference-guide/instruction-gxs.html\nDisallow: /maps-api/docs/js-api/reference-guide/instruction-rs.html\nDisallow: /catalog/?text=\nDisallow: /chat\nDisallow: /cgi-bin/\nDisallow: /cgi/\nDisallow: /clck\nDisallow: /collections/\nDisallow: /company/*.rss\nDisallow: /company/search\nDisallow: /comments/*\nDisallow: /courier\nDisallow: /conflagexp\nDisallow: /cy\nDisallow: /cycounter\nDisallow: /dzen\nDisallow: /experiments.xml\nDisallow: /formfeedback\nDisallow: /fresh\nDisallow: /goto_issue/\nDisallow: /goto_rubric/\nDisallow: /i/\nDisallow: /images-data\nDisallow: /images.html\nDisallow: /images/*\nDisallow: /images-apphost/*\nAllow: /images/$\nAllow: /images/smart/$\nAllow: /images/touch/$\nAllow: /images/touch/?\nAllow: /images/search?text=*&source=morda*\nAllow: /images/search?rpt=imageview\nAllow: /images/touch/search?text=*&source=morda*\nAllow: /images/touch/search?rpt=imageview\nAllow: /images/pad/search?text=*&source=morda*\nAllow: /images/pad/search?rpt=imageview\nDisallow: /index_m\nDisallow: /iznanka/\nDisallow: /int_search_summary\nDisallow: /delivery_tracking\nDisallow: /gundem\nDisallow: /jobs/skill-diagnostic/private/*\nDisallow: /keyboard_qwerty.html\nDisallow: /logotypes\nDisallow: /business/widget\nDisallow: /local/\nDisallow: /lab/neurodetector\nDisallow: /lab/habr-detector\nDisallow: /more_samples\nDisallow: /msearch\nDisallow: /map-constructor/loader*\nDisallow: /maps/print/*\nAllow: /maps/*?lang=ru$\nAllow: /maps/*?lang=uk$\nAllow: /maps/*?lang=kk$\nAllow: /maps/*?lang=uz$\nDisallow: /nmaps/*?\nDisallow: /mapeditor/*?\nDisallow: /metro/*?from\nDisallow: /masters\nDisallow: /services/executor/\nDisallow: /n?\nDisallow: /n$\nDisallow: /n/\nDisallow: /norobot\nDisallow: /opensearch.xml\nDisallow: /padsearch\nDisallow: /person\nDisallow: /podpiska/login.pl\nDisallow: /pogoda/search\nDisallow: /pogoda/404\nDisallow: /polling\nDisallow: /poll/\nDisallow: /portal/set/\nDisallow: /portal/tune/\nDisallow: /prefetch\nDisallow: /promo/*goodbye*\nDisallow: /promo/diskelement\nDisallow: /promo/launcher/mgoodbye/*\nDisallow: /promo/*welcome*\nDisallow: /promo/yobject/changelog*\nDisallow: /project/eda/partnership_services/$\nDisallow: /q/profile/*/answers\nDisallow: /q/profile/*/questions\nDisallow: /q/profile/*/verifies\nDisallow: /q/profile/*/articles\nDisallow: /q/profile/*/subscribers\nDisallow: /q/profile/*/org_answers\nDisallow: /q/user/*/answers\nDisallow: /q/user/*/questions\nDisallow: /q/user/*/verifies\nDisallow: /q/user/*/articles\nDisallow: /q/user/*/subscribers\nDisallow: /q/user/*/org_answers\nDisallow: /soft/bm/goodbye\nDisallow: /soft/chrome/ext-install/*\nDisallow: /soft/chrome/searchline-install/*\nDisallow: /soft/*goodbye*\nDisallow: /soft/punto/mac/uninstall/*\nDisallow: /soft/punto/win/uninstall/*\nDisallow: /soft/*welcome*\nDisallow: /soft/win/?*\nDisallow: /quotes\nDisallow: /realty*?*pinned_offer_id=\nDisallow: /realty*?*pinned_nb_id=\nDisallow: /realty/search\nDisallow: /realty*?*text=\nDisallow: /realty/result\nDisallow: /realty/map_pins\nDisallow: /realty/map_dust\nDisallow: /realty/callme\nDisallow: /redir\nDisallow: /region_map\nDisallow: /regions_list.xml\nDisallow: /regions.html?\nDisallow: /s/\nDisallow: /save\nDisallow: /safety/?\nDisallow: /safety?\nDisallow: /search\nAllow: /search/entity/touch/\nAllow: /search?products_mode=1\nAllow: /search/?products_mode=1\nAllow: /search?*&products_mode=1\nAllow: /search/?*&products_mode=1\nAllow: /search/touch?products_mode=1\nAllow: /search/touch/?products_mode=1\nAllow: /search/touch?*&products_mode=1\nAllow: /search/touch/?*&products_mode=1\nAllow: /search/products\nAllow: /products/$\nAllow: /products$\nDisallow: /setup\nDisallow: /showcaptcha\nDisallow: /sitesearch\nDisallow: /sl/*.html\nDisallow: /soft/extensions/goodbye\nDisallow: /sportagent\nDisallow: /storeclick\nDisallow: /storerequest\nDisallow: /subscribe/confirm.pl\nDisallow: /subscribe/view.pl\nAllow: /support/\nDisallow: /support/direct-images\nDisallow: /support/direct-tooltips\nDisallow: /support/distr\nDisallow: /support/dsp\nDisallow: /support/fe4be44a295cc679e19bf0b8f133083d\nDisallow: /support/maps-beta\nDisallow: /support/market-images\nDisallow: /support/security\nDisallow: /support/webmaster-images\nDisallow: /support/*zout_\nDisallow: /support/search-results/\nDisallow: /support/praktikum/flow.html\nDisallow: /support/praktikum-english/\nDisallow: /telsearch\nDisallow: /themes\nDisallow: /toggle-experiment\nDisallow: /touchsearch\nDisallow: /tune*retpath=\nAllow: /ugcpub/\nDisallow: /ugcpub/cabinet\nDisallow: /v$\nDisallow: /viewconfig$\nDisallow: /images/v$\nDisallow: /images/viewconfig$\nAllow: /tutor\nDisallow: /tutor/admin\nDisallow: /tutor/search/problems\nDisallow: /turbo\nDisallow: /versions\nDisallow: /video/*\nAllow: /video/$\nAllow: /video/?\nAllow: /video/sitemap\nAllow: /video/movies/entity\nAllow: /video/touch/movies/entity\nAllow: /video/touch/$\nAllow: /video/touch/?\nAllow: /video/preview\nAllow: /video/touch/preview\nAllow: /video/search?text=video$\nAllow: /video/touch/search?text=video$\nAllow: /video/search?text=%D0%B2%D0%B8%D0%B4%D0%B5%D0%BE$\nAllow: /video/touch/search?text=%D0%B2%D0%B8%D0%B4%D0%B5%D0%BE$\nAllow: /video/video_sitemap_*.xml\nAllow: /video/movies_sitemap_*.xml\nDisallow: /xmlsearch\nDisallow: /yandsearch\nDisallow: /yca/cy\nDisallow: /soft/distribution\nDisallow: /soft/?*\nDisallow: /promo/launcher/?*\nDisallow: /opera/?*\nDisallow: /firefox/?*\nDisallow: /soft/bm/?*\nDisallow: /soft/browsers/?*\nDisallow: /soft/punto/?*\nDisallow: /ie/?*\nDisallow: /soft/*goodbye*\nDisallow: /adult\nDisallow: /addappterless\nDisallow: /archive/api/image.jpg*\nAllow: /archive$\nAllow: /archive/project/sovsport$\nDisallow: /all?\nDisallow: /games/*?*reqId*\nDisallow: /games/*?*pageId*\nDisallow: /games/*?experimental=\nDisallow: /games/*?*app-id=\nDisallow: /games/*?*game-id=\nDisallow: /games/*?*div-rtx-reqid=\nDisallow: /games/*?*header=\nDisallow: /games/*?*from=\nDisallow: /games/*?*clid=\nDisallow: /games/*?*deviceid=\nDisallow: /games/*?*flags=\nDisallow: /games/*?*inapp_id=\nDisallow: /games/*?*k50id=\nDisallow: /games/*?*lang=\nDisallow: /games/*?*payload=\nDisallow: /games/*?*pfid=\nDisallow: /games/*?*promo_id=\nDisallow: /games/*?*promo_intent=\nDisallow: /games/*?*referrer=\nDisallow: /games/*?*utm_\nDisallow: /games/*?*ypr=\nDisallow: /products\nDisallow: /familysearch\nDisallow: /schoolsearch\nDisallow: /yandpage\nDisallow: *maps/covid19*\nDisallow: /promo/market/mobile/app*?from=\nDisallow: /support2/*zout_\nDisallow: /activity/\nDisallow: /sport\nDisallow: /support2/sport*\n\nAllow: /medicine\nDisallow: /medicine/search\nDisallow: /medicine?text=\nDisallow: /finance*?\nDisallow: /finance/assistant\n\nUser-agent: Yandex\nDisallow: /?\nDisallow: /403.html\nDisallow: /404.html\nDisallow: /500.html\nDisallow: /adddata\nDisallow: /adresa-segmentator\nDisallow: /advanced_engl.html\nDisallow: /advertising\nDisallow: /adv/news/vystavka-prodazha-na-poiske-poyavilas-tovarnaya-galereya\nDisallow: /adv?nw=\nDisallow: /adv?from=\nDisallow: /adv?email_form=\nDisallow: /adv?ref=\nDisallow: /alice/term/\nDisallow: /alice/*/term/\nDisallow: /alice/old-browser\nDisallow: /alice/chat/\nDisallow: /alice?share=\nDisallow: /all-supported-params\nDisallow: /articles\nDisallow: /aura/\nAllow: /aura/promo/\nDisallow: /blog/*?text=\nDisallow: /blog/*?tag=\nDisallow: /blog/*/*/*\nDisallow: /blog/punto\nDisallow: /MAPS/\nDisallow: /MaPS/\nDisallow: /MapS/\nDisallow: /Maps/\nDisallow: /mApS/\nDisallow: /catalog/?text=\nDisallow: /chat\nDisallow: /cgi-bin/\nDisallow: /cgi/\nDisallow: /clck\nDisallow: /collections/\nDisallow: /company/*.rss\nDisallow: /company/search\nDisallow: /business/widget\nDisallow: /comments/*\nDisallow: /courier\nDisallow: /products\nDisallow: /conflagexp\nDisallow: /cy\nDisallow: /cycounter\nDisallow: /dzen\nDisallow: /experiments.xml\nDisallow: /formfeedback\nDisallow: /fresh\nDisallow: /games/*?experimental=\nDisallow: /goto_issue/\nDisallow: /goto_rubric/\nDisallow: /i/\nDisallow: /images-data\nDisallow: /images.html\nDisallow: /images/*\nAllow: /images/$\nAllow: /images/smart/$\nAllow: /images/touch/$\nAllow: /images/touch/?\nAllow: /images/search?text=*&source=morda*\nAllow: /images/search?rpt=imageview\nAllow: /images/touch/search?text=*&source=morda*\nAllow: /images/touch/search?rpt=imageview\nAllow: /images/pad/search?text=*&source=morda*\nAllow: /images/pad/search?rpt=imageview\nDisallow: /index_m\nDisallow: /iznanka/\nDisallow: /int_search_summary\nDisallow: /delivery_tracking\nDisallow: /jobs/skill-diagnostic/private/*\nDisallow: /keyboard_qwerty.html\nDisallow: /logotypes\nDisallow: /local/\nDisallow: /lab/neurodetector\nDisallow: /lab/habr-detector\nDisallow: /more_samples\nDisallow: /msearch\nDisallow: /map-constructor/loader*\nDisallow: /maps/print/*\nDisallow: /harita/business/\nDisallow: /maps/business/\nDisallow: /maps-api/docs/js-api/reference-guide/instruction-gxs.html\nDisallow: /maps-api/docs/js-api/reference-guide/instruction-rs.html\nAllow: /maps/*?lang=ru$\nAllow: /maps/*?lang=uk$\nAllow: /maps/org/\nDisallow: /nmaps/*?\nDisallow: /mapeditor/*?\nDisallow: /metro/*?from\nDisallow: /masters\nDisallow: /services\nDisallow: /neuralsearch/api\nDisallow: /n?\nDisallow: /n$\nDisallow: /n/\nDisallow: /norobot\nDisallow: /opensearch.xml\nDisallow: /padsearch\nDisallow: /person\nDisallow: /podpiska/login.pl\nDisallow: /pogoda/amp/*\nDisallow: /pogoda/search\nDisallow: /pogoda/404\nDisallow: /pogoda/0\nDisallow: /polling\nDisallow: /poll/\nDisallow: /portal/set/\nDisallow: /portal/tune/\nDisallow: /promo/*goodbye*\nDisallow: /promo/diskelement\nDisallow: /promo/launcher/mgoodbye/*\nDisallow: /promo/*welcome*\nDisallow: /promo/yobject/changelog*\nDisallow: /project/eda/partnership_services/$\nDisallow: /q/profile/*/answers\nDisallow: /q/profile/*/questions\nDisallow: /q/profile/*/verifies\nDisallow: /q/profile/*/articles\nDisallow: /q/profile/*/subscribers\nDisallow: /q/profile/*/org_answers\nDisallow: /q/user/*/answers\nDisallow: /q/user/*/questions\nDisallow: /q/user/*/verifies\nDisallow: /q/user/*/articles\nDisallow: /q/user/*/subscribers\nDisallow: /q/user/*/org_answers\nDisallow: /soft/bm/goodbye\nDisallow: /soft/chrome/ext-install/*\nDisallow: /soft/chrome/searchline-install/*\nDisallow: /soft/*goodbye*\nDisallow: /soft/punto/mac/uninstall/*\nDisallow: /soft/punto/win/uninstall/*\nDisallow: /soft/*welcome*\nDisallow: /soft/win/?*\nDisallow: /quotes\nDisallow: /realty\nDisallow: /redir\nDisallow: /region_map\nDisallow: /regions_list.xml\nDisallow: /regions.html?\nDisallow: /s/\nDisallow: /save\nDisallow: /safety/?*\nDisallow: /search\nAllow: /search/entity/touch/\nDisallow: /setup\nDisallow: /showcaptcha\nDisallow: /sitesearch\nDisallow: /sl/*.html\nDisallow: /soft/extensions/goodbye\nDisallow: /sportagent\nDisallow: /storeclick\nDisallow: /storerequest\nDisallow: /subscribe/confirm.pl\nDisallow: /subscribe/view.pl\nAllow: /support/\nDisallow: /support/direct-images\nDisallow: /support/direct-tooltips\nDisallow: /support/distr\nDisallow: /support/dsp\nDisallow: /support/fe4be44a295cc679e19bf0b8f133083d\nDisallow: /support/maps-beta\nDisallow: /support/market-images\nDisallow: /support/security\nDisallow: /support/webmaster-images\nDisallow: /support/*zout_\nDisallow: /support/search-results/\nDisallow: /support/praktikum/flow.html\nDisallow: /support/praktikum-english/\nDisallow: /telsearch\nDisallow: /themes\nDisallow: /toggle-experiment\nDisallow: /touchsearch\nDisallow: /tune*retpath=\nDisallow: /8bit-fest/parents\nAllow: /tutor\nDisallow: /tutor/admin\nDisallow: /turbo\nAllow: /ugcpub/\nDisallow: /ugcpub/cabinet\nDisallow: /v$\nDisallow: /viewconfig$\nDisallow: /images/v$\nDisallow: /images/viewconfig$\nDisallow: /versions\nDisallow: /video/*\nAllow: /video/$\nAllow: /video/?\nAllow: /video/sitemap\nAllow: /video/touch/$\nAllow: /video/touch/?\nAllow: /video/preview\nAllow: /video/touch/preview\nAllow: /video/search?text=video$\nAllow: /video/touch/search?text=video$\nAllow: /video/search?text=%D0%B2%D0%B8%D0%B4%D0%B5%D0%BE$\nAllow: /video/touch/search?text=%D0%B2%D0%B8%D0%B4%D0%B5%D0%BE$\nAllow: /video/video_sitemap_*.xml\nAllow: /video/movies_sitemap_*.xml\nDisallow: /xmlsearch\nDisallow: /yandsearch\nDisallow: /yca/cy\nDisallow: /soft/distribution\nDisallow: /soft/?*\nDisallow: /promo/launcher/?*\nDisallow: /opera/?*\nDisallow: /firefox/?*\nDisallow: /soft/bm/?*\nDisallow: /soft/browsers/?*\nDisallow: /soft/punto/?*\nDisallow: /ie/?*\nDisallow: /soft/*goodbye*\nDisallow: /adult\nDisallow: /addappterless\nDisallow: /archive/api/image.jpg*\nAllow: /archive$\nAllow: /archive/project/sovsport$\nDisallow: /all?\nDisallow: /familysearch\nDisallow: /schoolsearch\nDisallow: /yandpage\nDisallow: *maps/covid19*\nDisallow: /promo/market/mobile/app*?from=\nDisallow: /support2/*zout_\nDisallow: /activity/\nDisallow: /sport\nDisallow: /support2/sport*\n\nDisallow: /medicine\nDisallow: /finance\n\nDisallow: /gundem\n\nUser-agent: Googlebot\nDisallow: /?\nDisallow: /403.html\nDisallow: /404.html\nDisallow: /500.html\nDisallow: /adddata\nDisallow: /adresa-segmentator\nDisallow: /advanced_engl.html\nDisallow: /advertising\nDisallow: /ads/\nDisallow: /adfox/\nDisallow: /an/\nDisallow: /alice/term/\nDisallow: /alice/*/term/\nDisallow: /alice/old-browser\nDisallow: /alice/store/orders/\nDisallow: /alice*?\nDisallow: /alice/*&\nDisallow: /alice/chat/\nDisallow: /alice?share=\nDisallow: /all-supported-params\nDisallow: /articles\nDisallow: /archive/search\nDisallow: /aura/\nAllow: /aura/promo/\nDisallow: /blog/*?text=\nDisallow: /blog/*?tag=\nDisallow: /blog/*/*/*\nDisallow: /blog/punto\nDisallow: /business/widget\nDisallow: /MAPS/\nDisallow: /MaPS/\nDisallow: /MapS/\nDisallow: /Maps/\nDisallow: /mApS/\nDisallow: /maps-api/docs/js-api/reference-guide/instruction-gxs.html\nDisallow: /maps-api/docs/js-api/reference-guide/instruction-rs.html\nDisallow: /catalog/?text=\nDisallow: /cgi-bin/\nDisallow: /cgi/\nDisallow: /clck\nDisallow: /collections/\nDisallow: /company/*.rss\nDisallow: /company/search\nDisallow: /comments/*\nDisallow: /courier\nDisallow: /conflagexp\nDisallow: /cy\nDisallow: /cycounter\nDisallow: /dzen\nDisallow: /experiments.xml\nDisallow: /formfeedback\nDisallow: /fresh\nDisallow: /goto_issue/\nDisallow: /goto_rubric/\nDisallow: /gundem\nDisallow: /i/\nDisallow: /images-data\nDisallow: /images.html\nDisallow: /images/\nDisallow: /images-apphost/\nAllow: /images$\nAllow: /images/$\nAllow: /images/?\nAllow: /images/smart/$\nAllow: /images/touch$\nAllow: /images/touch/$\nAllow: /images/touch/?\nAllow: /images/pad/search?text=*&source=morda*\nAllow: /images/pad/search?rpt=imageview\nAllow: /images/search?\nAllow: /images/touch/search?\n\nAllow: /images/sitemap\nDisallow: /index_m\nDisallow: /iznanka/\nDisallow: /int_search_summary\nDisallow: /delivery_tracking\nDisallow: /jobs/skill-diagnostic/private/*\nDisallow: /keyboard_qwerty.html\nDisallow: /logotypes\nDisallow: /local/\nDisallow: /lab/neurodetector\nDisallow: /lab/habr-detector\nDisallow: /more_samples\nDisallow: /msearch\nDisallow: /map-constructor/loader*\nDisallow: /maps/print/*\nDisallow: /harita/business/\nDisallow: /maps/business/\nAllow: /maps/*?lang=ru$\nAllow: /maps/*?lang=uk$\nAllow: /maps/*?lang=kk$\nAllow: /maps/*?lang=uz$\nDisallow: /nmaps/*?\nDisallow: /mapeditor/*?\nDisallow: /metro/*?from\nDisallow: /masters\nDisallow: /services/executor/\nDisallow: /neuralsearch/api\nDisallow: /n?\nDisallow: /n$\nDisallow: /n/\nDisallow: /norobot\nDisallow: /opensearch.xml\nDisallow: /padsearch\nAllow: /products/sitemaps/new/*\nDisallow: /person\nDisallow: /podpiska/login.pl\nDisallow: /pogoda/404\nDisallow: /polling\nDisallow: /poll/\nDisallow: /portal/set/\nDisallow: /portal/tune/\nDisallow: /prefetch\nDisallow: /products\nDisallow: /promo/*goodbye*\nDisallow: /promo/diskelement\nDisallow: /promo/launcher/mgoodbye/*\nDisallow: /promo/*welcome*\nDisallow: /promo/yobject/changelog*\nDisallow: /project/eda/partnership_services/$\nDisallow: /q/profile/*/answers\nDisallow: /q/profile/*/questions\nDisallow: /q/profile/*/verifies\nDisallow: /q/profile/*/articles\nDisallow: /q/profile/*/subscribers\nDisallow: /q/profile/*/org_answers\nDisallow: /q/user/*/answers\nDisallow: /q/user/*/questions\nDisallow: /q/user/*/verifies\nDisallow: /q/user/*/articles\nDisallow: /q/user/*/subscribers\nDisallow: /q/user/*/org_answers\nDisallow: /soft/bm/goodbye\nDisallow: /soft/chrome/ext-install/*\nDisallow: /soft/chrome/searchline-install/*\nDisallow: /soft/*goodbye*\nDisallow: /soft/punto/mac/uninstall/*\nDisallow: /soft/punto/win/uninstall/*\nDisallow: /soft/*welcome*\nDisallow: /soft/win/?*\nDisallow: /quotes\nDisallow: /realty*?*pinned_offer_id=\nDisallow: /realty*?*pinned_nb_id=\nDisallow: /realty/search\nDisallow: /realty*?*text=\nDisallow: /realty/result\nDisallow: /realty/map_pins\nDisallow: /realty/map_dust\nDisallow: /realty/callme\nDisallow: /redir\nDisallow: /region_map\nDisallow: /regions_list.xml\nDisallow: /regions.html?\nDisallow: /s/\nDisallow: /save\nDisallow: /safety/?\nDisallow: /safety?\nDisallow: /search\nAllow: /search/entity/touch/\nAllow: /search?products_mode=1\nAllow: /search/?products_mode=1\nAllow: /search?*&products_mode=1\nAllow: /search/?*&products_mode=1\nAllow: /search/touch?products_mode=1\nAllow: /search/touch/?products_mode=1\nAllow: /search/touch?*&products_mode=1\nAllow: /search/touch/?*&products_mode=1\nAllow: /search/products\nAllow: /products/$\nAllow: /products$\nDisallow: /setup\nDisallow: /showcaptcha\nDisallow: /sitesearch\nDisallow: /sl/*.html\nDisallow: /soft/extensions/goodbye\nDisallow: /sportagent\nDisallow: /storeclick\nDisallow: /storerequest\nDisallow: /subscribe/confirm.pl\nDisallow: /subscribe/view.pl\nAllow: /support/\nDisallow: /support/direct-images\nDisallow: /support/direct-tooltips\nDisallow: /support/distr\nDisallow: /support/dsp\nDisallow: /support/fe4be44a295cc679e19bf0b8f133083d\nDisallow: /support/maps-beta\nDisallow: /support/market-images\nDisallow: /support/security\nDisallow: /support/webmaster-images\nDisallow: /support/*zout_\nDisallow: /support/search-results/\nDisallow: /support/praktikum/flow.html\nDisallow: /support/praktikum-english/\nDisallow: /telsearch\nDisallow: /themes\nDisallow: /toggle-experiment\nDisallow: /touchsearch\nDisallow: /tune*retpath=\nDisallow: /turbo\nAllow: /ugcpub/\nDisallow: /ugcpub/cabinet\nDisallow: /v$\nDisallow: /viewconfig$\nDisallow: /images/v$\nDisallow: /images/viewconfig$\nAllow: /tutor\nDisallow: /tutor/admin\nDisallow: /tutor/search/problems\nDisallow: /versions\nDisallow: /video/\nAllow: /video/$\nAllow: /video/?\nAllow: /video/sitemap\nAllow: /video/movies/entity\nAllow: /video/touch/movies/entity\nAllow: /video/touch/$\nAllow: /video/touch/?\nAllow: /video/preview\nAllow: /video/touch/preview\nAllow: /video/search?text=video$\nAllow: /video/touch/search?text=video$\nAllow: /video/search?text=%D0%B2%D0%B8%D0%B4%D0%B5%D0%BE$\nAllow: /video/touch/search?text=%D0%B2%D0%B8%D0%B4%D0%B5%D0%BE$\nAllow: /video/movies/catalog\nAllow: /video/touch/movies/catalog\nAllow: /video/video_sitemap_*.xml\nAllow: /video/movies_sitemap_*.xml\nDisallow: /xmlsearch\nDisallow: /yandsearch\nDisallow: /yca/cy\nDisallow: /soft/distribution\nDisallow: /soft/?*\nDisallow: /promo/launcher/?*\nDisallow: /opera/?*\nDisallow: /firefox/?*\nDisallow: /soft/bm/?*\nDisallow: /soft/browsers/?*\nDisallow: /soft/punto/?*\nDisallow: /ie/?*\nDisallow: /soft/*goodbye*\nDisallow: /adult\nDisallow: /addappterless\nDisallow: /archive/api/image.jpg*\nAllow: /archive$\nAllow: /archive/project/sovsport$\nDisallow: /all?\nDisallow: /games/*?*reqId*\nDisallow: /games/*?*pageId*\nDisallow: /games/*?experimental=\nDisallow: /games/*?*app-id=\nDisallow: /games/*?*game-id=\nDisallow: /games/*?*div-rtx-reqid=\nDisallow: /games/*?*header=\nDisallow: /games/*?*from=\nDisallow: /games/*?*clid=\nDisallow: /games/*?*deviceid=\nDisallow: /games/*?*flags=\nDisallow: /games/*?*inapp_id=\nDisallow: /games/*?*k50id=\nDisallow: /games/*?*lang=\nDisallow: /games/*?*payload=\nDisallow: /games/*?*pfid=\nDisallow: /games/*?*promo_id=\nDisallow: /games/*?*promo_intent=\nDisallow: /games/*?*referrer=\nDisallow: /games/*?*utm_\nDisallow: /games/*?*ypr=\nDisallow: /familysearch\nDisallow: /schoolsearch\nDisallow: /yandpage\nDisallow: *maps/covid19*\nDisallow: /promo/market/mobile/app*?from=\nDisallow: /support2/*zout_\nDisallow: /activity/\nDisallow: /sport\nDisallow: /support2/sport*\n\nAllow: /medicine\nDisallow: /medicine/search\nDisallow: /medicine?text=\nDisallow: /finance*?\nDisallow: /finance/assistant\n\nUser-agent: OAI-SearchBot\nDisallow: /?\nDisallow: /403.html\nDisallow: /404.html\nDisallow: /500.html\nDisallow: /adddata\nDisallow: /adresa-segmentator\nDisallow: /advanced_engl.html\nDisallow: /advertising\nDisallow: /alice/term/\nDisallow: /alice/*/term/\nDisallow: /alice/old-browser\nDisallow: /alice/store/orders/\nDisallow: /alice*?\nDisallow: /alice/*&\nDisallow: /alice/chat/\nDisallow: /alice?share=\nDisallow: /all-supported-params\nDisallow: /articles\nDisallow: /aura/\nAllow: /aura/promo/\nDisallow: /blog/*?text=\nDisallow: /blog/*?tag=\nDisallow: /blog/*/*/*\nDisallow: /blog/punto\nDisallow: /MAPS/\nDisallow: /MaPS/\nDisallow: /MapS/\nDisallow: /Maps/\nDisallow: /mApS/\nDisallow: /harita/business/\nDisallow: /maps/business/\nDisallow: /maps-api/docs/js-api/reference-guide/instruction-gxs.html\nDisallow: /maps-api/docs/js-api/reference-guide/instruction-rs.html\nDisallow: /catalog/?text=\nDisallow: /chat\nDisallow: /cgi-bin/\nDisallow: /cgi/\nDisallow: /clck\nDisallow: /collections/\nDisallow: /company/*.rss\nDisallow: /company/search\nDisallow: /comments/*\nDisallow: /courier\nDisallow: /conflagexp\nDisallow: /cy\nDisallow: /cycounter\nDisallow: /dzen\nDisallow: /experiments.xml\nDisallow: /formfeedback\nDisallow: /fresh\nDisallow: /goto_issue/\nDisallow: /goto_rubric/\nDisallow: /gundem\nDisallow: /i/\nDisallow: /images-data\nDisallow: /images.html\nDisallow: /images/*\nDisallow: /images-apphost/*\nAllow: /images/$\nAllow: /images/smart/$\nAllow: /images/touch/$\nAllow: /images/touch/?\nAllow: /images/search?text=*&source=morda*\nAllow: /images/search?rpt=imageview\nAllow: /images/touch/search?text=*&source=morda*\nAllow: /images/touch/search?rpt=imageview\nAllow: /images/pad/search?text=*&source=morda*\nAllow: /images/pad/search?rpt=imageview\nDisallow: /index_m\nDisallow: /iznanka/\nDisallow: /int_search_summary\nDisallow: /delivery_tracking\nDisallow: /jobs/skill-diagnostic/private/*\nDisallow: /keyboard_qwerty.html\nDisallow: /logotypes\nDisallow: /business/widget\nDisallow: /local/\nDisallow: /lab/neurodetector\nDisallow: /lab/habr-detector\nDisallow: /more_samples\nDisallow: /msearch\nDisallow: /map-constructor/loader*\nDisallow: /maps/print/*\nAllow: /maps/*?lang=ru$\nAllow: /maps/*?lang=uk$\nAllow: /maps/*?lang=kk$\nAllow: /maps/*?lang=uz$\nDisallow: /nmaps/*?\nDisallow: /mapeditor/*?\nDisallow: /metro/*?from\nDisallow: /masters\nDisallow: /services/executor/\nDisallow: /n?\nDisallow: /n$\nDisallow: /n/\nDisallow: /neuralsearch/api\nDisallow: /norobot\nDisallow: /opensearch.xml\nDisallow: /padsearch\nDisallow: /person\nDisallow: /podpiska/login.pl\nDisallow: /pogoda/search\nDisallow: /pogoda/404\nDisallow: /pogoda/0\nDisallow: /polling\nDisallow: /poll/\nDisallow: /portal/set/\nDisallow: /portal/tune/\nDisallow: /prefetch\nDisallow: /promo/*goodbye*\nDisallow: /promo/diskelement\nDisallow: /promo/launcher/mgoodbye/*\nDisallow: /promo/*welcome*\nDisallow: /promo/yobject/changelog*\nDisallow: /project/eda/partnership_services/$\nDisallow: /q/profile/*/answers\nDisallow: /q/profile/*/questions\nDisallow: /q/profile/*/verifies\nDisallow: /q/profile/*/articles\nDisallow: /q/profile/*/subscribers\nDisallow: /q/profile/*/org_answers\nDisallow: /q/user/*/answers\nDisallow: /q/user/*/questions\nDisallow: /q/user/*/verifies\nDisallow: /q/user/*/articles\nDisallow: /q/user/*/subscribers\nDisallow: /q/user/*/org_answers\nDisallow: /soft/bm/goodbye\nDisallow: /soft/chrome/ext-install/*\nDisallow: /soft/chrome/searchline-install/*\nDisallow: /soft/*goodbye*\nDisallow: /soft/punto/mac/uninstall/*\nDisallow: /soft/punto/win/uninstall/*\nDisallow: /soft/*welcome*\nDisallow: /soft/win/?*\nDisallow: /quotes\nDisallow: /realty*?*pinned_offer_id=\nDisallow: /realty*?*pinned_nb_id=\nDisallow: /realty/search\nDisallow: /realty*?*text=\nDisallow: /realty/result\nDisallow: /realty/map_pins\nDisallow: /realty/map_dust\nDisallow: /realty/callme\nDisallow: /redir\nDisallow: /region_map\nDisallow: /regions_list.xml\nDisallow: /regions.html?\nDisallow: /s/\nDisallow: /save\nDisallow: /safety/?\nDisallow: /safety?\nDisallow: /search\nAllow: /search/entity/touch/\nDisallow: /setup\nDisallow: /showcaptcha\nDisallow: /sitesearch\nDisallow: /sl/*.html\nDisallow: /soft/extensions/goodbye\nDisallow: /sportagent\nDisallow: /storeclick\nDisallow: /storerequest\nDisallow: /subscribe/confirm.pl\nDisallow: /subscribe/view.pl\nAllow: /support/\nDisallow: /support/direct-images\nDisallow: /support/direct-tooltips\nDisallow: /support/distr\nDisallow: /support/dsp\nDisallow: /support/fe4be44a295cc679e19bf0b8f133083d\nDisallow: /support/maps-beta\nDisallow: /support/market-images\nDisallow: /support/security\nDisallow: /support/webmaster-images\nDisallow: /support/*zout_\nDisallow: /support/search-results/\nDisallow: /support/praktikum/flow.html\nDisallow: /support/praktikum-english/\nDisallow: /telsearch\nDisallow: /themes\nDisallow: /toggle-experiment\nDisallow: /touchsearch\nDisallow: /tune*retpath=\nDisallow: /turbo\nAllow: /ugcpub/\nDisallow: /ugcpub/cabinet\nDisallow: /v$\nDisallow: /viewconfig$\nDisallow: /images/v$\nDisallow: /images/viewconfig$\nAllow: /tutor\nDisallow: /tutor/admin\nDisallow: /tutor/search/problems\nDisallow: /versions\nDisallow: /video/*\nAllow: /video/$\nAllow: /video/?\nAllow: /video/sitemap\nAllow: /video/movies/entity\nAllow: /video/touch/movies/entity\nAllow: /video/touch/$\nAllow: /video/touch/?\nAllow: /video/preview\nAllow: /video/touch/preview\nAllow: /video/search?text=video$\nAllow: /video/touch/search?text=video$\nAllow: /video/search?text=%D0%B2%D0%B8%D0%B4%D0%B5%D0%BE$\nAllow: /video/touch/search?text=%D0%B2%D0%B8%D0%B4%D0%B5%D0%BE$\nAllow: /video/video_sitemap_*.xml\nAllow: /video/movies_sitemap_*.xml\nDisallow: /xmlsearch\nDisallow: /yandsearch\nDisallow: /yca/cy\nDisallow: /soft/distribution\nDisallow: /soft/?*\nDisallow: /promo/launcher/?*\nDisallow: /opera/?*\nDisallow: /firefox/?*\nDisallow: /soft/bm/?*\nDisallow: /soft/browsers/?*\nDisallow: /soft/punto/?*\nDisallow: /ie/?*\nDisallow: /soft/*goodbye*\nDisallow: /adult\nDisallow: /addappterless\nDisallow: /archive/api/image.jpg*\nAllow: /archive$\nAllow: /archive/project/sovsport$\nDisallow: /all?\nDisallow: /games/*?*reqId*\nDisallow: /games/*?*pageId*\nDisallow: /games/*?experimental=\nDisallow: /games/*?*app-id=\nDisallow: /games/*?*game-id=\nDisallow: /games/*?*div-rtx-reqid=\nDisallow: /games/*?*header=\nDisallow: /games/*?*from=\nDisallow: /games/*?*clid=\nDisallow: /games/*?*deviceid=\nDisallow: /games/*?*flags=\nDisallow: /games/*?*inapp_id=\nDisallow: /games/*?*k50id=\nDisallow: /games/*?*lang=\nDisallow: /games/*?*payload=\nDisallow: /games/*?*pfid=\nDisallow: /games/*?*promo_id=\nDisallow: /games/*?*promo_intent=\nDisallow: /games/*?*referrer=\nDisallow: /games/*?*utm_\nDisallow: /games/*?*ypr=\nDisallow: /familysearch\nDisallow: /schoolsearch\nDisallow: /yandpage\nDisallow: *maps/covid19*\nDisallow: /promo/market/mobile/app*?from=\nDisallow: /support2/*zout_\nDisallow: /activity/\nDisallow: /sport\nDisallow: /support2/sport\n\nAllow: /medicine\nDisallow: /medicine/search\nDisallow: /medicine?text=\nDisallow: /finance\nAllow: /finance/reviews\nAllow: /finance/banks/*/reviews\nDisallow: /finance/reviews*?*\nDisallow: /finance/banks/*/reviews*?\n\nSitemap: https://yandex.ru/pogoda/sitemap-index.xml\nSitemap: https://yandex.ru/support/sitemap.xml\nSitemap: https://yandex.ru/blog/sitemap.xml\nSitemap: https://frontend.s3.yandex.net/health/sitemap.xml\nSitemap: https://yandex.ru/uslugi/sitemap/sitemap.xml.gz\nSitemap: https://yandex.ru/turbo/public/sitemap.xml\nSitemap: https://yandex.ru/tutor/sitemap/?sitemap=sitemap.xml\nSitemap: https://yandex.ru/games/sitemaps/sitemap.index.xml\nSitemap: https://yandex.ru/q/sitemap/sitemap-index.xml\nSitemap: https://yandex.ru/images/sitemap.xml\nSitemap: https://yandex.ru/medicine/sitemaps/sitemap.xml\nSitemap: https://yandex.ru/yandexapp/ru/callerid/sitemap.xml\nSitemap: https://yandex.ru/video/sitemap_index.xml\nSitemap: https://yandex.ru/realty/sitemaps/human_friendly/sitemap_index_hfp-00001.xml\nSitemap: https://yandex.ru/images/sitemap_index.xml\nSitemap: https://yandex.ru/services/sitemaps/sitemap.xml\nSitemap: https://yandex.ru/archive/sitemap.xml\n\nClean-Param: ncrnd&redircnt&clid&_&win /\nClean-Param: utm_source&utm_medium&utm_campaign&utm_term&utm_content&utm_nooverride&utm_expid&gclid&_openstat&yclid&gsaid&_gs_rmark&_gs_ref&_gs_cttl&iframe /health/\nClean-Param: print=1 /tutor/\nClean-Param: w&w_question_id /q/question/*\nClean-Param: &appsearch_header=1&comments=1 /news/story/\nClean-Param: app-id&game-id&div-rtx-reqid&lang&platform&header&from&backQueryParams /games/\nClean-Param: ll&z&sll /maps/\nClean-param: oll&rtext&pt&source&via&from&origin&um&utm_medium&utm_source&portal&panorama%5Bspan%5D&panorama%5Bpoint%5D&panorama%5Bfull%5D&panorama%5Bdirection%5D&panorama%5Bair%5D&masstransit%5BstopId%5D&ol&poi%5Bpoint%5D&whatshere%5Bpoint%5D&whatshere%5Bzoom%5D&ouri&ruri&l&mode&tab&rtt&rtm&display-text&text&indoorLevel&trfm&trfst&no-distribution&distance&slices&scale&descy&descx&mapY&mapX&mapID&sctx&lay&where&results&map&id&spn&sspn&rtn&ost&profile-mode /maps/\nClean-Param: appsearch_header /pogoda\nClean-param: from&publication_id&email_form&id&page&test&year&month&lang&onlyWithCases&gcmes&cmnt_login&_aid&currentURL&res&cmnt_from&amp&comments&certId&mindbox-message-key&status&menu&block&version&ref&etext&ctime&nw&commentId&yqrid&erid&specializations&specializations[]&cities&workFormats&product&filters&autm_medium&industries&search_open&tasks&search_text&region&region_name&lastname /adv/\nClean-param: fl_checkpoint&clckid&begin&yprqee&products&yadclid&yadordid&direct&case_products&wvstest&modal&suggest_reqid&only-content&tb&source&p&m-message-key-id&journal&date&months[]&answer_email&url&intermedia&industry&service&country&K1[K2]&lr&budgetOrder&certificate&utm_source&utm_campaign&utm_medium&utm_term&utm_content /adv/\nClean-param: z&ll&mapTypeRoute&slug&uuid&app&is_user_location&app_with_layers&app_soyuz&from&dateRoute&clid&via&city&amp&topic&lat&lon&lang&le_Lightning&from&utm_source&utm_campaign&utm_medium&utm_content&utm_term /pogoda/\nClean-param: z&ll&mapTypeRoute&slug&uuid&app&is_user_location&app_with_layers&app_soyuz&from&dateRoute&clid&via&city&amp&topic&lat&lon&lang&le_Lightning&from&utm_source&utm_campaign&utm_medium&utm_content&utm_term /hava/\nClean-param: z&ll&mapTypeRoute&slug&uuid&app&is_user_location&app_with_layers&app_soyuz&from&dateRoute&clid&via&city&amp&topic&lat&lon&lang&le_Lightning&from&utm_source&utm_campaign&utm_medium&utm_content&utm_term /weather/\nClean-param: nw&from_block&mode&mar&photoType&regid /pogoda/\nClean-param: nw&from_block&mode&mar&photoType&regid /hava/\nClean-param: nw&from_block&mode&mar&photoType&regid /weather/\nClean-param: from_pumpkin&le_WindParticles&nowcast&yw_preset&gen_time&for_date&morda_geolocation&retpath&acc_maps&le_TemperatureBalloons&request&le_Borders&elmt&may&source&ids&june&july&referrer&PAGEN_1&from_render_fallback&preset&to&august&appsearch_ys&region&tm&banner-test-tags /pogoda/\nClean-param: stat-id&format-type&actual-format&april&test-tag&adgroup&campaign_id&c&nowcast_header&detailsType&utm_from&bar_exp&march&november&bounds&game&text&city_id&february&september&january&no_turbo&october&ref&december&weather_url&build&lp_source&tab&geoid&cameras&day&thndr_onbrd /pogoda/\nClean-param: from_pumpkin&le_WindParticles&nowcast&yw_preset&gen_time&for_date&morda_geolocation&retpath&acc_maps&le_TemperatureBalloons&request&le_Borders&elmt&may&source&ids&june&july&referrer&PAGEN_1&from_render_fallback&preset&to&august&appsearch_ys&region&tm&banner-test-tags&stat-id /hava/\nClean-param: format-type&actual-format&april&test-tag&adgroup&campaign_id&c&nowcast_header&detailsType&utm_from&bar_exp&march&november&bounds&game&text&city_id&february&september&january&no_turbo&october&ref&december&weather_url&build&lp_source&tab&geoid&cameras&day&thndr_onbrd /hava/\nClean-param: from_pumpkin&le_WindParticles&nowcast&yw_preset&gen_time&for_date&morda_geolocation&retpath&acc_maps&le_TemperatureBalloons&request&le_Borders&elmt&may&source&ids&june&july&referrer&PAGEN_1&from_render_fallback&preset&to&august&appsearch_ys&region&tm&banner-test-tags&stat-id /weather/\nClean-param: format-type&actual-format&april&test-tag&adgroup&campaign_id&c&nowcast_header&detailsType&utm_from&bar_exp&march&november&bounds&game&text&city_id&february&september&january&no_turbo&october&ref&december&weather_url&build&lp_source&tab&geoid&cameras&day&thndr_onbrd /weather/\nClean-param: utm_source&utm_medium&utm_term&utm_content&utm_campaign&yclid&referrer&ref&from&adj_campaign&win&lp_source&openstat /alice/\n","agents":{"GPTBot":true,"OAI-SearchBot":true,"ChatGPT-User":true,"ClaudeBot":true,"Claude-SearchBot":true,"PerplexityBot":true,"Google-Extended":true,"Applebot-Extended":true,"CCBot":true,"Bytespider":true,"Meta-ExternalAgent":true,"Amazonbot":true}},"sitemap":{"status":200,"content_type":"text/xml","bytes":14251,"sha256":"2986ab90759792be0498d0c6b23acd79cda9c5efb64d13758fdef6b79ac61a58","final_host":"yandex.ru","ms":68,"url":"https://yandex.ru/support/sitemap.xml","declared":true,"xml":true},"files":{"llms_txt":{"status":404,"content_type":"text/html","bytes":11777,"sha256":"633ff1e9052f3a12b0cf7fc5c205ecb0f9ecf5938acaa2e294ea6dc0db220494","final_host":"yandex.ru","ms":79,"path":"/llms.txt","html":true,"json":null,"present":false},"agents_md":{"status":404,"content_type":"text/html","bytes":11778,"sha256":"8a5a32a5d84207395ce120551076d59552d0fccdcd7063f0df5ef94cf7393762","final_host":"yandex.ru","ms":87,"path":"/agents.md","html":true,"json":null,"present":false},"agent_skills":{"status":404,"content_type":null,"bytes":9,"sha256":"0019dfc4b32d63c1392aa264aed2253c1e0c2fb09216f8e2cc269bbfb8bb49b5","final_host":"yandex.ru","ms":59,"path":"/.well-known/agent-skills/index.json","html":false,"json":null,"present":false},"mcp_card":{"status":404,"content_type":null,"bytes":9,"sha256":"0019dfc4b32d63c1392aa264aed2253c1e0c2fb09216f8e2cc269bbfb8bb49b5","final_host":"yandex.ru","ms":109,"path":"/.well-known/mcp/server-card.json","html":false,"json":null,"present":false},"api_catalog":{"status":404,"content_type":null,"bytes":9,"sha256":"0019dfc4b32d63c1392aa264aed2253c1e0c2fb09216f8e2cc269bbfb8bb49b5","final_host":"yandex.ru","ms":70,"path":"/.well-known/api-catalog","html":false,"json":null,"present":false},"a2a_card":{"status":404,"content_type":null,"bytes":9,"sha256":"0019dfc4b32d63c1392aa264aed2253c1e0c2fb09216f8e2cc269bbfb8bb49b5","final_host":"yandex.ru","ms":77,"path":"/.well-known/agent-card.json","html":false,"json":null,"present":false}},"rings":{"v":1,"candidates":[],"mcp":[],"a2a":null,"webmcp":{"model_context":false,"bridge":false,"mcp_json":false,"link_rel":false,"any":false},"handshakes":0,"get_path":{"status":404,"content_type":"text/html","ms":80,"error":null},"get_host":{"status":404,"content_type":"text/html","ms":305,"error":null}},"derived":{"reachable":true,"robots_served":true,"robots_sitemap":true,"content_signal":false,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":[],"blocks_any_ai":false,"files":{"llms_txt":false,"agents_md":false,"agent_skills":false,"mcp_card":false,"api_catalog":false,"a2a_card":false},"any_capability":false,"ai_refused":[],"ai_thin":[],"rings":{"mcp_declared":false,"mcp_live":false,"mcp_auth":false,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":2,"fh":"sso.passport.yandex.ru","fp_other_host":[]},"derive_v":3,"frame":"top1k","shard":2},{"v":1,"kind":"crawlcheck-census-record","run":14,"domain":"wordpress.com","rank":100,"measured_at":"2026-10-15T05:29:25.755Z","home":{"browser":{"status":403,"content_type":"text/html","bytes":7355,"sha256":"b4b94e6496f82953c3c9d3d25bb482409d0204bf741e0db198b944bfe1409172","final_host":"wordpress.com","ms":78,"identity":"Mobile browser","html":true},"gptbot":{"status":403,"content_type":"text/html","bytes":7373,"sha256":"d170fb9c6874841cac53f88b42e5fa8e0d3434ff05815590c04e31f829014be1","final_host":"wordpress.com","ms":79,"identity":"GPTBot","html":true},"claudebot":{"status":403,"content_type":"text/html","bytes":7373,"sha256":"d170fb9c6874841cac53f88b42e5fa8e0d3434ff05815590c04e31f829014be1","final_host":"wordpress.com","ms":64,"identity":"ClaudeBot","html":true},"oaisearch":{"status":403,"content_type":"text/html","bytes":7373,"sha256":"d170fb9c6874841cac53f88b42e5fa8e0d3434ff05815590c04e31f829014be1","final_host":"wordpress.com","ms":50,"identity":"OAI-SearchBot","html":true},"perplexity":{"status":403,"content_type":"text/html","bytes":7373,"sha256":"d170fb9c6874841cac53f88b42e5fa8e0d3434ff05815590c04e31f829014be1","final_host":"wordpress.com","ms":64,"identity":"PerplexityBot","html":true}},"robots":{"status":200,"content_type":"text/plain","bytes":1387,"sha256":"9aff78b09aab73c5fd9f66785cc5ba909506bdb4aa424d47bfa652690e443d4d","final_host":"wordpress.com","ms":168,"html":false,"text":"# If you are regularly crawling WordPress.com sites, please use our firehose to receive real-time push updates instead.\n# Please see https://developer.wordpress.com/docs/firehose/ for more details.\n\nSitemap: https://wordpress.com/sitemap.xml\nSitemap: https://wordpress.com/news-sitemap.xml\n\nSitemap: https://wordpress.com/go/sitemap.xml\nSitemap: https://wordpress.com/blog/sitemap.xml\nSitemap: https://wordpress.com/support/sitemap.xml\n\nUser-agent: *\nDisallow: /wp-admin/\nAllow: /wp-admin/admin-ajax.php\nDisallow: /typo/?subdomain=\nDisallow: /read/\nDisallow: /*/read/\nAllow: /forums/topic-tag/reader/\nAllow: /*/forums/topic-tag/reader/\nAllow: /support/reader/\nAllow: /*/support/reader/\nDisallow: /reader/\nDisallow: /*/reader/\nAllow: /log-in/\nAllow: /log-in$\nDisallow: /log-in?\nDisallow: /abuse/?*\nDisallow: /abuse?*\nDisallow: /plugins/?s=\nDisallow: /*/plugins/?s=\nDisallow: /*?aff=\nDisallow: /*&aff=\nDisallow: /*?affiliate=\nDisallow: /*&affiliate=\nDisallow: /*?cid=\nDisallow: /*&cid=\nDisallow: /*?irclickid=\nDisallow: /*&irclickid=\nDisallow: /*/?like_comment=\nDisallow: /*?retry=\nDisallow: /*?sid=\nDisallow: /*?action=\nDisallow: /wp-login.php\nDisallow: /wp-signup.php\nDisallow: /press-this.php\nDisallow: /remote-login.php\nDisallow: /activate/\nDisallow: /cgi-bin/\nDisallow: /mshots/v1/\nDisallow: /next/\nDisallow: /public.api/\n\n# This file was generated on Thu, 15 Oct 2026 03:33:23 +0000\n","agents":{"GPTBot":true,"OAI-SearchBot":true,"ChatGPT-User":true,"ClaudeBot":true,"Claude-SearchBot":true,"PerplexityBot":true,"Google-Extended":true,"Applebot-Extended":true,"CCBot":true,"Bytespider":true,"Meta-ExternalAgent":true,"Amazonbot":true}},"sitemap":null,"files":null,"derived":{"reachable":false,"robots_served":false,"robots_sitemap":false,"content_signal":false,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":[],"blocks_any_ai":false,"files":{"llms_txt":false,"agents_md":false,"agent_skills":false,"mcp_card":false,"api_catalog":false,"a2a_card":false},"any_capability":false,"ai_refused":[],"ai_thin":[],"rings":{"mcp_declared":false,"mcp_live":false,"mcp_auth":false,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":2,"fh":"wordpress.com","fp_other_host":[]},"derive_v":3,"frame":"top1k","shard":3}]}