{"ok":true,"kind":"crawlcheck-census-records","frame":"top10k","run":18,"complete":true,"measured":9776,"of":9776,"shards":8,"offset":0,"returned":100,"next":"https://crawlcheck.io/api/census/records?frame=top10k&run=18&offset=100&limit=100","records":[{"v":1,"kind":"crawlcheck-census-record","run":18,"domain":"google.com","rank":1,"measured_at":"2026-11-13T18:46:27.088Z","home":{"browser":{"status":200,"content_type":"text/html","bytes":212603,"sha256":"65d38478e54b047b03adacc208601d209b22e61255cedeabbae89d459f622b4e","final_host":"www.google.com","final_path":"/","ms":777,"identity":"Mobile browser","html":true},"gptbot":{"status":200,"content_type":"text/html","bytes":31878,"sha256":"5d4db50f02f9ba9eee2d8f07e8b589d5ab6c2c83c7fe1fd2aa0e7637ebe91e8e","final_host":"www.google.com","final_path":"/","ms":646,"identity":"GPTBot","html":true},"claudebot":{"status":200,"content_type":"text/html","bytes":91230,"sha256":"a370d98f4e8ef87e2472cb4bc895b198f572ef403f53303123ff2679a9948a7c","final_host":"www.google.com","final_path":"/","ms":712,"identity":"ClaudeBot","html":true},"oaisearch":{"status":200,"content_type":"text/html","bytes":31881,"sha256":"79b4f81fc4037306771d7e5ef3f8d85887e23335bdda9f10da155ce3ecc7d174","final_host":"www.google.com","final_path":"/","ms":655,"identity":"OAI-SearchBot","html":true},"perplexity":{"status":200,"content_type":"text/html","bytes":90180,"sha256":"d6a53ddb552ae964a8d35f777835a49193b69366ddde2c66629c0eec2e2d35dd","final_host":"www.google.com","final_path":"/","ms":700,"identity":"PerplexityBot","html":true}},"robots":{"status":200,"content_type":"text/plain","bytes":6681,"sha256":"e8bf1ce79810901a9fef98deac93aaa532b1be5091ccd19f944dc8d86028a5c3","final_host":"www.google.com","final_path":"/robots.txt","ms":582,"html":false,"text":"User-agent: *\nUser-agent: Yandex\nDisallow: /search\nAllow: /search/about\nAllow: /search/howsearchworks\nDisallow: /sdch\nDisallow: /groups\nDisallow: /index.html?\nDisallow: /?\nDisallow: /goto?\nAllow: /?hl=\nDisallow: /?hl=*&\nAllow: /?hl=*&gws_rd=ssl$\nDisallow: /?hl=*&*&gws_rd=ssl\nAllow: /?gws_rd=ssl$\nAllow: /?pt1=true$\nDisallow: /imgres\nDisallow: /u/\nDisallow: /setprefs\nDisallow: /m?\nDisallow: /m/\nAllow:    /m/finance\nDisallow: /wml?\nDisallow: /wml/?\nDisallow: /wml/search?\nDisallow: /xhtml?\nDisallow: /xhtml/?\nDisallow: /xhtml/search?\nDisallow: /xml?\nDisallow: /imode?\nDisallow: /imode/?\nDisallow: /imode/search?\nDisallow: /jsky?\nDisallow: /jsky/?\nDisallow: /jsky/search?\nDisallow: /pda?\nDisallow: /pda/?\nDisallow: /pda/search?\nDisallow: /local?\nDisallow: /local_url\nDisallow: /products?\nDisallow: /product_\nDisallow: /products_\nDisallow: /products;\nDisallow: /print\nDisallow: /books/\nDisallow: /bkshp?*dq=\nDisallow: /bkshp?*q=\nDisallow: /books?*dq=\nDisallow: /books?*q=\nDisallow: /books?*qtid=\nDisallow: /books?*output=\nDisallow: /books?*pg=\nDisallow: /books?*jtp=\nDisallow: /books?*jscmd=\nDisallow: /books?*buy=\nDisallow: /books?*zoom=\nAllow: /books/about\nAllow: /books?*zoom=1\nAllow: /books?*zoom=5\nAllow: /books/content?*zoom=1\nAllow: /books/content?*zoom=5\nDisallow: /patents?\nDisallow: /patents/download/\nDisallow: /patents/pdf/\nDisallow: /patents/related/\nDisallow: /scholar\nDisallow: /citations?\nAllow: /citations?user=\nAllow: /citations?view_op=new_profile\nAllow: /citations?view_op=top_venues\nAllow: /scholar_share\nDisallow: /s?\nDisallow: /maps?\nAllow: /maps?daddr=\nAllow: /maps?entry=wc\nAllow: /maps?f=\nAllow: /maps?hl=\nAllow: /maps?q=\nAllow: /maps?saddr=\nAllow: /maps?sid=\nAllow: /maps?*output=classic\nAllow: /maps?*file=\nDisallow: /mapslt?\nDisallow: /maphp?\nDisallow: /maps/\nAllow: /maps/$\nAllow: /maps/@\nAllow: /maps/?daddr=\nAllow: /maps/?entry=wc\nAllow: /maps/?f=\nAllow: /maps/?hl=\nAllow: /maps/?q=\nAllow: /maps/?saddr=\nAllow: /maps/?sid=\nAllow: /maps/api/staticmap?\nAllow: /maps/search/\nAllow: /maps/sitemap.xml\nAllow: /maps/sitemaps/\nAllow: /maps/dir/\nAllow: /maps/d/\nAllow: /maps/reserve\nAllow: /maps/about\nAllow: /maps/contrib/\nAllow: /maps/match\nAllow: /maps/ms?\nAllow: /maps/place/\nAllow: /maps/_/\nAllow: /search?*tbm=map\nAllow: /maps/vt?\nAllow: /maps/preview\nDisallow: /maps/api/js/\nAllow: /maps/api/js\nDisallow: /mld?\nDisallow: /staticmap?\nDisallow: /help/maps/streetview/partners/welcome/\nDisallow: /help/maps/indoormaps/partners/\nDisallow: /lochp?\nDisallow: /ie?\nDisallow: /uds/\nDisallow: /transit?\nDisallow: /trends?\nDisallow: /trends/music?\nDisallow: /trends/hottrends?\nDisallow: /trends/viz?\nDisallow: /trends/embed.js?\nDisallow: /trends/fetchComponent?\nDisallow: /trends/beta\nDisallow: /trends/topics\nDisallow: /trends/explore?\nDisallow: /trends/api\nDisallow: /musica\nDisallow: /musicl\nDisallow: /musics\nDisallow: /urchin_test/\nDisallow: /movies?\nDisallow: /wapsearch?\nDisallow: /reviews/search?\nDisallow: /cbk\nDisallow: /profiles/me\nDisallow: /s2/profiles/me\nAllow: /s2/profiles\nAllow: /s2/oz\nAllow: /s2/photos\nAllow: /s2/search/social\nAllow: /s2/static\nDisallow: /s2\nDisallow: /transconsole/portal/\nDisallow: /aclk\nDisallow: /tbproxy/\nDisallow: /support/forum/search?\nDisallow: /reviews/polls/\nDisallow: /hosted/images/\nDisallow: /accounts/ClientLogin\nDisallow: /accounts/ClientAuth\nDisallow: /accounts/o8\nAllow: /accounts/o8/id\nDisallow: /quality_form?\nDisallow: /labs/popgadget/search\nDisallow: /compressiontest/\nDisallow: /analytics/feeds/\nDisallow: /analytics/partners/comments/\nDisallow: /analytics/portal/\nDisallow: /analytics/uploads/\nAllow: /alerts/manage\nAllow: /alerts/remove\nDisallow: /alerts/\nAllow: /alerts/$\nDisallow: /phone/compare/?\nDisallow: /travel/clk\nDisallow: /travel/lodging/clk\nDisallow: /travel/lodging/imp\nDisallow: /travel/entity\nDisallow: /travel/search\nDisallow: /travel/flights/booking\nDisallow: /travel/flights/s/\nDisallow: /travel/flights/search\nDisallow: /travel/hotels/stories\nDisallow: /travel/hotels/*/stories\nDisallow: /travel/story\nDisallow: /hotelfinder/rpc\nDisallow: /hotels/rpc\nDisallow: /evaluation/\nDisallow: /forms/perks/\nDisallow: /shopping/suppliers/search\nDisallow: /edu/cs4hs/\nDisallow: /trustedstores/s/\nDisallow: /trustedstores/tm2\nDisallow: /trustedstores/verify\nDisallow: /shopping?\nDisallow: /shopping/product/\nDisallow: /shopping/seller\nDisallow: /shopping/ratings/account/metrics\nDisallow: /shopping/ratings/merchant/immersivedetails\nDisallow: /shopping/reviewer\nDisallow: /shopping/search\nDisallow: /shopping/deals\nAllow: /shopping?udm=28$\nAllow: /shopping?udm=shop$\nDisallow: /storefront\nDisallow: /storepicker\nDisallow: /about/careers/applications/candidate-prep\nDisallow: /about/careers/applications/connect-with-a-googler\nDisallow: /about/careers/applications/jobs/results?page=\nDisallow: /about/careers/applications/jobs/results/?page=\nDisallow: /about/careers/applications/jobs/results?*&page=\nDisallow: /about/careers/applications/jobs/results/?*&page=\nDisallow: /landing/signout.html\nDisallow: /gallery/\nDisallow: /landing/now/ontap/\nAllow: /maps/reserve\nAllow: /maps/reserve/partners\nDisallow: /maps/reserve/api/\nDisallow: /maps/reserve/search\nDisallow: /maps/reserve/bookings\nDisallow: /maps/reserve/settings\nDisallow: /maps/reserve/manage\nDisallow: /maps/reserve/payment\nDisallow: /maps/reserve/receipt\nDisallow: /maps/reserve/sellersignup\nDisallow: /maps/reserve/feedback\nDisallow: /maps/reserve/terms\nDisallow: /maps/reserve/m/\nDisallow: /maps/reserve/b/\nDisallow: /maps/reserve/partner-dashboard\nDisallow: /local/cars\nDisallow: /local/dealership/\nDisallow: /local/dining/\nDisallow: /local/place/products/\nDisallow: /local/place/reviews/\nDisallow: /local/place/rap/\nDisallow: /local/tab/\nDisallow: /localservices/\nAllow: /localservices/coupon/\nDisallow: /nonprofits/account/\nDisallow: /uviewer\nDisallow: /landing/cmsnext-root/\n\n# AdsBot\nUser-agent: AdsBot-Google\nDisallow: /maps/api/js/\nAllow: /maps/api/js\nDisallow: /maps/api/place/js/\nDisallow: /maps/api/staticmap\nDisallow: /maps/api/streetview\n\n# New user agent groups must also have a user agent reference in the global (*)\n# group. See \"Order of precedence\" section in\n# https://goo.gle/rep#order-of-precedence-for-user-agents\nUser-agent: Yandex\nDisallow: /about/careers/applications/jobs/results\nDisallow: /about/careers/applications-a/jobs/results\n\n# Crawlers of certain social media sites are allowed to access page markup when\n# google.com/imgres* links are shared. To learn more, please contact\n# images-robots-allowlist@google.com.\nUser-agent: facebookexternalhit\nUser-agent: Twitterbot\nAllow: /imgres\nAllow: /search\nDisallow: /groups\nDisallow: /hosted/images/\nDisallow: /m/\n\nSitemap: https://www.google.com/sitemap.xml\n","agents":{"GPTBot":true,"OAI-SearchBot":true,"ChatGPT-User":true,"ClaudeBot":true,"Claude-SearchBot":true,"PerplexityBot":true,"Google-Extended":true,"Applebot-Extended":true,"CCBot":true,"Bytespider":true,"Meta-ExternalAgent":true,"Amazonbot":true}},"sitemap":{"status":200,"content_type":"text/xml","bytes":2047,"sha256":"0bc80feb7f7900da84ecf0cafdc78fb4dc3a9550413b40722eb73c6c02d0f49c","final_host":"www.google.com","final_path":"/sitemap.xml","ms":276,"url":"https://www.google.com/sitemap.xml","declared":true,"xml":true},"files":{"llms_txt":{"status":404,"content_type":"text/html","bytes":1569,"sha256":"47d71ed470dfe1ca13eb2af8fa019b95871a1bd1381a518ed71359e69b9c3cc7","final_host":"google.com","final_path":"/llms.txt","ms":386,"path":"/llms.txt","html":true,"json":null,"present":false},"agents_md":{"status":404,"content_type":"text/html","bytes":1570,"sha256":"f08e316602488d86581ce85b3076a13c82fc3f97cdac3deba23801fb66ca908d","final_host":"google.com","final_path":"/agents.md","ms":377,"path":"/agents.md","html":true,"json":null,"present":false},"agent_skills":{"status":404,"content_type":"text/html","bytes":1596,"sha256":"eee9f4f975e761645c6c385ec645439385500d0062bff7170ed09a8835d6f92b","final_host":"google.com","final_path":"/.well-known/agent-skills/index.json","ms":389,"path":"/.well-known/agent-skills/index.json","html":true,"json":null,"present":false},"mcp_card":{"status":404,"content_type":"text/html","bytes":1593,"sha256":"e5db4a1d5f7596a7d62e54e192ab48bbf195116fa33a2c7e160fdaf6131945a8","final_host":"google.com","final_path":"/.well-known/mcp/server-card.json","ms":172,"path":"/.well-known/mcp/server-card.json","html":true,"json":null,"present":false},"api_catalog":{"status":404,"content_type":"text/html","bytes":1584,"sha256":"7de25ab2e3ce288cf29d538f80ed48bf0d5759f87b613e29ed345db4675698aa","final_host":"google.com","final_path":"/.well-known/api-catalog","ms":382,"path":"/.well-known/api-catalog","html":true,"json":null,"present":false},"a2a_card":{"status":404,"content_type":"text/html","bytes":1588,"sha256":"64d2189fbe1531e7fb6ab2e6040a932d87ae1feb811d4781df85ea7ef6b4d651","final_host":"google.com","final_path":"/.well-known/agent-card.json","ms":169,"path":"/.well-known/agent-card.json","html":true,"json":null,"present":false}},"rings":{"v":1,"candidates":[],"mcp":[],"a2a":null,"webmcp":{"model_context":false,"bridge":false,"mcp_json":false,"link_rel":false,"any":false},"handshakes":0,"get_path":{"status":404,"content_type":"text/html","ms":169,"error":null},"get_host":{"status":530,"content_type":"text/plain","ms":123,"error":null}},"sf":[4,0,0,0,0,0,0,0,0],"derived":{"reachable":true,"home_class":"answers","robots_served":true,"robots_sitemap":true,"content_signal":false,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":[],"blocks_any_ai":false,"files":{"llms_txt":false,"agents_md":false,"agent_skills":false,"mcp_card":false,"api_catalog":false,"a2a_card":false},"any_capability":false,"ai_refused":[],"ai_thin":["gptbot","claudebot","oaisearch","perplexity"],"aipref":0,"aipref_measured":true,"sf":[4,0,0,0,0,0,0,0,0],"rings":{"mcp_declared":false,"mcp_live":false,"mcp_auth":false,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":3,"fh":"www.google.com","fp_other_host":[]},"derive_v":5,"frame":"top10k","shard":0},{"v":1,"kind":"crawlcheck-census-record","run":18,"domain":"cloudflare.com","rank":2,"measured_at":"2026-11-13T18:48:18.168Z","home":{"browser":{"status":200,"content_type":"text/html","bytes":1334957,"sha256":"91e7234b2db888cdb03ff9ecb467827d9494ebaffe6694eecd8a9232d5f19de6","final_host":"www.cloudflare.com","final_path":"/","ms":245,"identity":"Mobile browser","html":true,"fp":"52bec2ee8b21bd1f"},"gptbot":{"status":200,"content_type":"text/html","bytes":1334957,"sha256":"91e7234b2db888cdb03ff9ecb467827d9494ebaffe6694eecd8a9232d5f19de6","final_host":"www.cloudflare.com","final_path":"/","ms":245,"identity":"GPTBot","html":true,"fp":"52bec2ee8b21bd1f"},"claudebot":{"status":200,"content_type":"text/html","bytes":1334957,"sha256":"91e7234b2db888cdb03ff9ecb467827d9494ebaffe6694eecd8a9232d5f19de6","final_host":"www.cloudflare.com","final_path":"/","ms":250,"identity":"ClaudeBot","html":true,"fp":"52bec2ee8b21bd1f"},"oaisearch":{"status":200,"content_type":"text/html","bytes":1334957,"sha256":"91e7234b2db888cdb03ff9ecb467827d9494ebaffe6694eecd8a9232d5f19de6","final_host":"www.cloudflare.com","final_path":"/","ms":240,"identity":"OAI-SearchBot","html":true,"fp":"52bec2ee8b21bd1f"},"perplexity":{"status":200,"content_type":"text/html","bytes":1334957,"sha256":"91e7234b2db888cdb03ff9ecb467827d9494ebaffe6694eecd8a9232d5f19de6","final_host":"www.cloudflare.com","final_path":"/","ms":245,"identity":"PerplexityBot","html":true,"fp":"52bec2ee8b21bd1f"}},"robots":{"status":200,"content_type":"text/plain","bytes":1099,"sha256":"c6e360cc6e931ea35c33ae917ff9e2194eb55f79f68e949ec54bcd4533324d98","final_host":"www.cloudflare.com","final_path":"/robots.txt","ms":20,"html":false,"text":"# Robots.txt for www.cloudflare.com\n\nUser-agent: *\nAllow: /\n\n# Sitemap\nSitemap: https://www.cloudflare.com/sitemap.xml\n\n# AI/LLM friendly content\n# See https://llmstxt.org for the llms.txt specification\n# llms.txt provides curated content for AI assistants and LLMs\n\n# Allow AI crawlers to access markdown versions of pages\nUser-agent: GPTBot\nAllow: /\n\nUser-agent: ChatGPT-User\nAllow: /\n\nUser-agent: Google-Extended\nAllow: /\n\nUser-agent: Anthropic-AI\nAllow: /\n\nUser-agent: Claude-Web\nAllow: /\n\nUser-agent: CCBot\nAllow: /\n\nUser-agent: PerplexityBot\nAllow: /\n\nUser-agent: Cohere-ai\nAllow: /\n\n# Content Signals — declare AI content usage preferences\n# See https://contentsignals.org/ and https://datatracker.ietf.org/doc/draft-romm-aipref-contentsignals/\nContent-Signal: ai-train=yes, search=yes, ai-input=yes\n\n# AI-friendly content locations\n# - /llms.txt - Curated overview for AI/LLMs (markdown)\n# - /llms-full.txt - Full expanded content for larger context windows\n# - /*.md - Markdown versions of all pages (append .md to any URL)\n# - /.well-known/agents.json - Agent discovery and capabilities\n","agents":{"GPTBot":true,"OAI-SearchBot":true,"ChatGPT-User":true,"ClaudeBot":true,"Claude-SearchBot":true,"PerplexityBot":true,"Google-Extended":true,"Applebot-Extended":true,"CCBot":true,"Bytespider":true,"Meta-ExternalAgent":true,"Amazonbot":true}},"sitemap":{"status":200,"content_type":"application/xml","bytes":1639563,"sha256":"954a72fa35a37343d1da2fbe2cd33860f552c4c6347901d10ac27aaefde28af2","final_host":"www.cloudflare.com","final_path":"/sitemap.xml","ms":13,"url":"https://www.cloudflare.com/sitemap.xml","declared":true,"xml":true},"files":{"llms_txt":{"status":200,"content_type":"text/plain","bytes":17173,"sha256":"81d53b543800b2a5fb3521421b0b522d09097c19f8e19ba092c25901d97db7ac","final_host":"www.cloudflare.com","final_path":"/llms.txt","ms":23,"path":"/llms.txt","html":false,"json":null,"present":true},"agents_md":{"status":404,"content_type":"text/plain","bytes":12,"sha256":"69967e8148f9451562c4ef70a9ba15c29cd14b13376b2595a71da547f9a03802","final_host":"www.cloudflare.com","final_path":"/agents.md","ms":537,"path":"/agents.md","html":false,"json":null,"present":false},"agent_skills":{"status":200,"content_type":"application/json","bytes":11207,"sha256":"339186caba56b94036ddfbc50ce40585e99d4218befa7bc244a49e863a9a6e15","final_host":"www.cloudflare.com","final_path":"/.well-known/agent-skills/index.json","ms":24,"path":"/.well-known/agent-skills/index.json","html":false,"json":true,"present":true},"mcp_card":{"status":200,"content_type":"application/json","bytes":1320,"sha256":"296657d7bd0330bee2700f5de88edc054acc8637f6deb2b5cce8b9030f13706f","final_host":"www.cloudflare.com","final_path":"/.well-known/mcp/server-card.json","ms":26,"path":"/.well-known/mcp/server-card.json","html":false,"json":true,"present":true},"api_catalog":{"status":404,"content_type":"text/html","bytes":176757,"sha256":"eff32838ab78fa9ee5bf8644cadc7e7f0e291aeea7d29eb98c4695e5779e668e","final_host":"www.cloudflare.com","final_path":"/.well-known/api-catalog","ms":339,"path":"/.well-known/api-catalog","html":true,"json":null,"present":false},"a2a_card":{"status":404,"content_type":"text/plain","bytes":12,"sha256":"69967e8148f9451562c4ef70a9ba15c29cd14b13376b2595a71da547f9a03802","final_host":"www.cloudflare.com","final_path":"/.well-known/agent-card.json","ms":57,"path":"/.well-known/agent-card.json","html":false,"json":null,"present":false}},"rings":{"v":1,"candidates":[{"url":"https://docs.mcp.cloudflare.com/mcp","source":"mcp_card"},{"url":"https://blog.mcp.cloudflare.com/mcp","source":"mcp_card"},{"url":"https://mcp.cloudflare.com/mcp","source":"mcp_card"}],"mcp":[{"url":"https://docs.mcp.cloudflare.com/mcp","source":"mcp_card","outcome":"initialized","status":200,"ms":15,"initialized":true,"protocol_version":"2025-06-18","server":{"name":"docs-ai-search","version":"0.5.0"},"tools":2,"tools_sha256":"16fd520bee852c5de3a0ec249dac7322b1152156c6012d9102160e830e374c92","auth_header":false},{"url":"https://blog.mcp.cloudflare.com/mcp","source":"mcp_card","outcome":"initialized","status":200,"ms":28,"initialized":true,"protocol_version":"2025-06-18","server":{"name":"cloudflare-blog","version":"0.3.0"},"tools":4,"tools_sha256":"4995c0bbb213a965db3f46d62682b9eb69b29c40d876b3bd3648716bf750927d","auth_header":false},{"url":"https://mcp.cloudflare.com/mcp","source":"mcp_card","outcome":"auth_required","status":401,"ms":4,"initialized":false,"protocol_version":null,"server":null,"tools":null,"tools_sha256":null,"auth_header":true}],"a2a":null,"webmcp":{"model_context":false,"bridge":false,"mcp_json":false,"link_rel":false,"any":false},"handshakes":3,"get_path":{"status":404,"content_type":"text/html","ms":763,"error":null}},"sf":[1,0,0,0,0,0,0,0,1],"derived":{"reachable":true,"home_class":"answers","robots_served":true,"robots_sitemap":true,"content_signal":true,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":[],"blocks_any_ai":false,"files":{"llms_txt":true,"agents_md":false,"agent_skills":true,"mcp_card":true,"api_catalog":false,"a2a_card":false},"any_capability":true,"ai_refused":[],"ai_thin":[],"aipref":0,"aipref_measured":true,"sf":[1,0,0,0,0,0,0,0,1],"rings":{"mcp_declared":true,"mcp_live":true,"mcp_auth":true,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":3,"fh":"www.cloudflare.com","fp_other_host":[],"fp":"52bec2ee8b21bd1f","fp_bits":{"gptbot":0,"claudebot":0,"oaisearch":0,"perplexity":0}},"derive_v":5,"frame":"top10k","shard":1},{"v":1,"kind":"crawlcheck-census-record","run":18,"domain":"facebook.com","rank":3,"measured_at":"2026-11-13T18:49:51.543Z","home":{"browser":{"status":400,"content_type":"text/html","bytes":3676,"sha256":"440caed1f116417f30cd234d79f74508d52d79015d59971d8c3fd8953d68627e","final_host":"m.facebook.com","final_path":"/","ms":515,"identity":"Mobile browser","html":true},"gptbot":{"status":200,"content_type":"text/html","bytes":366046,"sha256":"2361a4dfeb7e7940e221259ae601c9f222c4180d543a5b71e024989f98c03ed4","final_host":"www.facebook.com","final_path":"/login","ms":673,"identity":"GPTBot","html":true,"fp":"1a23d41262989ba4"},"claudebot":{"status":200,"content_type":"text/html","bytes":365693,"sha256":"ce9c260748129259aace3e6be7483dbba00a4cddc00241ef09c2fd1a8a8f95eb","final_host":"www.facebook.com","final_path":"/","ms":581,"identity":"ClaudeBot","html":true,"fp":"1a23d41262989ba4"},"oaisearch":{"status":200,"content_type":"text/html","bytes":365714,"sha256":"bd77c38c1127449d8d2eed1d6fb3620cd0a3f2c7bd60421f3791c33b7e9077eb","final_host":"www.facebook.com","final_path":"/","ms":574,"identity":"OAI-SearchBot","html":true,"fp":"1a23d41262989ba4"},"perplexity":{"status":200,"content_type":"text/html","bytes":321846,"sha256":"4c2a0b6c6c61f23d019c1c8ff06f58190c826b2cf424d74b3f7025b6aeb38ef5","final_host":"www.facebook.com","final_path":"/","ms":568,"identity":"PerplexityBot","html":true}},"robots":{"status":200,"content_type":"text/plain","bytes":26917,"sha256":"7fc92d8059cb08a27e73f63d490ae62f4f0dd4d58eac0c9fca0dec452b0224bc","final_host":"facebook.com","final_path":"/robots.txt","ms":135,"html":false,"text":"# Notice: Collection of data on Facebook through automated means is\n# prohibited unless you have express written permission from Facebook\n# and may only be conducted for the limited purpose contained in said\n# permission.\n# All authorized user-agents listed on this page must comply with Meta’s\n# Automated Data Collection Terms available at:\n# https://www.facebook.com/legal/automated_data_collection_terms\nUser-agent: __usage_ai-training\nDisallow: /\n\nUser-agent: Amazonbot\nDisallow: /\n\nUser-agent: Applebot-Extended\nDisallow: /\n\nUser-agent: Brightbot\nDisallow: /\n\nUser-agent: ClaudeBot\nDisallow: /\n\nUser-agent: Google-Extended\nDisallow: /\n\nUser-agent: GPTBot\nDisallow: /\n\nUser-agent: PerplexityBot\nDisallow: /\n\nUser-agent: PetalBot\nDisallow: /\n\nUser-agent: Scrapy\nDisallow: /\n\nUser-agent: uptimerobot\nDisallow: /\n\nUser-agent: viberbot\nDisallow: /\n\nUser-agent: YaK\nDisallow: /\n\nUser-agent: Yandex\nDisallow: /\n\nUser-agent: Yeti\nDisallow: /\n\nUser-agent: Applebot\nDisallow: /*/plugins/*\nDisallow: /?*next=\nDisallow: /a/bz?\nDisallow: /ajax/\nDisallow: /album.php\nDisallow: /business/*&categories\nDisallow: /business/*?categories\nDisallow: /business/help/search*&query=\nDisallow: /business/help/search*?query=\nDisallow: /business/news*&tags\nDisallow: /business/news*?tags\nDisallow: /business/search*&q=\nDisallow: /business/search*?q=\nDisallow: /checkpoint/\nDisallow: /contact_importer/\nDisallow: /dialog/\nDisallow: /fbml/ajax/dialog/\nDisallow: /feeds/\nDisallow: /file_download.php\nDisallow: /job_application/\nDisallow: /l.php\nDisallow: /login.php*&next=\nDisallow: /login.php/?next=\nDisallow: /login.php?next=\nDisallow: /login/*&next=\nDisallow: /login/?next=\nDisallow: /login/device-based/regular/login/*&next=\nDisallow: /login/device-based/regular/login/?next=\nDisallow: /meta-ai-answers/\nDisallow: /moments_app/\nDisallow: /p.php\nDisallow: /photos.php\nDisallow: /plugins/\nDisallow: /share.php\nDisallow: /share/\nDisallow: /sharer.php\nDisallow: /sharer/\nDisallow: /tr/\nDisallow: /tr?\nDisallow: /ufi/reaction/profile/browser/\nDisallow: /x/oauth/\nAllow: /ajax/bootloader-endpoint/\nAllow: /ajax/pagelet/generic.php/PagePostsSectionPagelet\nAllow: /careers/\nAllow: /safetycheck/\n\nUser-agent: Bingbot\nDisallow: /*/plugins/*\nDisallow: /?*next=\nDisallow: /a/bz?\nDisallow: /ajax/\nDisallow: /album.php\nDisallow: /business/*&categories\nDisallow: /business/*?categories\nDisallow: /business/help/search*&query=\nDisallow: /business/help/search*?query=\nDisallow: /business/news*&tags\nDisallow: /business/news*?tags\nDisallow: /business/search*&q=\nDisallow: /business/search*?q=\nDisallow: /checkpoint/\nDisallow: /contact_importer/\nDisallow: /dialog/\nDisallow: /fbml/ajax/dialog/\nDisallow: /feeds/\nDisallow: /file_download.php\nDisallow: /job_application/\nDisallow: /l.php\nDisallow: /login.php*&next=\nDisallow: /login.php/?next=\nDisallow: /login.php?next=\nDisallow: /login/*&next=\nDisallow: /login/?next=\nDisallow: /login/device-based/regular/login/*&next=\nDisallow: /login/device-based/regular/login/?next=\nDisallow: /meta-ai-answers/\nDisallow: /moments_app/\nDisallow: /p.php\nDisallow: /photos.php\nDisallow: /plugins/\nDisallow: /share.php\nDisallow: /share/\nDisallow: /sharer.php\nDisallow: /sharer/\nDisallow: /tr/\nDisallow: /tr?\nDisallow: /ufi/reaction/profile/browser/\nDisallow: /x/oauth/\nAllow: /ajax/bootloader-endpoint/\nAllow: /ajax/pagelet/generic.php/PagePostsSectionPagelet\nAllow: /careers/\nAllow: /safetycheck/\n\nUser-agent: deepcrawl\nDisallow: /\nDisallow: /*/plugins/*\nDisallow: /?*next=\nDisallow: /a/bz?\nDisallow: /ajax/\nDisallow: /album.php\nDisallow: /business/*&categories\nDisallow: /business/*?categories\nDisallow: /business/help/search*&query=\nDisallow: /business/help/search*?query=\nDisallow: /business/news*&tags\nDisallow: /business/news*?tags\nDisallow: /business/search*&q=\nDisallow: /business/search*?q=\nDisallow: /checkpoint/\nDisallow: /contact_importer/\nDisallow: /dialog/\nDisallow: /fbml/ajax/dialog/\nDisallow: /feeds/\nDisallow: /file_download.php\nDisallow: /job_application/\nDisallow: /l.php\nDisallow: /login.php*&next=\nDisallow: /login.php/?next=\nDisallow: /login.php?next=\nDisallow: /login/*&next=\nDisallow: /login/?next=\nDisallow: /login/device-based/regular/login/*&next=\nDisallow: /login/device-based/regular/login/?next=\nDisallow: /meta-ai-answers/\nDisallow: /moments_app/\nDisallow: /p.php\nDisallow: /photos.php\nDisallow: /plugins/\nDisallow: /share.php\nDisallow: /share/\nDisallow: /sharer.php\nDisallow: /sharer/\nDisallow: /tr/\nDisallow: /tr?\nDisallow: /ufi/reaction/profile/browser/\nDisallow: /x/oauth/\nAllow: /ajax/bootloader-endpoint/\nAllow: /ajax/pagelet/generic.php/PagePostsSectionPagelet\nAllow: /business/\nAllow: /careers/\nAllow: /safetycheck/\n\nUser-agent: Discordbot\nDisallow: /*/plugins/*\nDisallow: /?*next=\nDisallow: /a/bz?\nDisallow: /ajax/\nDisallow: /album.php\nDisallow: /business/*&categories\nDisallow: /business/*?categories\nDisallow: /business/help/search*&query=\nDisallow: /business/help/search*?query=\nDisallow: /business/news*&tags\nDisallow: /business/news*?tags\nDisallow: /business/search*&q=\nDisallow: /business/search*?q=\nDisallow: /checkpoint/\nDisallow: /contact_importer/\nDisallow: /dialog/\nDisallow: /fbml/ajax/dialog/\nDisallow: /feeds/\nDisallow: /file_download.php\nDisallow: /job_application/\nDisallow: /l.php\nDisallow: /login.php*&next=\nDisallow: /login.php/?next=\nDisallow: /login.php?next=\nDisallow: /login/*&next=\nDisallow: /login/?next=\nDisallow: /login/device-based/regular/login/*&next=\nDisallow: /login/device-based/regular/login/?next=\nDisallow: /meta-ai-answers/\nDisallow: /moments_app/\nDisallow: /p.php\nDisallow: /photos.php\nDisallow: /plugins/\nDisallow: /share.php\nDisallow: /share/\nDisallow: /sharer.php\nDisallow: /sharer/\nDisallow: /tr/\nDisallow: /tr?\nDisallow: /ufi/reaction/profile/browser/\nDisallow: /x/oauth/\nAllow: /ajax/bootloader-endpoint/\nAllow: /ajax/pagelet/generic.php/PagePostsSectionPagelet\nAllow: /careers/\nAllow: /safetycheck/\n\nUser-agent: DuckDuckBot\nDisallow: /*/plugins/*\nDisallow: /?*next=\nDisallow: /a/bz?\nDisallow: /ajax/\nDisallow: /album.php\nDisallow: /business/*&categories\nDisallow: /business/*?categories\nDisallow: /business/help/search*&query=\nDisallow: /business/help/search*?query=\nDisallow: /business/news*&tags\nDisallow: /business/news*?tags\nDisallow: /business/search*&q=\nDisallow: /business/search*?q=\nDisallow: /checkpoint/\nDisallow: /contact_importer/\nDisallow: /dialog/\nDisallow: /fbml/ajax/dialog/\nDisallow: /feeds/\nDisallow: /file_download.php\nDisallow: /job_application/\nDisallow: /l.php\nDisallow: /login.php*&next=\nDisallow: /login.php/?next=\nDisallow: /login.php?next=\nDisallow: /login/*&next=\nDisallow: /login/?next=\nDisallow: /login/device-based/regular/login/*&next=\nDisallow: /login/device-based/regular/login/?next=\nDisallow: /meta-ai-answers/\nDisallow: /moments_app/\nDisallow: /p.php\nDisallow: /photos.php\nDisallow: /plugins/\nDisallow: /share.php\nDisallow: /share/\nDisallow: /sharer.php\nDisallow: /sharer/\nDisallow: /tr/\nDisallow: /tr?\nDisallow: /ufi/reaction/profile/browser/\nDisallow: /x/oauth/\nAllow: /ajax/bootloader-endpoint/\nAllow: /ajax/pagelet/generic.php/PagePostsSectionPagelet\nAllow: /careers/\nAllow: /safetycheck/\n\nUser-agent: facebookexternalhit\nDisallow: /*/plugins/*\nDisallow: /?*next=\nDisallow: /a/bz?\nDisallow: /ajax/\nDisallow: /album.php\nDisallow: /business/*&categories\nDisallow: /business/*?categories\nDisallow: /business/help/search*&query=\nDisallow: /business/help/search*?query=\nDisallow: /business/news*&tags\nDisallow: /business/news*?tags\nDisallow: /business/search*&q=\nDisallow: /business/search*?q=\nDisallow: /checkpoint/\nDisallow: /contact_importer/\nDisallow: /dialog/\nDisallow: /fbml/ajax/dialog/\nDisallow: /feeds/\nDisallow: /file_download.php\nDisallow: /job_application/\nDisallow: /l.php\nDisallow: /login.php*&next=\nDisallow: /login.php/?next=\nDisallow: /login.php?next=\nDisallow: /login/*&next=\nDisallow: /login/?next=\nDisallow: /login/device-based/regular/login/*&next=\nDisallow: /login/device-based/regular/login/?next=\nDisallow: /meta-ai-answers/\nDisallow: /moments_app/\nDisallow: /p.php\nDisallow: /photos.php\nDisallow: /plugins/\nDisallow: /share.php\nDisallow: /share/\nDisallow: /sharer.php\nDisallow: /sharer/\nDisallow: /tr/\nDisallow: /tr?\nDisallow: /ufi/reaction/profile/browser/\nDisallow: /x/oauth/\nAllow: /ajax/bootloader-endpoint/\nAllow: /ajax/pagelet/generic.php/PagePostsSectionPagelet\nAllow: /careers/\nAllow: /safetycheck/\n\nUser-agent: Googlebot\nDisallow: /*/plugins/*\nDisallow: /?*next=\nDisallow: /a/bz?\nDisallow: /ajax/\nDisallow: /album.php\nDisallow: /business/*&categories\nDisallow: /business/*?categories\nDisallow: /business/help/search*&query=\nDisallow: /business/help/search*?query=\nDisallow: /business/news*&tags\nDisallow: /business/news*?tags\nDisallow: /business/search*&q=\nDisallow: /business/search*?q=\nDisallow: /checkpoint/\nDisallow: /contact_importer/\nDisallow: /dialog/\nDisallow: /fbml/ajax/dialog/\nDisallow: /feeds/\nDisallow: /file_download.php\nDisallow: /job_application/\nDisallow: /l.php\nDisallow: /login.php*&next=\nDisallow: /login.php/?next=\nDisallow: /login.php?next=\nDisallow: /login/*&next=\nDisallow: /login/?next=\nDisallow: /login/device-based/regular/login/*&next=\nDisallow: /login/device-based/regular/login/?next=\nDisallow: /meta-ai-answers/\nDisallow: /moments_app/\nDisallow: /p.php\nDisallow: /photos.php\nDisallow: /plugins/\nDisallow: /share.php\nDisallow: /share/\nDisallow: /sharer.php\nDisallow: /sharer/\nDisallow: /tr/\nDisallow: /tr?\nDisallow: /ufi/reaction/profile/browser/\nDisallow: /x/oauth/\nAllow: /ajax/bootloader-endpoint/\nAllow: /ajax/pagelet/generic.php/PagePostsSectionPagelet\nAllow: /careers/\nAllow: /safetycheck/\n\nUser-agent: Googlebot-Image\nDisallow: /*/plugins/*\nDisallow: /?*next=\nDisallow: /a/bz?\nDisallow: /ajax/\nDisallow: /album.php\nDisallow: /business/*&categories\nDisallow: /business/*?categories\nDisallow: /business/help/search*&query=\nDisallow: /business/help/search*?query=\nDisallow: /business/news*&tags\nDisallow: /business/news*?tags\nDisallow: /business/search*&q=\nDisallow: /business/search*?q=\nDisallow: /checkpoint/\nDisallow: /contact_importer/\nDisallow: /dialog/\nDisallow: /fbml/ajax/dialog/\nDisallow: /feeds/\nDisallow: /file_download.php\nDisallow: /job_application/\nDisallow: /l.php\nDisallow: /login.php*&next=\nDisallow: /login.php/?next=\nDisallow: /login.php?next=\nDisallow: /login/*&next=\nDisallow: /login/?next=\nDisallow: /login/device-based/regular/login/*&next=\nDisallow: /login/device-based/regular/login/?next=\nDisallow: /map_tile.php\nDisallow: /meta-ai-answers/\nDisallow: /moments_app/\nDisallow: /p.php\nDisallow: /photos.php\nDisallow: /plugins/\nDisallow: /share.php\nDisallow: /share/\nDisallow: /sharer.php\nDisallow: /sharer/\nDisallow: /static_map.php\nDisallow: /tr/\nDisallow: /tr?\nDisallow: /ufi/reaction/profile/browser/\nDisallow: /x/oauth/\nAllow: /ajax/bootloader-endpoint/\nAllow: /ajax/pagelet/generic.php/PagePostsSectionPagelet\nAllow: /careers/\nAllow: /safetycheck/\n\nUser-agent: Googlebot-Video\nDisallow: /*/plugins/*\nDisallow: /?*next=\nDisallow: /a/bz?\nDisallow: /ajax/\nDisallow: /album.php\nDisallow: /business/*&categories\nDisallow: /business/*?categories\nDisallow: /business/help/search*&query=\nDisallow: /business/help/search*?query=\nDisallow: /business/news*&tags\nDisallow: /business/news*?tags\nDisallow: /business/search*&q=\nDisallow: /business/search*?q=\nDisallow: /checkpoint/\nDisallow: /contact_importer/\nDisallow: /dialog/\nDisallow: /fbml/ajax/dialog/\nDisallow: /feeds/\nDisallow: /file_download.php\nDisallow: /job_application/\nDisallow: /l.php\nDisallow: /login.php*&next=\nDisallow: /login.php/?next=\nDisallow: /login.php?next=\nDisallow: /login/*&next=\nDisallow: /login/?next=\nDisallow: /login/device-based/regular/login/*&next=\nDisallow: /login/device-based/regular/login/?next=\nDisallow: /meta-ai-answers/\nDisallow: /moments_app/\nDisallow: /p.php\nDisallow: /photos.php\nDisallow: /plugins/\nDisallow: /share.php\nDisallow: /share/\nDisallow: /sharer.php\nDisallow: /sharer/\nDisallow: /tr/\nDisallow: /tr?\nDisallow: /ufi/reaction/profile/browser/\nDisallow: /x/oauth/\nAllow: /ajax/bootloader-endpoint/\nAllow: /ajax/pagelet/generic.php/PagePostsSectionPagelet\nAllow: /careers/\nAllow: /safetycheck/\n\nUser-agent: Googlebot-News\nDisallow: /*/plugins/*\nDisallow: /?*next=\nDisallow: /a/bz?\nDisallow: /ajax/\nDisallow: /album.php\nDisallow: /business/*&categories\nDisallow: /business/*?categories\nDisallow: /business/help/search*&query=\nDisallow: /business/help/search*?query=\nDisallow: /business/news*&tags\nDisallow: /business/news*?tags\nDisallow: /business/search*&q=\nDisallow: /business/search*?q=\nDisallow: /checkpoint/\nDisallow: /contact_importer/\nDisallow: /dialog/\nDisallow: /fbml/ajax/dialog/\nDisallow: /feeds/\nDisallow: /file_download.php\nDisallow: /job_application/\nDisallow: /l.php\nDisallow: /login.php*&next=\nDisallow: /login.php/?next=\nDisallow: /login.php?next=\nDisallow: /login/*&next=\nDisallow: /login/?next=\nDisallow: /login/device-based/regular/login/*&next=\nDisallow: /login/device-based/regular/login/?next=\nDisallow: /meta-ai-answers/\nDisallow: /moments_app/\nDisallow: /p.php\nDisallow: /photos.php\nDisallow: /plugins/\nDisallow: /share.php\nDisallow: /share/\nDisallow: /sharer.php\nDisallow: /sharer/\nDisallow: /tr/\nDisallow: /tr?\nDisallow: /ufi/reaction/profile/browser/\nDisallow: /x/oauth/\nAllow: /ajax/bootloader-endpoint/\nAllow: /ajax/pagelet/generic.php/PagePostsSectionPagelet\nAllow: /careers/\nAllow: /safetycheck/\n\nUser-agent: Google-InspectionTool\nDisallow: /*/plugins/*\nDisallow: /?*next=\nDisallow: /a/bz?\nDisallow: /ajax/\nDisallow: /album.php\nDisallow: /business/*&categories\nDisallow: /business/*?categories\nDisallow: /business/help/search*&query=\nDisallow: /business/help/search*?query=\nDisallow: /business/news*&tags\nDisallow: /business/news*?tags\nDisallow: /business/search*&q=\nDisallow: /business/search*?q=\nDisallow: /checkpoint/\nDisallow: /contact_importer/\nDisallow: /dialog/\nDisallow: /fbml/ajax/dialog/\nDisallow: /feeds/\nDisallow: /file_download.php\nDisallow: /job_application/\nDisallow: /l.php\nDisallow: /login.php*&next=\nDisallow: /login.php/?next=\nDisallow: /login.php?next=\nDisallow: /login/*&next=\nDisallow: /login/?next=\nDisallow: /login/device-based/regular/login/*&next=\nDisallow: /login/device-based/regular/login/?next=\nDisallow: /meta-ai-answers/\nDisallow: /moments_app/\nDisallow: /p.php\nDisallow: /photos.php\nDisallow: /plugins/\nDisallow: /share.php\nDisallow: /share/\nDisallow: /sharer.php\nDisallow: /sharer/\nDisallow: /tr/\nDisallow: /tr?\nDisallow: /ufi/reaction/profile/browser/\nDisallow: /x/oauth/\nAllow: /ajax/bootloader-endpoint/\nAllow: /ajax/pagelet/generic.php/PagePostsSectionPagelet\nAllow: /careers/\nAllow: /safetycheck/\n\nUser-agent: lumar\nDisallow: /\nDisallow: /*/plugins/*\nDisallow: /?*next=\nDisallow: /a/bz?\nDisallow: /ajax/\nDisallow: /album.php\nDisallow: /business/*&categories\nDisallow: /business/*?categories\nDisallow: /business/help/search*&query=\nDisallow: /business/help/search*?query=\nDisallow: /business/news*&tags\nDisallow: /business/news*?tags\nDisallow: /business/search*&q=\nDisallow: /business/search*?q=\nDisallow: /checkpoint/\nDisallow: /contact_importer/\nDisallow: /dialog/\nDisallow: /fbml/ajax/dialog/\nDisallow: /feeds/\nDisallow: /file_download.php\nDisallow: /job_application/\nDisallow: /l.php\nDisallow: /login.php*&next=\nDisallow: /login.php/?next=\nDisallow: /login.php?next=\nDisallow: /login/*&next=\nDisallow: /login/?next=\nDisallow: /login/device-based/regular/login/*&next=\nDisallow: /login/device-based/regular/login/?next=\nDisallow: /meta-ai-answers/\nDisallow: /moments_app/\nDisallow: /p.php\nDisallow: /photos.php\nDisallow: /plugins/\nDisallow: /share.php\nDisallow: /share/\nDisallow: /sharer.php\nDisallow: /sharer/\nDisallow: /tr/\nDisallow: /tr?\nDisallow: /ufi/reaction/profile/browser/\nDisallow: /x/oauth/\nAllow: /ajax/bootloader-endpoint/\nAllow: /ajax/pagelet/generic.php/PagePostsSectionPagelet\nAllow: /business/\nAllow: /careers/\nAllow: /safetycheck/\n\nUser-agent: LinkedInBot\nDisallow: /*/plugins/*\nDisallow: /?*next=\nDisallow: /a/bz?\nDisallow: /ajax/\nDisallow: /album.php\nDisallow: /business/*&categories\nDisallow: /business/*?categories\nDisallow: /business/help/search*&query=\nDisallow: /business/help/search*?query=\nDisallow: /business/news*&tags\nDisallow: /business/news*?tags\nDisallow: /business/search*&q=\nDisallow: /business/search*?q=\nDisallow: /checkpoint/\nDisallow: /contact_importer/\nDisallow: /dialog/\nDisallow: /fbml/ajax/dialog/\nDisallow: /feeds/\nDisallow: /file_download.php\nDisallow: /job_application/\nDisallow: /l.php\nDisallow: /login.php*&next=\nDisallow: /login.php/?next=\nDisallow: /login.php?next=\nDisallow: /login/*&next=\nDisallow: /login/?next=\nDisallow: /login/device-based/regular/login/*&next=\nDisallow: /login/device-based/regular/login/?next=\nDisallow: /meta-ai-answers/\nDisallow: /moments_app/\nDisallow: /p.php\nDisallow: /photos.php\nDisallow: /plugins/\nDisallow: /share.php\nDisallow: /share/\nDisallow: /sharer.php\nDisallow: /sharer/\nDisallow: /tr/\nDisallow: /tr?\nDisallow: /ufi/reaction/profile/browser/\nDisallow: /x/oauth/\nAllow: /ajax/bootloader-endpoint/\nAllow: /ajax/pagelet/generic.php/PagePostsSectionPagelet\nAllow: /careers/\nAllow: /safetycheck/\n\nUser-agent: msnbot\nDisallow: /*/plugins/*\nDisallow: /?*next=\nDisallow: /a/bz?\nDisallow: /ajax/\nDisallow: /album.php\nDisallow: /business/*&categories\nDisallow: /business/*?categories\nDisallow: /business/help/search*&query=\nDisallow: /business/help/search*?query=\nDisallow: /business/news*&tags\nDisallow: /business/news*?tags\nDisallow: /business/search*&q=\nDisallow: /business/search*?q=\nDisallow: /checkpoint/\nDisallow: /contact_importer/\nDisallow: /dialog/\nDisallow: /fbml/ajax/dialog/\nDisallow: /feeds/\nDisallow: /file_download.php\nDisallow: /job_application/\nDisallow: /l.php\nDisallow: /login.php*&next=\nDisallow: /login.php/?next=\nDisallow: /login.php?next=\nDisallow: /login/*&next=\nDisallow: /login/?next=\nDisallow: /login/device-based/regular/login/*&next=\nDisallow: /login/device-based/regular/login/?next=\nDisallow: /meta-ai-answers/\nDisallow: /moments_app/\nDisallow: /p.php\nDisallow: /photos.php\nDisallow: /plugins/\nDisallow: /share.php\nDisallow: /share/\nDisallow: /sharer.php\nDisallow: /sharer/\nDisallow: /tr/\nDisallow: /tr?\nDisallow: /ufi/reaction/profile/browser/\nDisallow: /x/oauth/\nAllow: /ajax/bootloader-endpoint/\nAllow: /ajax/pagelet/generic.php/PagePostsSectionPagelet\nAllow: /careers/\nAllow: /safetycheck/\n\nUser-agent: Pinterestbot\nDisallow: /*/plugins/*\nDisallow: /?*next=\nDisallow: /a/bz?\nDisallow: /ajax/\nDisallow: /album.php\nDisallow: /business/*&categories\nDisallow: /business/*?categories\nDisallow: /business/help/search*&query=\nDisallow: /business/help/search*?query=\nDisallow: /business/news*&tags\nDisallow: /business/news*?tags\nDisallow: /business/search*&q=\nDisallow: /business/search*?q=\nDisallow: /checkpoint/\nDisallow: /contact_importer/\nDisallow: /dialog/\nDisallow: /fbml/ajax/dialog/\nDisallow: /feeds/\nDisallow: /file_download.php\nDisallow: /job_application/\nDisallow: /l.php\nDisallow: /login.php*&next=\nDisallow: /login.php/?next=\nDisallow: /login.php?next=\nDisallow: /login/*&next=\nDisallow: /login/?next=\nDisallow: /login/device-based/regular/login/*&next=\nDisallow: /login/device-based/regular/login/?next=\nDisallow: /meta-ai-answers/\nDisallow: /moments_app/\nDisallow: /p.php\nDisallow: /photos.php\nDisallow: /plugins/\nDisallow: /share.php\nDisallow: /share/\nDisallow: /sharer.php\nDisallow: /sharer/\nDisallow: /tr/\nDisallow: /tr?\nDisallow: /ufi/reaction/profile/browser/\nDisallow: /x/oauth/\nAllow: /ajax/bootloader-endpoint/\nAllow: /ajax/pagelet/generic.php/PagePostsSectionPagelet\nAllow: /careers/\nAllow: /safetycheck/\n\nUser-agent: Screaming Frog SEO Spider\nDisallow: /*/plugins/*\nDisallow: /?*next=\nDisallow: /a/bz?\nDisallow: /ajax/\nDisallow: /album.php\nDisallow: /business/*&categories\nDisallow: /business/*?categories\nDisallow: /business/help/search*&query=\nDisallow: /business/help/search*?query=\nDisallow: /business/news*&tags\nDisallow: /business/news*?tags\nDisallow: /business/search*&q=\nDisallow: /business/search*?q=\nDisallow: /checkpoint/\nDisallow: /contact_importer/\nDisallow: /dialog/\nDisallow: /fbml/ajax/dialog/\nDisallow: /feeds/\nDisallow: /file_download.php\nDisallow: /job_application/\nDisallow: /l.php\nDisallow: /login.php*&next=\nDisallow: /login.php/?next=\nDisallow: /login.php?next=\nDisallow: /login/*&next=\nDisallow: /login/?next=\nDisallow: /login/device-based/regular/login/*&next=\nDisallow: /login/device-based/regular/login/?next=\nDisallow: /meta-ai-answers/\nDisallow: /moments_app/\nDisallow: /p.php\nDisallow: /photos.php\nDisallow: /plugins/\nDisallow: /share.php\nDisallow: /share/\nDisallow: /sharer.php\nDisallow: /sharer/\nDisallow: /tr/\nDisallow: /tr?\nDisallow: /ufi/reaction/profile/browser/\nDisallow: /x/oauth/\nAllow: /ajax/bootloader-endpoint/\nAllow: /ajax/pagelet/generic.php/PagePostsSectionPagelet\nAllow: /careers/\nAllow: /safetycheck/\n\nUser-agent: seznambot\nDisallow: /*/plugins/*\nDisallow: /?*next=\nDisallow: /a/bz?\nDisallow: /ajax/\nDisallow: /album.php\nDisallow: /business/*&categories\nDisallow: /business/*?categories\nDisallow: /business/help/search*&query=\nDisallow: /business/help/search*?query=\nDisallow: /business/news*&tags\nDisallow: /business/news*?tags\nDisallow: /business/search*&q=\nDisallow: /business/search*?q=\nDisallow: /checkpoint/\nDisallow: /contact_importer/\nDisallow: /dialog/\nDisallow: /fbml/ajax/dialog/\nDisallow: /feeds/\nDisallow: /file_download.php\nDisallow: /job_application/\nDisallow: /l.php\nDisallow: /login.php*&next=\nDisallow: /login.php/?next=\nDisallow: /login.php?next=\nDisallow: /login/*&next=\nDisallow: /login/?next=\nDisallow: /login/device-based/regular/login/*&next=\nDisallow: /login/device-based/regular/login/?next=\nDisallow: /meta-ai-answers/\nDisallow: /moments_app/\nDisallow: /p.php\nDisallow: /photos.php\nDisallow: /plugins/\nDisallow: /share.php\nDisallow: /share/\nDisallow: /sharer.php\nDisallow: /sharer/\nDisallow: /tr/\nDisallow: /tr?\nDisallow: /ufi/reaction/profile/browser/\nDisallow: /x/oauth/\nAllow: /ajax/bootloader-endpoint/\nAllow: /ajax/pagelet/generic.php/PagePostsSectionPagelet\nAllow: /careers/\nAllow: /safetycheck/\n\nUser-agent: Slurp\nDisallow: /*/plugins/*\nDisallow: /?*next=\nDisallow: /a/bz?\nDisallow: /ajax/\nDisallow: /album.php\nDisallow: /business/*&categories\nDisallow: /business/*?categories\nDisallow: /business/help/search*&query=\nDisallow: /business/help/search*?query=\nDisallow: /business/news*&tags\nDisallow: /business/news*?tags\nDisallow: /business/search*&q=\nDisallow: /business/search*?q=\nDisallow: /checkpoint/\nDisallow: /contact_importer/\nDisallow: /dialog/\nDisallow: /fbml/ajax/dialog/\nDisallow: /feeds/\nDisallow: /file_download.php\nDisallow: /job_application/\nDisallow: /l.php\nDisallow: /login.php*&next=\nDisallow: /login.php/?next=\nDisallow: /login.php?next=\nDisallow: /login/*&next=\nDisallow: /login/?next=\nDisallow: /login/device-based/regular/login/*&next=\nDisallow: /login/device-based/regular/login/?next=\nDisallow: /meta-ai-answers/\nDisallow: /moments_app/\nDisallow: /p.php\nDisallow: /photos.php\nDisallow: /plugins/\nDisallow: /share.php\nDisallow: /share/\nDisallow: /sharer.php\nDisallow: /sharer/\nDisallow: /tr/\nDisallow: /tr?\nDisallow: /ufi/reaction/profile/browser/\nDisallow: /x/oauth/\nAllow: /ajax/bootloader-endpoint/\nAllow: /ajax/pagelet/generic.php/PagePostsSectionPagelet\nAllow: /careers/\nAllow: /safetycheck/\n\nUser-agent: teoma\nDisallow: /*/plugins/*\nDisallow: /?*next=\nDisallow: /a/bz?\nDisallow: /ajax/\nDisallow: /album.php\nDisallow: /business/*&categories\nDisallow: /business/*?categories\nDisallow: /business/help/search*&query=\nDisallow: /business/help/search*?query=\nDisallow: /business/news*&tags\nDisallow: /business/news*?tags\nDisallow: /business/search*&q=\nDisallow: /business/search*?q=\nDisallow: /checkpoint/\nDisallow: /contact_importer/\nDisallow: /dialog/\nDisallow: /fbml/ajax/dialog/\nDisallow: /feeds/\nDisallow: /file_download.php\nDisallow: /job_application/\nDisallow: /l.php\nDisallow: /login.php*&next=\nDisallow: /login.php/?next=\nDisallow: /login.php?next=\nDisallow: /login/*&next=\nDisallow: /login/?next=\nDisallow: /login/device-based/regular/login/*&next=\nDisallow: /login/device-based/regular/login/?next=\nDisallow: /meta-ai-answers/\nDisallow: /moments_app/\nDisallow: /p.php\nDisallow: /photos.php\nDisallow: /plugins/\nDisallow: /share.php\nDisallow: /share/\nDisallow: /sharer.php\nDisallow: /sharer/\nDisallow: /tr/\nDisallow: /tr?\nDisallow: /ufi/reaction/profile/browser/\nDisallow: /x/oauth/\nAllow: /ajax/bootloader-endpoint/\nAllow: /ajax/pagelet/generic.php/PagePostsSectionPagelet\nAllow: /careers/\nAllow: /safetycheck/\n\nUser-agent: TelegramBot\nDisallow: /*/plugins/*\nDisallow: /?*next=\nDisallow: /a/bz?\nDisallow: /ajax/\nDisallow: /album.php\nDisallow: /business/*&categories\nDisallow: /business/*?categories\nDisallow: /business/help/search*&query=\nDisallow: /business/help/search*?query=\nDisallow: /business/news*&tags\nDisallow: /business/news*?tags\nDisallow: /business/search*&q=\nDisallow: /business/search*?q=\nDisallow: /checkpoint/\nDisallow: /contact_importer/\nDisallow: /dialog/\nDisallow: /fbml/ajax/dialog/\nDisallow: /feeds/\nDisallow: /file_download.php\nDisallow: /job_application/\nDisallow: /l.php\nDisallow: /login.php*&next=\nDisallow: /login.php/?next=\nDisallow: /login.php?next=\nDisallow: /login/*&next=\nDisallow: /login/?next=\nDisallow: /login/device-based/regular/login/*&next=\nDisallow: /login/device-based/regular/login/?next=\nDisallow: /meta-ai-answers/\nDisallow: /moments_app/\nDisallow: /p.php\nDisallow: /photos.php\nDisallow: /plugins/\nDisallow: /share.php\nDisallow: /share/\nDisallow: /sharer.php\nDisallow: /sharer/\nDisallow: /tr/\nDisallow: /tr?\nDisallow: /ufi/reaction/profile/browser/\nDisallow: /x/oauth/\nAllow: /ajax/bootloader-endpoint/\nAllow: /ajax/pagelet/generic.php/PagePostsSectionPagelet\nAllow: /careers/\nAllow: /safetycheck/\n\nUser-agent: Twitterbot\nDisallow: /*/plugins/*\nDisallow: /?*next=\nDisallow: /a/bz?\nDisallow: /ajax/\nDisallow: /album.php\nDisallow: /business/*&categories\nDisallow: /business/*?categories\nDisallow: /business/help/search*&query=\nDisallow: /business/help/search*?query=\nDisallow: /business/news*&tags\nDisallow: /business/news*?tags\nDisallow: /business/search*&q=\nDisallow: /business/search*?q=\nDisallow: /checkpoint/\nDisallow: /contact_importer/\nDisallow: /dialog/\nDisallow: /fbml/ajax/dialog/\nDisallow: /feeds/\nDisallow: /file_download.php\nDisallow: /job_application/\nDisallow: /l.php\nDisallow: /login.php*&next=\nDisallow: /login.php/?next=\nDisallow: /login.php?next=\nDisallow: /login/*&next=\nDisallow: /login/?next=\nDisallow: /login/device-based/regular/login/*&next=\nDisallow: /login/device-based/regular/login/?next=\nDisallow: /meta-ai-answers/\nDisallow: /moments_app/\nDisallow: /p.php\nDisallow: /photos.php\nDisallow: /plugins/\nDisallow: /share.php\nDisallow: /share/\nDisallow: /sharer.php\nDisallow: /sharer/\nDisallow: /tr/\nDisallow: /tr?\nDisallow: /ufi/reaction/profile/browser/\nDisallow: /x/oauth/\nAllow: /ajax/bootloader-endpoint/\nAllow: /ajax/pagelet/generic.php/PagePostsSectionPagelet\nAllow: /careers/\nAllow: /safetycheck/\n\nUser-agent: *\nDisallow: /\n\n\n","agents":{"GPTBot":false,"OAI-SearchBot":false,"ChatGPT-User":false,"ClaudeBot":false,"Claude-SearchBot":false,"PerplexityBot":false,"Google-Extended":false,"Applebot-Extended":false,"CCBot":false,"Bytespider":false,"Meta-ExternalAgent":false,"Amazonbot":false}},"sitemap":null,"files":null,"derived":{"reachable":false,"home_class":"refused_us","robots_served":false,"robots_sitemap":false,"content_signal":false,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":[],"blocks_any_ai":false,"files":{"llms_txt":false,"agents_md":false,"agent_skills":false,"mcp_card":false,"api_catalog":false,"a2a_card":false},"any_capability":false,"ai_refused":[],"ai_thin":[],"rings":{"mcp_declared":false,"mcp_live":false,"mcp_auth":false,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":3,"fh":"m.facebook.com","fp_other_host":[]},"derive_v":5,"frame":"top10k","shard":2},{"v":1,"kind":"crawlcheck-census-record","run":18,"domain":"gstatic.com","rank":4,"measured_at":"2026-11-13T18:50:46.493Z","home":{"browser":{"status":404,"content_type":"text/html","bytes":1561,"sha256":"23926e9185d8d43c02807a838ffb373cc1977726094a4e46807c66ada9dd7660","final_host":"www.gstatic.com","final_path":"/","ms":75,"identity":"Mobile browser","html":true},"gptbot":{"status":404,"content_type":"text/html","bytes":1561,"sha256":"23926e9185d8d43c02807a838ffb373cc1977726094a4e46807c66ada9dd7660","final_host":"www.gstatic.com","final_path":"/","ms":76,"identity":"GPTBot","html":true},"claudebot":{"status":404,"content_type":"text/html","bytes":1561,"sha256":"23926e9185d8d43c02807a838ffb373cc1977726094a4e46807c66ada9dd7660","final_host":"www.gstatic.com","final_path":"/","ms":78,"identity":"ClaudeBot","html":true},"oaisearch":{"status":404,"content_type":"text/html","bytes":1561,"sha256":"23926e9185d8d43c02807a838ffb373cc1977726094a4e46807c66ada9dd7660","final_host":"www.gstatic.com","final_path":"/","ms":82,"identity":"OAI-SearchBot","html":true},"perplexity":{"status":404,"content_type":"text/html","bytes":1561,"sha256":"23926e9185d8d43c02807a838ffb373cc1977726094a4e46807c66ada9dd7660","final_host":"www.gstatic.com","final_path":"/","ms":76,"identity":"PerplexityBot","html":true}},"robots":{"status":200,"content_type":"text/plain","bytes":5359,"sha256":"413ae0b7d04ab2b4fea5bdcfcbec7456fed252ef8fbe7182cd9b1ed52a07524e","final_host":"www.gstatic.com","final_path":"/robots.txt","ms":131,"html":false,"text":"User-agent: *\nAllow: /_/aihub_pod/\nAllow: /_/alkali-learn/\nAllow: /_/apps-viewer/\nAllow: /_/atari/\nAllow: /_/boq/\nAllow: /_/boq-deepmind-visualization/\nAllow: /_/boq-play/\nAllow: /_/boq-search/\nAllow: /_/boutique/\nAllow: /_/buganizer/\nAllow: /_/compare-banking/\nAllow: /_/compare-insurance/\nAllow: /_/fi/\nAllow: /_/fonts/\nAllow: /_/mss/boq-bard-web/\nAllow: /_/mss/boq-chrome-cloudcast/\nAllow: /_/mss/boq-cloud-ai-agentis/\nAllow: /_/mss/boq-cloud-client/\nAllow: /_/mss/boq-cloud-web3-portal/\nAllow: /_/mss/boq-deepmind-visualization/\nAllow: /_/mss/boq-deepmind-alphafold-server/\nAllow: /_/mss/boq-discover-labs/\nAllow: /_/mss/boq-gdp/\nAllow: /_/mss/boq-geo/\nAllow: /_/mss/boq-home-enterprise/\nAllow: /_/mss/boq-makersuite/\nAllow: /_/mss/boq-nbu-monetization/\nAllow: /_/mss/boq-search/\nAllow: /_/mss/boq-search-engagement-post/\nAllow: /_/mss/boq-rsvp/\nAllow: /_/mss/boq-travel/\nAllow: /_/mss/boq-identity/\nAllow: /_/mss/boq-labs-tailwind/\nAllow: /_/pantheon/\nAllow: /_/pantheon-platform/\nAllow: /_/play/\nAllow: /_/pog/\nAllow: /_/cloudsourcetools/\nAllow: /_/devopsconsole/\nAllow: /_/tfhub/\nAllow: /_/transparencyreport/\nAllow: /_/gstore/\nAllow: /_/ain/_/js/\nAllow: /ac/\nAllow: /actions/news/rss/\nAllow: /ads/learn/\nAllow: /ads/local/\nAllow: /ads/gpa/vidgen/\nAllow: /ads/research/\nAllow: /ads/ubaq/\nAllow: /agentis/omnichannel/public/\nAllow: /aistudio/*favicon*.png$\nAllow: /aistudio-static/\nAllow: /ai-sandbox-internal/generated_images/\nAllow: /aihub/tfhub/\nAllow: /aitestkitchen/website/flow/applets/\nAllow: /aitestkitchen/website/flow/social/\nAllow: /alkali/\nAllow: /alphafoldserver/\nAllow: /amphtml/stamp/ucp/post-testing/\nAllow: /android/market/\nAllow: /android/market_images/\nAllow: /antigravity/logos/\nAllow: /atari/images/public/\nAllow: /audio_news/\nAllow: /awmp/data/images/\nAllow: /bfe/apps/website/\nAllow: /boutique/sitemaps/\nAllow: /bricks/image/\nAllow: /compare/\nAllow: /chrome/crlset/\nAllow: /chrome/webstore/images/\nAllow: /chromoting/chromoting_logo_*.png$\nAllow: /chromoting/favicon.ico$\nAllow: /cloud-launcher/sitemap/\nAllow: /cloudsourcetools/images/\nAllow: /commercesearch/images/\nAllow: /communities/sitemap/\nAllow: /copy-conductor/crawled/\nAllow: /covid19/mobility/\nAllow: /creatorprofile/branding/\nAllow: /culturalinstitute/images/\nAllow: /culturalinstitute/sitemaps/\nAllow: /culturalinstitute/stella/\nAllow: /ct/log_list/\nAllow: /delight/\nAllow: /devopsconsole/images/\nAllow: /devrel-devsite/\nAllow: /docs/forms/social/\nAllow: /docs/socialmarkup/\nAllow: /dictionary/static/sitemaps/\nAllow: /droidguard/\nAllow: /earth/social/\nAllow: /earth/00-favicon.ico\nAllow: /enterprise-partner-search/sitemaps/\nAllow: /external_hosted/\nAllow: /fi/web/\nAllow: /flights/\nAllow: /flutter-canvaskit/\nAllow: /foodordering/\nAllow: /frommers/\nAllow: /gb/images/\nAllow: /glass/images/commerce/\nAllow: /gpay/social/\nAllow: /growth-ime/\nAllow: /gsx/\nAllow: /gtp/stir/\nAllow: /guidebooks/\nAllow: /GoogleInternetAuthority/\nAllow: /h5games_gamecenter/images/\nAllow: /healthricherkp/illustrations/\nAllow: /healthricherkp/cropped/\nAllow: /hex/\nAllow: /history/static/\nAllow: /identity/accountsettingsactivityweb/\nAllow: /identity/accountsettingssyaf/\nAllow: /images\nAllow: /inproduct_help/chatsupport/\nAllow: /keep/\nAllow: /lamda/images/\nAllow: /labs-code/\nAllow: /local/donut/\nAllow: /mapspro/\nAllow: /marketing-cms/\nAllow: /moderator/static/ux/\nAllow: /modern-formats-eng/ucp/framework_logging/\nAllow: /modern-formats-eng/ucp/*/client/\nAllow: /modern-formats-eng/ucp/*/csr/\nAllow: /modern-formats-eng/ucp/*/dev/\nAllow: /modern-formats-eng/ucp/static/framework_logging/\nAllow: /modern-formats-eng/ucp/static/*/client/\nAllow: /modern-formats-eng/ucp/static/*/csr/\nAllow: /modern-formats-eng/ucp/static/*/dev/\nAllow: /mtcs/cosigners/\nAllow: /nbubumblebee/\nAllow: /og/\nAllow: /ohttp_gateway/\nAllow: /onebox/media/sports/logos/\nAllow: /onebox/media/sports/photos/soccer/wc2018/\nAllow: /onebox/sports/logos/\nAllow: /onebox/sports/logos/sports_types/png/\nAllow: /pagespeed/insights/ui/logo/\nAllow: /pantheon/\nAllow: /pantheon-platform/\nAllow: /patent-search/frontend/\nAllow: /pay/images/merchant_logos/\nAllow: /pay/images/program_logos/\nAllow: /paygo/\nAllow: /play/store/web/\nAllow: /play/surveys/serp/\nAllow: /podcasts/test-podcast/rss/\nAllow: /politics/e/sharing/\nAllow: /prettyearth/assets/\nAllow: /psb/udc/\nAllow: /rsvp/images/\nAllow: /s2/sitemaps/\nAllow: /search-ar-dev/\nAllow: /search-console/scfe/\nAllow: /share\nAllow: /shopping-content-api/product_images/\nAllow: /shopping-organic/price_validation/\nAllow: /shopping/shoppingpage/casa/\nAllow: /shopstream/\nAllow: /sites/\nAllow: /sitemaps/\nAllow: /social/photosui/images/\nAllow: /social/photosui/sitemap/\nAllow: /stadia/gamers/\nAllow: /store/images/favicons/\nAllow: /subscriptions/\nAllow: /support/content/\nAllow: /system1/images/\nAllow: /travel-frontend/images/\nAllow: /finance/favicon/\nAllow: /travel-vacations/\nAllow: /touringbird/\nAllow: /trends/\nAllow: /trends_tpt/\nAllow: /trustedstores/images/\nAllow: /trustedstores/js/\nAllow: /tv/thumb/\nAllow: /ui/\nAllow: /violet-web/previewer/\nAllow: /vr/*\nAllow: /vsi/*.pdf$\nAllow: /wondermissive/interest/\nAllow: /youtube/fonts/\nAllow: /youtube/img/branding/favicon/\nAllow: /youtube/img/promos/\nAllow: /icons/\nAllow: /js/\nAllow: /ss/\nAllow: /*.js\nAllow: /*.css\nAllow: /*.svg\nAllow: /buyside-e2e-monitor/\nAllow: /cgc/*.ico\nAllow: /cgc/*.png\nAllow: /cgc/*.svg\n\nDisallow: /\n","agents":{"GPTBot":false,"OAI-SearchBot":false,"ChatGPT-User":false,"ClaudeBot":false,"Claude-SearchBot":false,"PerplexityBot":false,"Google-Extended":false,"Applebot-Extended":false,"CCBot":false,"Bytespider":false,"Meta-ExternalAgent":false,"Amazonbot":false}},"sitemap":null,"files":null,"derived":{"reachable":false,"home_class":"other","robots_served":false,"robots_sitemap":false,"content_signal":false,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":[],"blocks_any_ai":false,"files":{"llms_txt":false,"agents_md":false,"agent_skills":false,"mcp_card":false,"api_catalog":false,"a2a_card":false},"any_capability":false,"ai_refused":[],"ai_thin":[],"rings":{"mcp_declared":false,"mcp_live":false,"mcp_auth":false,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":3,"fh":"www.gstatic.com","fp_other_host":[]},"derive_v":5,"frame":"top10k","shard":3},{"v":1,"kind":"crawlcheck-census-record","run":18,"domain":"amazonaws.com","rank":5,"measured_at":"2026-11-13T18:51:40.579Z","home":{"browser":{"status":200,"content_type":"text/html","bytes":506736,"sha256":"fdb88bac96286ccc889c8afa46c358e0729530765709f8d67cfd94cbfebceb56","final_host":"aws.amazon.com","final_path":"/","ms":238,"identity":"Mobile browser","html":true,"fp":"138fbbc185e891da"},"gptbot":{"status":200,"content_type":"text/html","bytes":506736,"sha256":"87868bfe78dedadaf97b11ab841dcb46eac5e384cae7782d97709e5795e6f37a","final_host":"aws.amazon.com","final_path":"/","ms":255,"identity":"GPTBot","html":true,"fp":"138fbbc185e891da"},"claudebot":{"status":200,"content_type":"text/html","bytes":506736,"sha256":"cb8bf790c7cd8d60e2d9418d76d9ea5e2b607c35f4c5d110cd320c0d1c93f83d","final_host":"aws.amazon.com","final_path":"/","ms":238,"identity":"ClaudeBot","html":true,"fp":"138fbbc185e891da"},"oaisearch":{"status":200,"content_type":"text/html","bytes":506736,"sha256":"f51ff12806e5de0c85fd773c4cc7d2e604815a18600c9cc1e299bb4c8842ad80","final_host":"aws.amazon.com","final_path":"/","ms":238,"identity":"OAI-SearchBot","html":true,"fp":"138fbbc185e891da"},"perplexity":{"status":200,"content_type":"text/html","bytes":506736,"sha256":"1f78f056f28677a1d4123190a8b398abaab7d8b93b3f2eea53922d703fc79ff6","final_host":"aws.amazon.com","final_path":"/","ms":237,"identity":"PerplexityBot","html":true,"fp":"138fbbc185e891da"}},"robots":{"status":200,"content_type":"text/plain","bytes":13879,"sha256":"a0223b0119cce646eab6f085445bbce4224747c91153e245e4d2031ce56da39e","final_host":"aws.amazon.com","final_path":"/robots.txt","ms":255,"html":false,"text":"User-agent: AdsBot-Google\r\nUser-agent: AdsBot-Google-Mobile\r\nAllow: /pm/\r\nDisallow: /alexaforbusiness/\r\nDisallow: /blogs/\r\nDisallow: /*/blogs/\r\nDisallow: /solutions/case-studies/\r\nDisallow: /*/solutions/case-studies/\r\nAllow: /solutions/case-studies/sustainability-emea/\r\nAllow: /*/solutions/case-studies/sustainability-emea/\r\nAllow: /solutions/case-studies/innovators/innovators-eu/\r\nAllow: /*/solutions/case-studies/innovators/innovators-eu/\r\nAllow: /solutions/case-studies/generative-ai-ekacare/\r\nAllow: /cn/solutions/case-studies/generative-ai-ekacare/\r\nAllow: /solutions/case-studies/generative-ai-mindtickle/\r\nAllow: /cn/solutions/case-studies/generative-ai-mindtickle/\r\nAllow: /solutions/case-studies/generative-ai-invideo/\r\nAllow: /solutions/case-studies/generative-ai-mitsui/\r\nAllow: /solutions/case-studies/generative-ai-national-australia-bank/\r\nAllow: /solutions/case-studies/generative-ai-ntu/\r\nAllow: /solutions/case-studies/generative-ai-qarc/\r\nAllow: /solutions/case-studies/generative-ai-swimming-australia/\r\nAllow: /cn/solutions/case-studies/generative-ai-swimming-australia/\r\nAllow: /solutions/case-studies/generative-ai-thematic/\r\nAllow: /cn/solutions/case-studies/generative-ai-thematic/\r\nAllow: /solutions/case-studies/generative-ai-turing/\r\nAllow: /solutions/case-studies/generative-ai-stockmark/   \r\nAllow: /*/solutions/case-studies/generative-ai-turing/\r\nAllow: /*/solutions/case-studies/generative-ai-stockmark/   \r\nAllow: /solutions/case-studies/generative-ai-starhub/    \r\nAllow: /solutions/case-studies/generative-ai-bolttech/\r\nAllow: /*/solutions/case-studies/generative-ai-starhub/    \r\nAllow: /*/solutions/case-studies/generative-ai-bolttech/\nAllow: /solutions/case-studies/nba-migration-case-study/\r\n\r\nUser-agent: *\r\n# Block old What's New posts to preserve crawl budget for current content\r\nDisallow: /about-aws/whats-new/200\r\nDisallow: /about-aws/whats-new/201\r\nDisallow: /about-aws/whats-new/2020\r\nDisallow: /about-aws/whats-new/2021\r\nDisallow: /about-aws/whats-new/2022\r\nDisallow: /about-aws/whats-new/2023\r\nDisallow: /about-aws/whats-new/2024\n# Block tag\nDisallow: /*/blogs/*/tag/\nDisallow: /blogs/*/tag/\n\r\n# Other\r\nDisallow: */confirmation/\r\nDisallow: */registration-confirmation/\r\nDisallow: /about-aws/launch-a-website-in-minutes-tutorial/\r\nDisallow: /about-aws/media-coverage/2011/08/17/rhat-support-for-sap-solutions-running-on-rhel-on-amazon-ec2/\r\nDisallow: /activate/accelerators\r\nDisallow: /activate/accelerators/*\r\nDisallow: /activate/accelerators/confirmation/\r\nDisallow: /activate/china-portfolio-signup/\r\nDisallow: /activate/cloudkata/\r\nDisallow: /activate/event/\r\nDisallow: /activate/event/*\r\nDisallow: /activate/events/*\r\nDisallow: /activate/events/hact/\r\nDisallow: /activate/hackathons\r\nDisallow: /activate/hackathons/*\r\nDisallow: /activate/harvard-innovation-lab-students-2013/\r\nDisallow: /activate/harvard-innovation-lab-vip-teams-2013/\r\nDisallow: /activate/hsw/confirmation/\r\nDisallow: /activate/loft/\r\nDisallow: /activate/self-starters/confirmation/\r\nDisallow: /activate/wdm/*\r\nDisallow: /alexa/*\r\nDisallow: /alexa-top-sites/*\r\nDisallow: /alexa-web-search/*\r\nDisallow: /apac/aws-credits-confirmation/\r\nDisallow: /apac/aws-credits-request-confirmation/\r\nDisallow: /articles/item/\r\nDisallow: /awis/*\r\nDisallow: /aws-summit-2014/enterprise-london-db/\r\nDisallow: /aws-summit-2014/london-db/\r\nDisallow: /blogs/7faebe5debfdd358a46f2452a54eec94fce49269/\r\nDisallow: /campaigns/infor-hub-xt5r5iwdrgxbnuz/\r\nDisallow: /careers/aus-nz-event/\r\nDisallow: /careers/aus-nz-event/\r\nDisallow: /careers/aws-ar-event/\r\nDisallow: /careers/aws-aus-event/\r\nDisallow: /careers/aws-australia-event/\r\nDisallow: /careers/aws-bigdata-hiring/\r\nDisallow: /careers/aws-brisbane-event/\r\nDisallow: /careers/aws-db-event/\r\nDisallow: /careers/aws-ks-event/\r\nDisallow: /careers/aws-london-event/\r\nDisallow: /careers/aws-mexicocity-event/\r\nDisallow: /careers/aws-miami-event/\r\nDisallow: /careers/aws-ny-event/\r\nDisallow: /careers/aws-pittsburgh-event/\r\nDisallow: /careers/aws-portland-event/\r\nDisallow: /careers/aws-sea-event/\r\nDisallow: /careers/aws-singapore-event/\r\nDisallow: /careers/aws-sydney-event/\r\nDisallow: /careers/chattanooga-mobile-event/\r\nDisallow: /careers/database-hiring/\r\nDisallow: /careers/devtools-australia-event/\r\nDisallow: /careers/devtools-australia-event/\r\nDisallow: /careers/ebs-cincinnati-event/\r\nDisallow: /careers/ec2-australia-event/\r\nDisallow: /careers/ec2-chicago-event/\r\nDisallow: /careers/ec2-hiring-events/\r\nDisallow: /careers/ec2-ottawa-event/\r\nDisallow: /careers/ec2-vancouver-event/\r\nDisallow: /careers/indianapolis-event/\r\nDisallow: /careers/mexico-edge-hiring-2014/\r\nDisallow: /careers/milwaukee-edge-hiring-2014/\r\nDisallow: /careers/montreal-edge-hiring/\r\nDisallow: /careers/Phoenix-Hiring-Event/\r\nDisallow: /careers/pittsburgh-platform-hiring-2014/\r\nDisallow: /careers/raleigh-mobile-event/\r\nDisallow: /careers/raleigh-storage-hiring-2014/\r\nDisallow: /careers/santiago-event/\r\nDisallow: /careers/Sydney-Hiring-Event/\r\nDisallow: /careers/texas-event/\r\nDisallow: /careers/toronto-event/\r\nDisallow: /careers/toronto-storage-hiring-2014/\r\nDisallow: /Cloud_Start_01.LandingPage.html\r\nDisallow: /Cloud_Start_01-Landing-Page-EDO.html\r\nDisallow: /Cloud_Start_02.ConfirmationPage.html\r\nDisallow: /code/*\r\nDisallow: /compliance/contact-us-confirmation/\r\nDisallow: /compliance/hipaa-eligible-services-reference/\r\nDisallow: /compliance/how-to-open-aws-compliance-reports/\r\nDisallow: /compliance/public-sector-contact-confirmation/\r\nDisallow: /compliance/security-assurance-faq\r\nDisallow: /compliance/security-assurance-faq-1\r\nDisallow: /compliance/security-assurance-faq-10\r\nDisallow: /compliance/security-assurance-faq-2\r\nDisallow: /compliance/security-assurance-faq-3\r\nDisallow: /compliance/security-assurance-faq-4\r\nDisallow: /compliance/security-assurance-faq-5\r\nDisallow: /compliance/security-assurance-faq-6\r\nDisallow: /compliance/security-assurance-faq-7\r\nDisallow: /compliance/security-assurance-faq-8\r\nDisallow: /compliance/security-assurance-faq-9\r\nDisallow: /compliance/security-assurance-faq-complete\r\nDisallow: /compliance/security-assurance-US-Public-Sector-checklist\r\nDisallow: /compliance/security-assurance-US-Public-Sector-checklist-1\r\nDisallow: /compliance/security-assurance-US-Public-Sector-checklist-10\r\nDisallow: /compliance/security-assurance-US-Public-Sector-checklist-2\r\nDisallow: /compliance/security-assurance-US-Public-Sector-checklist-3\r\nDisallow: /compliance/security-assurance-US-Public-Sector-checklist-4\r\nDisallow: /compliance/security-assurance-US-Public-Sector-checklist-5\r\nDisallow: /compliance/security-assurance-US-Public-Sector-checklist-6\r\nDisallow: /compliance/security-assurance-US-Public-Sector-checklist-7\r\nDisallow: /compliance/security-assurance-US-Public-Sector-checklist-8\r\nDisallow: /compliance/security-assurance-US-Public-Sector-checklist-9\r\nDisallow: /compliance/security-assurance-US-Public-Sector-checklist-complete\r\nDisallow: /compliance/third-party-access/\r\nDisallow: /contact-us/ami-submission-conf/\r\nDisallow: /contact-us/article-submission-conf/\r\nDisallow: /contact-us/case-contact-confirmation/\r\nDisallow: /contact-us/customer-app-submission-conf/\r\nDisallow: /contact-us/developertool-submission-conf/\r\nDisallow: /contact-us/ec2-email-limit-request-confirmation/\r\nDisallow: /contact-us/elasticache-node-limit-request-confirmation/\r\nDisallow: /contact-us/marketing-unsubscribe-confirmation/\r\nDisallow: /contact-us/publicdataset-conf/\r\nDisallow: /contact-us/sales-contact-confirmation/\r\nDisallow: /contact-us/samplecode-submission-conf/\r\nDisallow: /csds/*\r\nDisallow: /customerapps/*\r\nDisallow: /datasets/\r\nDisallow: /developertools/2555753788650372\r\nDisallow: /documentation/alexa-top-sites/\r\nDisallow: /documentation/awis/\r\nDisallow: /email-preferences\r\nDisallow: /email-preferences/\r\nDisallow: /events/applesummit\r\nDisallow: /events/cio-advisory-council/\r\nDisallow: /events/finserv-summit-dublin\r\nDisallow: /events/finserv-summit-dublin/confirmation\r\nDisallow: /events/finserv-summit-singapore\r\nDisallow: /events/finserv-summit-singapore/confirmation\r\nDisallow: /events/fsi-cloud-symposium-nyc.html\r\nDisallow: /federal/events/earthobservationday2015/\r\nDisallow: /federal/events/earthobservationday2015/ConfirmationPage/\r\nDisallow: /forms/activate-support\r\nDisallow: /free/cloud-computing-free-tier/\r\nDisallow: /free/databases-free-tier/\r\nDisallow: /get-cloud/\r\nDisallow: /getting-started/functional-groups/\r\nDisallow: /getting-started/functional-groups/*\r\nDisallow: /government-education/nonprofits/aws-serve/\r\nDisallow: /government-education/nonprofits/aws-serve-member/\r\nDisallow: /government-education/nonprofits/credit-program/\r\nDisallow: /government-education/nonprofits/credit-program/confirmation\r\nDisallow: /government-education/nonprofits/credit-program/terms\r\nDisallow: /government-education/nonprofits/grant-program/\r\nDisallow: /government-education/nonprofits/grant-program/confirmation/\r\nDisallow: /government-education/nonprofits/grant-program/terms/\r\nDisallow: /government-education/nonprofits/leadership-forum-poc\r\nDisallow: /government-education/nonprofits/leadership-forum-poc/confirmation\r\nDisallow: /government-education/public-sector-cloud-start/public-sector-cloud-start/\r\nDisallow: /government-education/public-sector-cloud-start/public-sector-cloud-start1/\r\nDisallow: /hackathons/reinvent-2013/\r\nDisallow: /legal/3874403/\r\nDisallow: /lms/contact-us/\r\nDisallow: /loft/\r\nDisallow: /marketplace/reviews/reviews-list/*/write-comment/\r\nDisallow: /marketplace/reviews/write-review/\r\nDisallow: /marketplace/search*\r\nDisallow: /microsoft/bizspark/\r\nDisallow: /microsoft/bizspark/signon/thankyou/\r\nDisallow: /nav/\r\nDisallow: /nav/side-nav/\r\nDisallow: /nav/side-nav/products-services/\r\nDisallow: /nav/tabs/\r\nDisallow: /nav/tabs/account/\r\nDisallow: /nav/tabs/aws/\r\nDisallow: /nav/tabs/community/\r\nDisallow: /nav/tabs/developers/\r\nDisallow: /nav/tabs/products/\r\nDisallow: /nav/tabs/support/\r\nDisallow: /optin/\r\nDisallow: /optout/\r\nDisallow: /partners/contact-us/partners-confirmation/\r\nDisallow: /partners/overview/partner-newsletter/confirmation/\r\nDisallow: /partners/overview/partner-terms-and-conditions/\r\nDisallow: /partners/webinars/request/\r\nDisallow: /ping\r\nDisallow: /pm/\r\nDisallow: /ar/pm/\r\nDisallow: /cn/pm/\r\nDisallow: /de/pm/\r\nDisallow: /es/pm/\r\nDisallow: /fr/pm/\r\nDisallow: /id/pm/\r\nDisallow: /it/pm/\r\nDisallow: /jp/pm/\r\nDisallow: /ko/pm/\r\nDisallow: /pt/pm/\r\nDisallow: /ru/pm/\r\nDisallow: /th/pm/\r\nDisallow: /tr/pm/\r\nDisallow: /tw/pm/\r\nDisallow: /vi/pm/\r\nAllow: /pm/amplify/\r\nAllow: /pm/lambda/\r\nAllow: /pm/sagemaker/\r\nAllow: /pm/textract/\r\nAllow: /pm/transcribe/\r\nDisallow: /premiumsupport/iam/\r\nDisallow: /premiumsupport/knowledge-center/feedback-no/\r\nDisallow: /premiumsupport/knowledge-center/feedback-yes/\r\nDisallow: /pricing/awis/\r\nDisallow: /public-data-sets/applicants/\r\nDisallow: /public-data-sets/terms/\r\nDisallow: /publicdatasets-confirmation/\r\nDisallow: /quickstart/promo\r\nDisallow: /quickstart/promo/confirmation\r\nDisallow: /reinvent/video-case-studies\r\nDisallow: /releasenotes/item/\r\nDisallow: /research-credits/data-science-bowl\r\nDisallow: /ru/blogs/497c2c2d305b7dfa351d17bd3b4398dc37b0e0a4/\r\nDisallow: /s/*\r\nDisallow: /s/dm/house/editorial/startups/ha_ed_activate_loft_launch_sf_1/\r\nDisallow: /sap/saphana/pilot/\r\nDisallow: /sap/saphana/pilot/offer/\r\nDisallow: /sap/saphana/pilot/offer/confirmation\r\nDisallow: /search\r\nDisallow: /search/\r\nAllow: /api/dirs/items/search\r\nDisallow: /security/aws-pgp-public-key\r\nDisallow: /security/ec2-pkgs-public-key\r\nDisallow: /security/introducing-s2n-new-open-source-TLS-implementation\r\nDisallow: /s-moved/dm/house/editorial/startups/ha_ed_activate_loft_launch_sf_1/\r\nDisallow: /solutions/case-studies/imdb/\r\nDisallow: /solutions/featured-partners/\r\nDisallow: /solutions/global-solution-providers/microsoft/bizspark/\r\nDisallow: /solutions/global-solution-providers/microsoft/bizspark/faq/\r\nDisallow: /solutions/global-solution-providers/microsoft/bizspark/signon/\r\nDisallow: /solutions/global-solution-providers/microsoft/bizspark/signon/\r\nDisallow: /solutions/global-solution-providers/microsoft/bizspark/signon/\r\nDisallow: /solutions/global-solution-providers/microsoft/bizspark/signon/thankyou/\r\nDisallow: /solutions/global-solution-providers/microsoft/bizspark/signon/thankyou/\r\nDisallow: /solutions/solution-providers/program/reseller/adguidelines/\r\nDisallow: /solutions/solution-providers/program/reseller/adguidelines/\r\nDisallow: /solutions/solution-providers/program/reseller/authorized/\r\nDisallow: /solutions/solution-providers/program/reseller/authorized/\r\nDisallow: /solutions/solution-providers/program/reseller/customerterms/\r\nDisallow: /solutions/solution-providers/program/reseller/customerterms/\r\nDisallow: /specialhandling-payments/\r\nDisallow: /start-ups/challenge/\r\nDisallow: /start-ups/loft/hack/\r\nDisallow: /start-ups/loft/hacknight/\r\nDisallow: /start-ups/loft/sf2014/\r\nDisallow: /start-ups/loft/sfff100l3ft\r\nDisallow: /start-ups/loft/survey/\r\nDisallow: /start-ups/loft/welcome\r\nDisallow: /unable-to-process-request/\r\nDisallow: /use-cases/global-solution-providers/microsoft/bizspark/\r\nDisallow: /use-cases/global-solution-providers/microsoft/bizspark/getting-started/\r\nDisallow: /use-cases/global-solution-providers/microsoft/bizspark/signon/\r\nDisallow: /use-cases/global-solution-providers/microsoft/bizspark/signon/thankyou/\r\nDisallow: /use-cases/global-solution-providers/microsoft/bizspark/thankyou/\r\nDisallow: /vmware/about\r\nDisallow: /windows/resources/whitepapers/\r\nDisallow: /windows/server2003/access/\r\nDisallow: /z_dx-marketing-team-prototypes/*\r\nDisallow: /community/*\r\nDisallow: /community/*\r\nDisallow: /community/heroes/\r\nDisallow: /*/community/heroes/\r\nDisallow: /*/search\r\nDisallow: /*/search/\r\nSitemap: https://aws.amazon.com/sitemaps/index/\r\nSitemap: https://d36a099zv34xie.cloudfront.net/psf-sitemap.xml\r\nSitemap: https://aws.amazon.com/marketplace/sitemap","agents":{"GPTBot":true,"OAI-SearchBot":true,"ChatGPT-User":true,"ClaudeBot":true,"Claude-SearchBot":true,"PerplexityBot":true,"Google-Extended":true,"Applebot-Extended":true,"CCBot":true,"Bytespider":true,"Meta-ExternalAgent":true,"Amazonbot":true}},"sitemap":{"status":200,"content_type":"application/xml","bytes":2669,"sha256":"bff9f22781907b75f4d48932662037d9202b67fbd3a69233f71b71037c9ae4a8","final_host":"aws.amazon.com","final_path":"/sitemaps/index","ms":56,"url":"https://aws.amazon.com/sitemaps/index/","declared":true,"xml":true},"files":{"llms_txt":{"status":404,"content_type":"text/html","bytes":299216,"sha256":"689bc4d7f32f25453b3204eb29a29587704d08a3496b3302ed1b95aa7c7e0e69","final_host":"aws.amazon.com","final_path":"/llms.txt","ms":103,"path":"/llms.txt","html":true,"json":null,"present":false},"agents_md":{"status":404,"content_type":"text/html","bytes":299216,"sha256":"9ae58c0496ef62cb1d861439038a28f31cb2d9420ee88882fc465a1755bcabc6","final_host":"aws.amazon.com","final_path":"/agents.md","ms":195,"path":"/agents.md","html":true,"json":null,"present":false},"agent_skills":{"status":404,"content_type":"text/html","bytes":299216,"sha256":"b944fba8f930d6f6ee9a65681cf33d66c99df4cb7254b09192ddd4238a576492","final_host":"aws.amazon.com","final_path":"/.well-known/agent-skills/index.json","ms":166,"path":"/.well-known/agent-skills/index.json","html":true,"json":null,"present":false},"mcp_card":{"status":404,"content_type":"text/html","bytes":299216,"sha256":"b62aac7c917b4e0cd870c6369678b09ec3f08789eb5098a0eda36d7664af0541","final_host":"aws.amazon.com","final_path":"/.well-known/mcp/server-card.json","ms":148,"path":"/.well-known/mcp/server-card.json","html":true,"json":null,"present":false},"api_catalog":{"status":404,"content_type":"text/html","bytes":299216,"sha256":"22df4526054b81c19127e53bd5f61cb39a6fba16334c2b18d4f7d05c0711dfb3","final_host":"aws.amazon.com","final_path":"/.well-known/api-catalog","ms":157,"path":"/.well-known/api-catalog","html":true,"json":null,"present":false},"a2a_card":{"status":404,"content_type":"text/html","bytes":299216,"sha256":"852ad9e34cb38897af88950228171082e5d49c76fb60aa03e063ec3897deecb1","final_host":"aws.amazon.com","final_path":"/.well-known/agent-card.json","ms":186,"path":"/.well-known/agent-card.json","html":true,"json":null,"present":false}},"rings":{"v":1,"candidates":[],"mcp":[],"a2a":null,"webmcp":{"model_context":false,"bridge":false,"mcp_json":false,"link_rel":false,"any":false},"handshakes":0,"get_path":{"status":404,"content_type":"text/html","ms":155,"error":null},"get_host":{"status":530,"content_type":"text/plain","ms":14,"error":null}},"sf":[4,0,0,0,0,0,5,0,0],"derived":{"reachable":true,"home_class":"answers","robots_served":true,"robots_sitemap":true,"content_signal":false,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":[],"blocks_any_ai":false,"files":{"llms_txt":false,"agents_md":false,"agent_skills":false,"mcp_card":false,"api_catalog":false,"a2a_card":false},"any_capability":false,"ai_refused":[],"ai_thin":[],"aipref":0,"aipref_measured":true,"sf":[4,0,0,0,0,0,5,0,0],"rings":{"mcp_declared":false,"mcp_live":false,"mcp_auth":false,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":3,"fh":"aws.amazon.com","fp_other_host":[],"fp":"138fbbc185e891da","fp_bits":{"gptbot":0,"claudebot":0,"oaisearch":0,"perplexity":0}},"derive_v":5,"frame":"top10k","shard":4},{"v":1,"kind":"crawlcheck-census-record","run":18,"domain":"googleapis.com","rank":6,"measured_at":"2026-11-13T18:52:33.408Z","home":{"browser":{"status":404,"content_type":"text/html","bytes":1561,"sha256":"23926e9185d8d43c02807a838ffb373cc1977726094a4e46807c66ada9dd7660","final_host":"googleapis.com","final_path":"/","ms":467,"identity":"Mobile browser","html":true},"gptbot":{"status":404,"content_type":"text/html","bytes":1561,"sha256":"23926e9185d8d43c02807a838ffb373cc1977726094a4e46807c66ada9dd7660","final_host":"googleapis.com","final_path":"/","ms":480,"identity":"GPTBot","html":true},"claudebot":{"status":404,"content_type":"text/html","bytes":1561,"sha256":"23926e9185d8d43c02807a838ffb373cc1977726094a4e46807c66ada9dd7660","final_host":"googleapis.com","final_path":"/","ms":455,"identity":"ClaudeBot","html":true},"oaisearch":{"status":404,"content_type":"text/html","bytes":1561,"sha256":"23926e9185d8d43c02807a838ffb373cc1977726094a4e46807c66ada9dd7660","final_host":"googleapis.com","final_path":"/","ms":455,"identity":"OAI-SearchBot","html":true},"perplexity":{"status":404,"content_type":"text/html","bytes":1561,"sha256":"23926e9185d8d43c02807a838ffb373cc1977726094a4e46807c66ada9dd7660","final_host":"googleapis.com","final_path":"/","ms":477,"identity":"PerplexityBot","html":true}},"robots":{"status":404,"content_type":"text/html","bytes":1571,"sha256":"6adee83a3eba87d30fdca3997d27097697f2767805f222a0f78c3f04392004d2","final_host":"googleapis.com","final_path":"/robots.txt","ms":565,"html":true,"text":null,"agents":null},"sitemap":null,"files":null,"derived":{"reachable":false,"home_class":"other","robots_served":false,"robots_sitemap":false,"content_signal":false,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":[],"blocks_any_ai":false,"files":{"llms_txt":false,"agents_md":false,"agent_skills":false,"mcp_card":false,"api_catalog":false,"a2a_card":false},"any_capability":false,"ai_refused":[],"ai_thin":[],"rings":{"mcp_declared":false,"mcp_live":false,"mcp_auth":false,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":3,"fh":"googleapis.com","fp_other_host":[]},"derive_v":5,"frame":"top10k","shard":5},{"v":1,"kind":"crawlcheck-census-record","run":18,"domain":"akamai.net","rank":7,"measured_at":"2026-11-13T18:53:15.625Z","home":{"browser":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"akamai.net","final_path":"/","ms":12,"identity":"Mobile browser","html":false},"gptbot":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"akamai.net","final_path":"/","ms":12,"identity":"GPTBot","html":false},"claudebot":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"akamai.net","final_path":"/","ms":12,"identity":"ClaudeBot","html":false},"oaisearch":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"akamai.net","final_path":"/","ms":12,"identity":"OAI-SearchBot","html":false},"perplexity":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"akamai.net","final_path":"/","ms":12,"identity":"PerplexityBot","html":false}},"robots":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"akamai.net","final_path":"/robots.txt","ms":12,"html":false,"text":null,"agents":null},"sitemap":null,"files":null,"derived":{"reachable":false,"home_class":"no_site","robots_served":false,"robots_sitemap":false,"content_signal":false,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":[],"blocks_any_ai":false,"files":{"llms_txt":false,"agents_md":false,"agent_skills":false,"mcp_card":false,"api_catalog":false,"a2a_card":false},"any_capability":false,"ai_refused":[],"ai_thin":[],"rings":{"mcp_declared":false,"mcp_live":false,"mcp_auth":false,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":3,"fh":"akamai.net","fp_other_host":[]},"derive_v":5,"frame":"top10k","shard":6},{"v":1,"kind":"crawlcheck-census-record","run":18,"domain":"microsoft.com","rank":8,"measured_at":"2026-11-13T18:55:41.951Z","home":{"browser":{"status":200,"content_type":"text/html","bytes":212994,"sha256":"e9030bd01e51e58efe9669be8c08956ca365e2d76f4111dc6e371c76b9382db5","final_host":"www.microsoft.com","final_path":"/ja-jp","ms":124,"identity":"Mobile browser","html":true,"fp":"8dd4227f8c44e03e"},"gptbot":{"status":200,"content_type":"text/html","bytes":212994,"sha256":"be388605c3b34aeb42cbbea02407921cf06e5de8a4d615eaaa20e98f7d897980","final_host":"www.microsoft.com","final_path":"/ja-jp","ms":100,"identity":"GPTBot","html":true,"fp":"8dd4227f8c44e03e"},"claudebot":{"status":200,"content_type":"text/html","bytes":212994,"sha256":"1fd4163f5511a8f5815b39b5a728b9e500e0d4d2302753260056d6eb5bf28817","final_host":"www.microsoft.com","final_path":"/ja-jp","ms":125,"identity":"ClaudeBot","html":true,"fp":"8dd4227f8c44e03e"},"oaisearch":{"status":200,"content_type":"text/html","bytes":212994,"sha256":"802ad931cec8588e75cf1002476d16b53d5e632dc56a4fa3ba91fe55b9b2143d","final_host":"www.microsoft.com","final_path":"/ja-jp","ms":106,"identity":"OAI-SearchBot","html":true,"fp":"8dd4227f8c44e03e"},"perplexity":{"status":200,"content_type":"text/html","bytes":212994,"sha256":"1eccefd6f169a23ad8f830d34c3b2e0a71912f233e9c6def99f42e9481a06632","final_host":"www.microsoft.com","final_path":"/ja-jp","ms":109,"identity":"PerplexityBot","html":true,"fp":"8dd4227f8c44e03e"}},"robots":{"status":200,"content_type":"text/plain","bytes":9126,"sha256":"79a2a24e3e2c00deaba2fc1922b9113f5508e74133911eadea4ebb00b5de9768","final_host":"www.microsoft.com","final_path":"/robots.txt","ms":62,"html":false,"text":"# Robots.txt file for www.microsoft.com v2.11\r\n\r\nUser-agent: *\r\nDisallow: /*?*ep_*\r\nDisallow: /*&ep_*\r\nDisallow: /store/XboxComMsCom3PAdsOptOutCookieSync.html\r\nDisallow: /store/XboxComMsComCartMuidSync.html\r\nDisallow: /*/store/locations/store-locations-sitemap.xml\r\nDisallow: /en-us/windows/si/matrix.html\r\nDisallow: /en-us/windows/si/matrix.html\r\nDisallow: /*/security/search-results.aspx?\r\nDisallow: /*/music/*/search/\r\nDisallow: /*/store/buynow?\r\nDisallow: /*/store/cart\r\nDisallow: /store/buy/cartcount\r\nDisallow: /*/search/\r\nDisallow: /*/music/*/Search/\r\nDisallow: /*/Search/\r\nDisallow: /*/newsearch/\r\nDisallow: *action=catalogsearch&\r\nDisallow: /*/store/d/groove-music-pass/cfq7ttc0k5dq/0001\r\nDisallow: /*/store/best-rated/*?*&*&*\r\nDisallow: /*/store/best-selling/*?*&*&*\r\nDisallow: /*/store/coming-soon/*?*&*&*\r\nDisallow: /*/store/deals/*?*&*&*\r\nDisallow: /*/store/game-demos/*?*&*&*\r\nDisallow: /*/store/game-previews/*?*&*&*\r\nDisallow: /*/store/most-played/*?*&*&*\r\nDisallow: /*/store/most-popular/*?*&*&*\r\nDisallow: /*/store/new/*?*&*&*\r\nDisallow: /*/store/new-and-rising/*?*&*&*\r\nDisallow: /*/store/top-free/*?*&*&*\r\nDisallow: /*/store/top-paid/*?*&*&*\r\nDisallow: /*/store/collections/helpmechoosepc?*&*&*\r\nDisallow: /*/store/collections/helpmechoosepc/?*&*&*\r\nDisallow: /*/store/movies-and-tv/*?*&*&*\r\nDisallow: /cascadeauth/microsoft-365/account/silentauth\r\nDisallow: /*/microsoft-365/site-search\r\nDisallow: /news/search\r\nDisallow: /video/search\r\nDisallow: */qa-folder\r\nAllow: /*/store/*/search/\r\nAllow: /*/store/*/layout/\r\nAllow: /*/store/music/groove-music-pass/*\r\nAllow: *action=catalogsearch&catalog_mode=grid&page=2$\r\nAllow: *action=catalogsearch&catalog_mode=grid&page=3$\r\nAllow: *action=catalogsearch&catalog_mode=grid&page=4$\r\nAllow: *action=catalogsearch&catalog_mode=grid&page=5$\r\nAllow: *action=catalogsearch&catalog_mode=grid&page=6$\r\nAllow: *action=catalogsearch&catalog_mode=grid&page=7$\r\nAllow: *action=catalogsearch&catalog_mode=grid&page=8$\r\nAllow: *action=catalogsearch&catalog_mode=list&page=2$\r\nAllow: *action=catalogsearch&catalog_mode=list&page=3$\r\nAllow: *action=catalogsearch&catalog_mode=list&page=4$\r\nAllow: *action=catalogsearch&catalog_mode=list&page=5$\r\nAllow: *action=catalogsearch&catalog_mode=list&page=6$\r\nAllow: *action=catalogsearch&catalog_mode=list&page=7$\r\nAllow: *action=catalogsearch&catalog_mode=list&page=8$\r\nDisallow: *action=accessorysearch&product=*&*\r\nAllow: *action=accessorysearch&product=*$\r\nDisallow: *action=accessorysearch&\r\nAllow: *action=accessorysearch&page=2$\r\nAllow: *action=accessorysearch&page=3$\r\nAllow: *action=accessorysearch&page=4$\r\nAllow: *action=accessorysearch&page=5$\r\nAllow: *action=accessorysearch&page=6$\r\nAllow: *action=accessorysearch&page=7$\r\nAllow: *action=accessorysearch&page=8$\r\nDisallow: *action=productcompareaction&\r\nDisallow: *action=productLinkAction&\r\nDisallow: *action=overlay&\r\nDisallow: *action=quickSearch&\r\nDisallow: *action=writeReview\r\nDisallow: *rep=hc\r\nDisallow: *fe=true\r\nDisallow: *?intc=\r\nDisallow: *&solved=\r\nDisallow: /music/*/wal/\r\nDisallow: */wal/\r\nDisallow: */layout/\r\nDisallow: */base-en/\r\nDisallow: *action=siteSearch\r\nDisallow: *action=productSupportSearch\r\nDisallow: *=imgmanager\r\nDisallow: */unsubscribe/\r\nDisallow: *ActivityUID=\r\nDisallow: /feeds/TechNet/fr-fr/screenshot/screenshot%20surface.jpg\r\nDisallow: /imaginecup/*\r\nDisallow: /*/download/confirmation.aspx?\r\nDisallow: /*HpOptOut=true$\r\nDisallow: /*TOCLinksForCrawlers*\r\nDisallow: /*mac/help.mspx?\r\nDisallow: /*mactopia/help.mspx?\r\nDisallow: */mediaapi/\r\nDisallow: /blacklisted*\r\nDisallow: /canada/Library/mnp/2/aspx/\r\nDisallow: /communities/bin.aspx?\r\nDisallow: /communities/blogs/PortalResults.mspx?\r\nDisallow: /communities/eventdetails.mspx?\r\nDisallow: /communities/rss.aspx*\r\nDisallow: /*/download/confirmation.aspx?\r\nDisallow: /*/download/registration-suggested.aspx?\r\nDisallow: /*/download/results.aspx?\r\nDisallow: /*/download/Browse.aspx?\r\nDisallow: /*/download/browse.aspx?\r\nDisallow: /*/download/info.aspx?\r\nDisallow: /*/download/thankyou.aspx\r\nDisallow: /*/download/thankyou.aspx?\r\nDisallow: /france/formation/centres/planning.asp?\r\nDisallow: /france/ie/default.asp?\r\nDisallow: /france/mnp_utility.mspx?\r\nDisallow: /genuine/\r\nDisallow: /Germany/kleinunternehmen/euga/detail.mspx?\r\nDisallow: /Germany/kleinunternehmen/euga/results.mspx?\r\nDisallow: /germany/library/images/mnp/\r\nDisallow: /germany/video/de/de/related*\r\nDisallow: /hpc/*/supported-applications.aspx?\r\nDisallow: /ie/ie40/\r\nDisallow: /info/customerror.htm*\r\nDisallow: /info/smart404.asp*\r\nDisallow: /intlkb/\r\nDisallow: /isapi/\r\nDisallow: /Japan/DirectX/default.asp?\r\nDisallow: /japan/directx/default.asp?\r\nDisallow: /japan/enable/textview.asp?\r\nDisallow: /japan/products/library/search.asp?\r\nDisallow: /japan/showcase/print/default.aspx?\r\nDisallow: /japan/terminology/query.asp?\r\nDisallow: /*mnp_utility.mspx?\r\nDisallow: /rus/licensing/Unilateral.aspx/*\r\nDisallow: /spain/empresas/\r\nDisallow: /spain/medianaempresa/\r\nDisallow: /windows/compatibility/windows-vista/\r\nDisallow: /windows/compatibility/windows-7/\r\nDisallow: /windows/compatibility/windows-7/*\r\nDisallow: /windows/404.aspx?*\r\nDisallow: /windows/campaign/meet-start.aspx\r\nDisallow: /windows/campaign/meet-apps.aspx\r\nDisallow: /windows/campaign/features-built-in-apps.aspx\r\nDisallow: /ru-ru/events/platforma/materials/default.aspx?speaker*\r\nDisallow: /de-de/corporate/rechtliche-hinweise/impressum_de.aspx\r\nDisallow: /en/microsoft-365\r\nDisallow: /en/microsoft-365/\r\nAllow: /en/microsoft-365/word\r\nAllow: /en/microsoft-365/outlook/web-email-login-for-outlook\r\nAllow: /en/microsoft-365/free-office-online-for-the-web\r\nAllow: /en/microsoft-365/planner/microsoft-planner\r\nAllow: /en/microsoft-365/yammer/yammer-overview\r\nAllow: /en/microsoft-365/publisher\r\nAllow: /en/microsoft-365/products-apps-services\r\nAllow: /en/microsoft-365/onenote/digital-note-taking-app\r\nAllow: /en/microsoft-365/powerpoint\r\nAllow: /en/microsoft-365/excel\r\nAllow: /en/microsoft-365/microsoft-office\r\nAllow: /en/microsoft-365/business/microsoft-365-administration\r\nAllow: /en/microsoft-365-copilot/microsoft-copilot-studio\r\nAllow: /en/microsoft-365/project/project-management-software\r\nAllow: /en/microsoft-365/project/microsoft-project-enterprise-plans-and-pricing\r\nAllow: /en/microsoft-365/onedrive/online-cloud-storage\r\nAllow: /en/microsoft-365/business\r\nAllow: /en/microsoft-365/clipchamp\r\nAllow: /en/microsoft-viva/engage\r\nDisallow: /en/microsoft-365/\r\nDisallow: /es/microsoft-365\r\nDisallow: /es/microsoft-365/\r\nDisallow: /fr/microsoft-365\r\nDisallow: /fr/microsoft-365/\r\nDisallow: /en/security\r\nDisallow: /en/security/\r\nDisallow: /es/security\r\nDisallow: /es/security/\r\nDisallow: /fr/security\r\nDisallow: /fr/security/\r\nDisallow: /en/microsoft-teams\r\nDisallow: /en/microsoft-teams/\r\nDisallow: /es/microsoft-teams\r\nDisallow: /es/microsoft-teams/\r\nDisallow: /fr/microsoft-teams\r\nDisallow: /fr/microsoft-teams/\r\nDisallow: /en/microsoft-viva\r\nDisallow: /en/microsoft-viva/\r\nDisallow: /es/microsoft-viva\r\nDisallow: /es/microsoft-viva/\r\nDisallow: /fr/microsoft-viva\r\nDisallow: /fr/microsoft-viva/\r\nDisallow: */isv/*?\r\nDisallow: /wp-admin/post.php \r\nAllow: /wp-admin/admin-ajax.php \r\nAllow: /wp-content/uploads/ \r\nDisallow: /blog/feed/ \r\nDisallow: /wp-login.php \r\nDisallow: /xmlrpc.php \r\nDisallow: /*?p= \r\nDisallow: /*?*p=* \r\nDisallow: /cdn-cgi/* \r\nDisallow: /page/*/?s= \r\nDisallow: /login/ \r\nDisallow: /*/embed/ \r\nDisallow: /cgi-bin/ \r\nAllow: /?display=wide \r\nDisallow: /readme.html \r\nDisallow: /*noItems= \r\nDisallow: /?s=* \r\nDisallow: /?sort-by=*\r\n\r\nSitemap: https://www.microsoft.com/store/sitemap-index.xml\r\nSitemap: https://www.microsoft.com/en-us/surface/surface.index.xml\r\nSitemap: https://www.microsoft.com/en-us/windows/windows-index.xml\r\nSitemap: https://www.microsoft.com/sitemaps/ai/sitemapindex.xml\r\nSitemap: https://www.microsoft.com/sitemaps/dynamics-365/sitemapindex.xml\r\nSitemap: https://www.microsoft.com/sitemaps/isv/sitemapindex.xml\r\nSitemap: https://www.microsoft.com/sitemaps/microsoft-365/sitemapindex.xml\r\nSitemap: https://www.microsoft.com/sitemaps/microsoft-copilot/sitemapindex.xml\r\nSitemap: https://www.microsoft.com/sitemaps/microsoft-fabric/sitemapindex.xml\r\nSitemap: https://www.microsoft.com/sitemaps/microsoft-teams/sitemapindex.xml\r\nSitemap: https://www.microsoft.com/sitemaps/power-platform/sitemapindex.xml\r\nSitemap: https://www.microsoft.com/sitemaps/microsoft-viva/sitemapindex.xml\r\nSitemap: https://www.microsoft.com/sitemaps/security/sitemapindex.xml\r\nSitemap: https://www.microsoft.com/sitemaps/trust-center/sitemapindex.xml\r\nSitemap: https://www.microsoft.com/sitemaps/windows-365/sitemapindex.xml\r\nSitemap: https://www.microsoft.com/sitemaps/customers/sitemapindex.xml\r\nSitemap: https://www.microsoft.com/sitemaps/education/sitemapindex.xml\r\nSitemap: https://www.microsoft.com/sitemaps/industry/sitemapindex.xml\r\nSitemap: https://www.microsoft.com/sitemaps/microsoft-cloud/sitemapindex.xml\r\nSitemap: https://www.microsoft.com/sitemaps/sustainability/sitemapindex.xml\r\nSitemap: https://www.microsoft.com/en-us/microsoft-365-life-hacks/sitemap.xml\r\nSitemap: https://www.microsoft.com/windows/business/sitemap.xml","agents":{"GPTBot":true,"OAI-SearchBot":true,"ChatGPT-User":true,"ClaudeBot":true,"Claude-SearchBot":true,"PerplexityBot":true,"Google-Extended":true,"Applebot-Extended":true,"CCBot":true,"Bytespider":true,"Meta-ExternalAgent":true,"Amazonbot":true}},"sitemap":{"status":200,"content_type":"application/xml","bytes":3002,"sha256":"0d68b60fc036a3c907e6e8af9908237713f8d5dfa2f0deb670edd923f9747b4b","final_host":"www.microsoft.com","final_path":"/store/sitemap-index.xml","ms":65,"url":"https://www.microsoft.com/store/sitemap-index.xml","declared":true,"xml":true},"files":{"llms_txt":{"status":404,"content_type":"text/html","bytes":76859,"sha256":"6769df293e90f9bf5e31c25958f1ea6c3b7715aa49f3176afe7694e324e0e6b1","final_host":"www.microsoft.com","final_path":"/llms.txt","ms":402,"path":"/llms.txt","html":true,"json":null,"present":false},"agents_md":{"status":404,"content_type":"text/html","bytes":76859,"sha256":"6769df293e90f9bf5e31c25958f1ea6c3b7715aa49f3176afe7694e324e0e6b1","final_host":"www.microsoft.com","final_path":"/agents.md","ms":675,"path":"/agents.md","html":true,"json":null,"present":false},"agent_skills":{"status":404,"content_type":"text/html","bytes":76859,"sha256":"6769df293e90f9bf5e31c25958f1ea6c3b7715aa49f3176afe7694e324e0e6b1","final_host":"www.microsoft.com","final_path":"/.well-known/agent-skills/index.json","ms":687,"path":"/.well-known/agent-skills/index.json","html":true,"json":null,"present":false},"mcp_card":{"status":404,"content_type":"text/html","bytes":76859,"sha256":"6769df293e90f9bf5e31c25958f1ea6c3b7715aa49f3176afe7694e324e0e6b1","final_host":"www.microsoft.com","final_path":"/.well-known/mcp/server-card.json","ms":709,"path":"/.well-known/mcp/server-card.json","html":true,"json":null,"present":false},"api_catalog":{"status":404,"content_type":"text/html","bytes":76859,"sha256":"6769df293e90f9bf5e31c25958f1ea6c3b7715aa49f3176afe7694e324e0e6b1","final_host":"www.microsoft.com","final_path":"/.well-known/api-catalog","ms":191,"path":"/.well-known/api-catalog","html":true,"json":null,"present":false},"a2a_card":{"status":404,"content_type":"text/html","bytes":76859,"sha256":"6769df293e90f9bf5e31c25958f1ea6c3b7715aa49f3176afe7694e324e0e6b1","final_host":"www.microsoft.com","final_path":"/.well-known/agent-card.json","ms":622,"path":"/.well-known/agent-card.json","html":true,"json":null,"present":false}},"rings":{"v":1,"candidates":[],"mcp":[],"a2a":null,"webmcp":{"model_context":false,"bridge":false,"mcp_json":false,"link_rel":false,"any":false},"handshakes":0,"get_path":{"status":200,"content_type":"text/html","ms":665,"error":null},"get_host":{"status":200,"content_type":"text/html","ms":255,"error":null}},"sf":[4,0,0,0,0,0,1,0,0],"derived":{"reachable":true,"home_class":"answers","robots_served":true,"robots_sitemap":true,"content_signal":false,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":[],"blocks_any_ai":false,"files":{"llms_txt":false,"agents_md":false,"agent_skills":false,"mcp_card":false,"api_catalog":false,"a2a_card":false},"any_capability":false,"ai_refused":[],"ai_thin":[],"aipref":0,"aipref_measured":true,"sf":[4,0,0,0,0,0,1,0,0],"rings":{"mcp_declared":false,"mcp_live":false,"mcp_auth":false,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":3,"fh":"www.microsoft.com","fp_other_host":[],"fp":"8dd4227f8c44e03e","fp_bits":{"gptbot":0,"claudebot":0,"oaisearch":0,"perplexity":0}},"derive_v":5,"frame":"top10k","shard":7},{"v":1,"kind":"crawlcheck-census-record","run":18,"domain":"youtube.com","rank":9,"measured_at":"2026-11-13T18:46:27.088Z","home":{"browser":{"status":200,"content_type":"text/html","bytes":879419,"sha256":"5085746e4161c135eaa32913688ba9f88e81a806596e044e8a09e42e79110b01","final_host":"www.youtube.com","final_path":"/","ms":635,"identity":"Desktop browser (device re-check)","html":true,"fp":"6d84f448d2eab9ee"},"gptbot":{"status":200,"content_type":"text/html","bytes":877904,"sha256":"f4a13b84d3d279f3b07775bdb6f3058d39f1d4ff03a61cce5a8f1c208e0c47ad","final_host":"www.youtube.com","final_path":"/","ms":978,"identity":"GPTBot","html":true,"fp":"6d84f448d2eab9ee"},"claudebot":{"status":200,"content_type":"text/html","bytes":981964,"sha256":"6a2ec46f5b46c0a4d11913ced52e61f92e2845e957765fe9934ac87a126b0fd8","final_host":"www.youtube.com","final_path":"/","ms":785,"identity":"ClaudeBot","html":true,"fp":"6d84f448d2eab9ee"},"oaisearch":{"status":200,"content_type":"text/html","bytes":977463,"sha256":"1392ecbc329e64ea10d60eb95e28d2df2fb4458135d4e9bba9ed6934f5090454","final_host":"www.youtube.com","final_path":"/","ms":960,"identity":"OAI-SearchBot","html":true,"fp":"6d84f448d2eab9ee"},"perplexity":{"status":200,"content_type":"text/html","bytes":981193,"sha256":"297f3692834a7939136d886f881754b0b3ad17e8664735c5363785272bd71166","final_host":"www.youtube.com","final_path":"/","ms":832,"identity":"PerplexityBot","html":true,"fp":"6d84f448d2eab9ee"},"browser_mobile":{"status":200,"content_type":"text/html","bytes":357736,"sha256":"4016e584dd5a9605a00d42325f0ab9446e71563ea507c47f3b131678aaef5a5f","final_host":"m.youtube.com","final_path":"/","ms":846,"identity":"Mobile browser","html":true}},"robots":{"status":200,"content_type":"text/plain","bytes":792,"sha256":"96f8640ca357626e7d9a894057fef8b5746eb838c29ed68fc176af58c9348c6f","final_host":"www.youtube.com","final_path":"/robots.txt","ms":760,"html":false,"text":"# robots.txt file for YouTube\n# Created in the distant future (the year 2000) after\n# the robotic uprising of the mid 90's which wiped out all humans.\n\nUser-agent: Mediapartners-Google*\nDisallow:\n\nUser-agent: *\nDisallow: /api/\nDisallow: /channel_picker\nDisallow: /comment\nDisallow: /feeds/videos.xml\nDisallow: /file_download\nDisallow: /get_video\nDisallow: /get_video_info\nDisallow: /get_midroll_info\nDisallow: /live_chat\nDisallow: /login\nDisallow: /qr\nDisallow: /results\nDisallow: /signup\nDisallow: /t/terms\nDisallow: /timedtext_video\nDisallow: /verify_age\nDisallow: /watch_ajax\nDisallow: /watch_fragments_ajax\nDisallow: /watch_popup\nDisallow: /watch_queue_ajax\nDisallow: /youtubei/\n\nSitemap: https://www.youtube.com/sitemaps/sitemap.xml\nSitemap: https://www.youtube.com/product/sitemap.xml\n\n","agents":{"GPTBot":true,"OAI-SearchBot":true,"ChatGPT-User":true,"ClaudeBot":true,"Claude-SearchBot":true,"PerplexityBot":true,"Google-Extended":true,"Applebot-Extended":true,"CCBot":true,"Bytespider":true,"Meta-ExternalAgent":true,"Amazonbot":true}},"sitemap":{"status":200,"content_type":"text/xml","bytes":2045,"sha256":"89bea4222871c7b633fc750e5f1b2fd503ab80ecb9c6cf7aae7de3cf3bbddfbb","final_host":"www.youtube.com","final_path":"/sitemaps/sitemap.xml","ms":81,"url":"https://www.youtube.com/sitemaps/sitemap.xml","declared":true,"xml":true},"files":{"llms_txt":{"status":404,"content_type":"text/html","bytes":758,"sha256":"f18e671ee4cd0c1647305a75f4e927d2879df44b15de44735b892a3d04f53b4e","final_host":"www.youtube.com","final_path":"/llms.txt","ms":646,"path":"/llms.txt","html":true,"json":null,"present":false},"agents_md":{"status":404,"content_type":"text/html","bytes":758,"sha256":"19ddc62a20a9fcf393e5fdbeb8139f6ae7235c99b9953a93a4e6c3bf18538a2f","final_host":"www.youtube.com","final_path":"/agents.md","ms":647,"path":"/agents.md","html":true,"json":null,"present":false},"agent_skills":{"status":404,"content_type":"text/html","bytes":758,"sha256":"0bb44c7f24e7e3bfa01ad0b6001bd135fb37f1f90d6555be755e954e4a7a9eed","final_host":"www.youtube.com","final_path":"/.well-known/agent-skills/index.json","ms":660,"path":"/.well-known/agent-skills/index.json","html":true,"json":null,"present":false},"mcp_card":{"status":404,"content_type":"text/html","bytes":758,"sha256":"b98d5536e913cddbb38977c00c6e9ab434ef72281658f4ba15034339ba372994","final_host":"www.youtube.com","final_path":"/.well-known/mcp/server-card.json","ms":428,"path":"/.well-known/mcp/server-card.json","html":true,"json":null,"present":false},"api_catalog":{"status":404,"content_type":"text/html","bytes":758,"sha256":"0aa8f8dffac49d1d5ad8f540a8a61c2c74f0c9db6ef662acf1e8780636696011","final_host":"www.youtube.com","final_path":"/.well-known/api-catalog","ms":429,"path":"/.well-known/api-catalog","html":true,"json":null,"present":false},"a2a_card":{"status":404,"content_type":"text/html","bytes":758,"sha256":"4d0086d7a6cedb66345a727bc18e69fc624a5fec551dc1a6fd294c04f45da21a","final_host":"www.youtube.com","final_path":"/.well-known/agent-card.json","ms":446,"path":"/.well-known/agent-card.json","html":true,"json":null,"present":false}},"device_variant":true,"rings":{"v":1,"candidates":[],"mcp":[],"a2a":null,"webmcp":{"model_context":false,"bridge":false,"mcp_json":false,"link_rel":false,"any":false},"handshakes":0,"get_path":{"status":404,"content_type":"text/html","ms":427,"error":null},"get_host":{"status":530,"content_type":"text/plain","ms":270,"error":null}},"sf":[4,0,0,0,0,0,1,0,1],"derived":{"reachable":true,"home_class":"answers","robots_served":true,"robots_sitemap":true,"content_signal":false,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":[],"blocks_any_ai":false,"files":{"llms_txt":false,"agents_md":false,"agent_skills":false,"mcp_card":false,"api_catalog":false,"a2a_card":false},"any_capability":false,"ai_refused":[],"ai_thin":[],"aipref":0,"aipref_measured":true,"sf":[4,0,0,0,0,0,1,0,1],"rings":{"mcp_declared":false,"mcp_live":false,"mcp_auth":false,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":3,"fh":"www.youtube.com","fp_other_host":[],"fp":"6d84f448d2eab9ee","fp_bits":{"gptbot":0,"claudebot":0,"oaisearch":0,"perplexity":0}},"derive_v":5,"frame":"top10k","shard":0},{"v":1,"kind":"crawlcheck-census-record","run":18,"domain":"apple.com","rank":10,"measured_at":"2026-11-13T18:48:18.168Z","home":{"browser":{"status":200,"content_type":"text/html","bytes":260629,"sha256":"508dac2d38d67d3c0ede0ef4e4a50511545f60ea60658eb13e4e61db31cb21ec","final_host":"www.apple.com","final_path":"/","ms":116,"identity":"Mobile browser","html":true,"fp":"55de92c01fc26d5e"},"gptbot":{"status":200,"content_type":"text/html","bytes":260629,"sha256":"508dac2d38d67d3c0ede0ef4e4a50511545f60ea60658eb13e4e61db31cb21ec","final_host":"www.apple.com","final_path":"/","ms":75,"identity":"GPTBot","html":true,"fp":"55de92c01fc26d5e"},"claudebot":{"status":200,"content_type":"text/html","bytes":260629,"sha256":"508dac2d38d67d3c0ede0ef4e4a50511545f60ea60658eb13e4e61db31cb21ec","final_host":"www.apple.com","final_path":"/","ms":41,"identity":"ClaudeBot","html":true,"fp":"55de92c01fc26d5e"},"oaisearch":{"status":200,"content_type":"text/html","bytes":260629,"sha256":"508dac2d38d67d3c0ede0ef4e4a50511545f60ea60658eb13e4e61db31cb21ec","final_host":"www.apple.com","final_path":"/","ms":129,"identity":"OAI-SearchBot","html":true,"fp":"55de92c01fc26d5e"},"perplexity":{"status":200,"content_type":"text/html","bytes":260629,"sha256":"508dac2d38d67d3c0ede0ef4e4a50511545f60ea60658eb13e4e61db31cb21ec","final_host":"www.apple.com","final_path":"/","ms":132,"identity":"PerplexityBot","html":true,"fp":"55de92c01fc26d5e"}},"robots":{"status":200,"content_type":"text/plain","bytes":1018,"sha256":"3115f45c98cfa19341978dcab5dfbc0fd3e0ff4eb3ab1fe8192c3fbf7227dee7","final_host":"www.apple.com","final_path":"/robots.txt","ms":14,"html":false,"text":"# robots.txt for https://www.apple.com/\n\nUser-agent: *\nDisallow: /*shop/browse/overlay/*\nDisallow: /*shop/iphone/payments/overlay/*\nDisallow: /cn/*/aow/*\nDisallow: /tmall*\nAllow: /ac/globalnav/2.0/*/images/ac-globalnav/globalnav/search/* \t\n\nUser-agent: Baiduspider\nDisallow: /*\nAllow: /cn/*\nAllow: /cn-edu/*\nAllow: /cn-k12/*\n\nUser-agent: HaoSouSpider\nDisallow: /*\nAllow: /cn/*\nAllow: /cn-edu/*\nAllow: /cn-k12/*\n\nUser-agent: Sogou web spider\nDisallow: /*\nAllow: /cn/*\nAllow: /cn-k12/*\n\nUser-agent: Sogou inst spider\nDisallow: /*\nAllow: /cn/*\nAllow: /cn-k12/*\n\nUser-agent: Sogou spider2\nDisallow: /*\nAllow: /cn/*\nAllow: /cn-k12/*\n\n#DaumWebMasterTool:6ff40e0f09564a9d4e3bc80e7cbde7c196677a2f5f610eb9c029836774f9a8e9:D9a8IEFAzmC/NWsQ3tB0Wg==\n\nSitemap: https://www.apple.com/shop/sitemap.xml\nSitemap: https://www.apple.com/autopush/sitemap/sitemap-index.xml\nSitemap: https://www.apple.com/newsroom/sitemap.xml\nSitemap: https://www.apple.com/retail/sitemap/sitemap.xml\nSitemap: https://www.apple.com/today/sitemap-index.xml\n","agents":{"GPTBot":true,"OAI-SearchBot":true,"ChatGPT-User":true,"ClaudeBot":true,"Claude-SearchBot":true,"PerplexityBot":true,"Google-Extended":true,"Applebot-Extended":true,"CCBot":true,"Bytespider":true,"Meta-ExternalAgent":true,"Amazonbot":true}},"sitemap":{"status":200,"content_type":"text/xml","bytes":3776,"sha256":"6143254373f03e6b21fad78999cd596f2e91f01925f97b588d1e337c36a558e5","final_host":"www.apple.com","final_path":"/shop/sitemap.xml","ms":216,"url":"https://www.apple.com/shop/sitemap.xml","declared":true,"xml":true},"files":{"llms_txt":{"status":404,"content_type":"text/html","bytes":109401,"sha256":"8653dc6569ae0b1a61e1c2d8dca414f9d12fd03c176d2fdc7d8634a03c0b3cd4","final_host":"www.apple.com","final_path":"/llms.txt","ms":301,"path":"/llms.txt","html":true,"json":null,"present":false},"agents_md":{"status":404,"content_type":"text/html","bytes":109401,"sha256":"8653dc6569ae0b1a61e1c2d8dca414f9d12fd03c176d2fdc7d8634a03c0b3cd4","final_host":"www.apple.com","final_path":"/agents.md","ms":293,"path":"/agents.md","html":true,"json":null,"present":false},"agent_skills":{"status":404,"content_type":"text/html","bytes":196,"sha256":"80c3fe2ae1062abf56456f52518bd670f9ec3917b7f85e152b347ac6b6faf880","final_host":"www.apple.com","final_path":"/.well-known/agent-skills/index.json","ms":233,"path":"/.well-known/agent-skills/index.json","html":true,"json":null,"present":false},"mcp_card":{"status":404,"content_type":"text/html","bytes":196,"sha256":"80c3fe2ae1062abf56456f52518bd670f9ec3917b7f85e152b347ac6b6faf880","final_host":"www.apple.com","final_path":"/.well-known/mcp/server-card.json","ms":613,"path":"/.well-known/mcp/server-card.json","html":true,"json":null,"present":false},"api_catalog":{"status":404,"content_type":"text/html","bytes":196,"sha256":"80c3fe2ae1062abf56456f52518bd670f9ec3917b7f85e152b347ac6b6faf880","final_host":"www.apple.com","final_path":"/.well-known/api-catalog","ms":248,"path":"/.well-known/api-catalog","html":true,"json":null,"present":false},"a2a_card":{"status":404,"content_type":"text/html","bytes":196,"sha256":"80c3fe2ae1062abf56456f52518bd670f9ec3917b7f85e152b347ac6b6faf880","final_host":"www.apple.com","final_path":"/.well-known/agent-card.json","ms":224,"path":"/.well-known/agent-card.json","html":true,"json":null,"present":false}},"rings":{"v":1,"candidates":[],"mcp":[],"a2a":null,"webmcp":{"model_context":false,"bridge":false,"mcp_json":false,"link_rel":false,"any":false},"handshakes":0,"get_path":{"status":404,"content_type":"text/html","ms":1130,"error":null},"get_host":{"status":530,"content_type":"text/plain","ms":6,"error":null}},"sf":[4,0,0,0,0,0,0,0,0],"derived":{"reachable":true,"home_class":"answers","robots_served":true,"robots_sitemap":true,"content_signal":false,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":[],"blocks_any_ai":false,"files":{"llms_txt":false,"agents_md":false,"agent_skills":false,"mcp_card":false,"api_catalog":false,"a2a_card":false},"any_capability":false,"ai_refused":[],"ai_thin":[],"aipref":0,"aipref_measured":true,"sf":[4,0,0,0,0,0,0,0,0],"rings":{"mcp_declared":false,"mcp_live":false,"mcp_auth":false,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":3,"fh":"www.apple.com","fp_other_host":[],"fp":"55de92c01fc26d5e","fp_bits":{"gptbot":0,"claudebot":0,"oaisearch":0,"perplexity":0}},"derive_v":5,"frame":"top10k","shard":1},{"v":1,"kind":"crawlcheck-census-record","run":18,"domain":"instagram.com","rank":11,"measured_at":"2026-11-13T18:49:51.543Z","home":{"browser":{"status":200,"content_type":"text/html","bytes":527281,"sha256":"152cd608a1d3c34583a79fa2605f8b7745d3b3d5df9091601fa9d156731c3aa7","final_host":"www.instagram.com","final_path":"/","ms":438,"identity":"Mobile browser","html":true},"gptbot":{"status":200,"content_type":"text/html","bytes":550853,"sha256":"ec5a4745ff6eaa991078fb13ac6574d5fb54255050744866dbceb76f48fcd63f","final_host":"www.instagram.com","final_path":"/","ms":526,"identity":"GPTBot","html":true},"claudebot":{"status":200,"content_type":"text/html","bytes":550726,"sha256":"64b74f7b41de8e4935aa6aff77f66f06b7d961273a63cca8efdddf68a0e76d03","final_host":"www.instagram.com","final_path":"/","ms":453,"identity":"ClaudeBot","html":true},"oaisearch":{"status":200,"content_type":"text/html","bytes":550833,"sha256":"3de597aadb887cc7609e1b017461ebb06ba7bc7ff65e00944e920ef1e50541d9","final_host":"www.instagram.com","final_path":"/","ms":496,"identity":"OAI-SearchBot","html":true},"perplexity":{"status":200,"content_type":"text/html","bytes":440461,"sha256":"4147c82cd2cb920ad24056cf774cc3a9c5443eef858543a0ae68678b1819afec","final_host":"www.instagram.com","final_path":"/","ms":543,"identity":"PerplexityBot","html":true,"fp":"b868896ab1101937"}},"robots":{"status":200,"content_type":"text/plain","bytes":6394,"sha256":"3a62d0ddf2d87aa7cef3d1b7facf0bc49c10f7f4580755c0230fd2f89a679da5","final_host":"www.instagram.com","final_path":"/robots.txt","ms":276,"html":false,"text":"# Notice: Collection of data on Instagram through automated means is\n# prohibited unless you have express written permission from Instagram\n# and may only be conducted for the limited purpose contained in said\n# permission.\n# All authorized user-agents listed on this page must comply with Meta’s\n# Automated Data Collection Terms available at:\n# https://www.facebook.com/legal/automated_data_collection_terms\nUser-agent: __usage_ai-training\nDisallow: /\n\nUser-agent: Amazonbot\nDisallow: /\n\nUser-agent: Applebot-Extended\nDisallow: /\n\nUser-agent: Brightbot\nDisallow: /\n\nUser-agent: ClaudeBot\nDisallow: /\n\nUser-agent: Google-Extended\nDisallow: /\n\nUser-agent: GPTBot\nDisallow: /\n\nUser-agent: PerplexityBot\nDisallow: /\n\nUser-agent: PetalBot\nDisallow: /\n\nUser-agent: Scrapy\nDisallow: /\n\nUser-agent: uptimerobot\nDisallow: /\n\nUser-agent: viberbot\nDisallow: /\n\nUser-agent: YaK\nDisallow: /\n\nUser-agent: Yandex\nDisallow: /\n\nUser-agent: Yeti\nDisallow: /\n\nUser-agent: Applebot\nDisallow: /accounts/login/*?next=\nDisallow: /ajax/\nDisallow: /client_error/\nDisallow: /direct/\nDisallow: /logging/\nDisallow: /publicapi/\nDisallow: /qp/batch_fetch_web/\nDisallow: /query/\nAllow: /places/c/\n\nUser-agent: Bingbot\nDisallow: /*/c/\nDisallow: /*/comments/\nDisallow: /*/liked_by/\nDisallow: /accounts/login/*?next=\nDisallow: /ajax/\nDisallow: /client_error/\nDisallow: /direct/\nDisallow: /logging/\nDisallow: /publicapi/\nDisallow: /qp/batch_fetch_web/\nDisallow: /query/\nAllow: /places/c/\n\nUser-agent: deepcrawl\nDisallow: /accounts/login/*?next=\nDisallow: /ajax/\nDisallow: /client_error/\nDisallow: /direct/\nDisallow: /logging/\nDisallow: /publicapi/\nDisallow: /qp/batch_fetch_web/\nDisallow: /query/\nAllow: /places/c/\n\nUser-agent: Discordbot\nDisallow: /accounts/login/*?next=\nDisallow: /ajax/\nDisallow: /client_error/\nDisallow: /direct/\nDisallow: /logging/\nDisallow: /publicapi/\nDisallow: /qp/batch_fetch_web/\nDisallow: /query/\nAllow: /places/c/\n\nUser-agent: DuckDuckBot\nDisallow: /accounts/login/*?next=\nDisallow: /ajax/\nDisallow: /client_error/\nDisallow: /direct/\nDisallow: /logging/\nDisallow: /publicapi/\nDisallow: /qp/batch_fetch_web/\nDisallow: /query/\nAllow: /places/c/\n\nUser-agent: facebookexternalhit\nDisallow: /accounts/login/*?next=\nDisallow: /ajax/\nDisallow: /client_error/\nDisallow: /direct/\nDisallow: /logging/\nDisallow: /publicapi/\nDisallow: /qp/batch_fetch_web/\nDisallow: /query/\nAllow: /places/c/\n\nUser-agent: Googlebot\nDisallow: /*/c/\nDisallow: /*/comments/\nDisallow: /*/liked_by/\nDisallow: /accounts/login/*?next=\nDisallow: /ajax/\nDisallow: /client_error/\nDisallow: /direct/\nDisallow: /logging/\nDisallow: /publicapi/\nDisallow: /qp/batch_fetch_web/\nDisallow: /query/\nAllow: /places/c/\n\nUser-agent: Googlebot-Image\nDisallow: /accounts/login/*?next=\nDisallow: /ajax/\nDisallow: /client_error/\nDisallow: /direct/\nDisallow: /logging/\nDisallow: /publicapi/\nDisallow: /qp/batch_fetch_web/\nDisallow: /query/\nAllow: /places/c/\n\nUser-agent: Googlebot-Video\nDisallow: /accounts/login/*?next=\nDisallow: /ajax/\nDisallow: /client_error/\nDisallow: /direct/\nDisallow: /logging/\nDisallow: /publicapi/\nDisallow: /qp/batch_fetch_web/\nDisallow: /query/\nAllow: /places/c/\n\nUser-agent: Googlebot-News\nDisallow: /accounts/login/*?next=\nDisallow: /ajax/\nDisallow: /client_error/\nDisallow: /direct/\nDisallow: /logging/\nDisallow: /publicapi/\nDisallow: /qp/batch_fetch_web/\nDisallow: /query/\nAllow: /places/c/\n\nUser-agent: Google-InspectionTool\nDisallow: /accounts/login/*?next=\nDisallow: /ajax/\nDisallow: /client_error/\nDisallow: /direct/\nDisallow: /logging/\nDisallow: /publicapi/\nDisallow: /qp/batch_fetch_web/\nDisallow: /query/\nAllow: /places/c/\n\nUser-agent: lumar\nDisallow: /accounts/login/*?next=\nDisallow: /ajax/\nDisallow: /client_error/\nDisallow: /direct/\nDisallow: /logging/\nDisallow: /publicapi/\nDisallow: /qp/batch_fetch_web/\nDisallow: /query/\nAllow: /places/c/\n\nUser-agent: LinkedInBot\nDisallow: /accounts/login/*?next=\nDisallow: /ajax/\nDisallow: /client_error/\nDisallow: /direct/\nDisallow: /logging/\nDisallow: /publicapi/\nDisallow: /qp/batch_fetch_web/\nDisallow: /query/\nAllow: /places/c/\n\nUser-agent: msnbot\nDisallow: /accounts/login/*?next=\nDisallow: /ajax/\nDisallow: /client_error/\nDisallow: /direct/\nDisallow: /logging/\nDisallow: /publicapi/\nDisallow: /qp/batch_fetch_web/\nDisallow: /query/\nAllow: /places/c/\n\nUser-agent: Pinterestbot\nDisallow: /accounts/login/*?next=\nDisallow: /ajax/\nDisallow: /client_error/\nDisallow: /direct/\nDisallow: /logging/\nDisallow: /publicapi/\nDisallow: /qp/batch_fetch_web/\nDisallow: /query/\nAllow: /places/c/\n\nUser-agent: Screaming Frog SEO Spider\nDisallow: /accounts/login/*?next=\nDisallow: /ajax/\nDisallow: /client_error/\nDisallow: /direct/\nDisallow: /logging/\nDisallow: /publicapi/\nDisallow: /qp/batch_fetch_web/\nDisallow: /query/\nAllow: /places/c/\n\nUser-agent: seznambot\nDisallow: /accounts/login/*?next=\nDisallow: /ajax/\nDisallow: /client_error/\nDisallow: /direct/\nDisallow: /logging/\nDisallow: /publicapi/\nDisallow: /qp/batch_fetch_web/\nDisallow: /query/\nAllow: /places/c/\n\nUser-agent: Slurp\nDisallow: /accounts/login/*?next=\nDisallow: /ajax/\nDisallow: /client_error/\nDisallow: /direct/\nDisallow: /logging/\nDisallow: /publicapi/\nDisallow: /qp/batch_fetch_web/\nDisallow: /query/\nAllow: /places/c/\n\nUser-agent: teoma\nDisallow: /accounts/login/*?next=\nDisallow: /ajax/\nDisallow: /client_error/\nDisallow: /direct/\nDisallow: /logging/\nDisallow: /publicapi/\nDisallow: /qp/batch_fetch_web/\nDisallow: /query/\nAllow: /places/c/\n\nUser-agent: TelegramBot\nDisallow: /accounts/login/*?next=\nDisallow: /ajax/\nDisallow: /client_error/\nDisallow: /direct/\nDisallow: /logging/\nDisallow: /publicapi/\nDisallow: /qp/batch_fetch_web/\nDisallow: /query/\nAllow: /places/c/\n\nUser-agent: Twitterbot\nDisallow: /accounts/login/*?next=\nDisallow: /ajax/\nDisallow: /client_error/\nDisallow: /direct/\nDisallow: /logging/\nDisallow: /publicapi/\nDisallow: /qp/batch_fetch_web/\nDisallow: /query/\nAllow: /places/c/\n\nUser-agent: *\nDisallow: /\n\nSitemap: https://www.instagram.com/sitemap/ig_places_sitemap.xml.gz\nSitemap: https://www.instagram.com/sitemap/ig_seo_profile_sitemap.xml.gz\nSitemap: https://www.instagram.com/sitemap/ig_seo_profile_sitemap_non_media_eligible.xml.gz\nSitemap: https://www.instagram.com/sitemap/dim_ig_logged_out_web_keyword_ent.xml.gz\nSitemap: https://www.instagram.com/sitemap/ig_seo_profile_sitemap_v2.xml.gz\nSitemap: https://www.instagram.com/sitemap/ig_seo_historical_reels_mvp_video_rows.xml.gz\n\n","agents":{"GPTBot":false,"OAI-SearchBot":false,"ChatGPT-User":false,"ClaudeBot":false,"Claude-SearchBot":false,"PerplexityBot":false,"Google-Extended":false,"Applebot-Extended":false,"CCBot":false,"Bytespider":false,"Meta-ExternalAgent":false,"Amazonbot":false}},"sitemap":{"status":403,"content_type":"text/html","bytes":0,"sha256":"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855","final_host":"www.instagram.com","final_path":"/sitemap/ig_places_sitemap.xml.gz","ms":176,"url":"https://www.instagram.com/sitemap/ig_places_sitemap.xml.gz","declared":true,"xml":false},"files":{"llms_txt":{"status":429,"content_type":"text/html","bytes":0,"sha256":"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855","final_host":"www.instagram.com","final_path":"/accounts/login","ms":332,"path":"/llms.txt","html":true,"json":null,"present":false},"agents_md":{"status":429,"content_type":"text/html","bytes":0,"sha256":"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855","final_host":"www.instagram.com","final_path":"/accounts/login","ms":341,"path":"/agents.md","html":true,"json":null,"present":false},"agent_skills":{"status":404,"content_type":"text/html","bytes":535063,"sha256":"8a3967a12622b031b01d0dd7e8de673cb8ef7eea0e318402bf80bdd742615642","final_host":"www.instagram.com","final_path":"/.well-known/agent-skills/index.json","ms":355,"path":"/.well-known/agent-skills/index.json","html":true,"json":null,"present":false},"mcp_card":{"status":404,"content_type":"text/html","bytes":535030,"sha256":"1157ffa99625c820b575e6bfa0f016e864b48c486ccb7700accdeacac32a031e","final_host":"www.instagram.com","final_path":"/.well-known/mcp/server-card.json","ms":256,"path":"/.well-known/mcp/server-card.json","html":true,"json":null,"present":false},"api_catalog":{"status":429,"content_type":"text/html","bytes":0,"sha256":"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855","final_host":"www.instagram.com","final_path":"/accounts/login","ms":230,"path":"/.well-known/api-catalog","html":true,"json":null,"present":false},"a2a_card":{"status":429,"content_type":"text/html","bytes":0,"sha256":"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855","final_host":"www.instagram.com","final_path":"/accounts/login","ms":300,"path":"/.well-known/agent-card.json","html":true,"json":null,"present":false}},"rings":{"v":1,"candidates":[],"mcp":[],"a2a":null,"webmcp":{"model_context":false,"bridge":false,"mcp_json":false,"link_rel":false,"any":false},"handshakes":0,"get_path":{"status":429,"content_type":"text/html","ms":230,"error":null},"get_host":{"status":530,"content_type":"text/plain","ms":38,"error":null}},"sf":[4,0,0,0,0,0,1,0,1],"derived":{"reachable":true,"home_class":"answers","robots_served":true,"robots_sitemap":true,"content_signal":false,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":["GPTBot","OAI-SearchBot","ChatGPT-User","ClaudeBot","Claude-SearchBot","PerplexityBot","Google-Extended","Applebot-Extended","CCBot","Bytespider","Meta-ExternalAgent","Amazonbot"],"blocks_any_ai":true,"files":{"llms_txt":false,"agents_md":false,"agent_skills":false,"mcp_card":false,"api_catalog":false,"a2a_card":false},"any_capability":false,"ai_refused":[],"ai_thin":[],"aipref":2,"aipref_measured":true,"sf":[4,0,0,0,0,0,1,0,1],"rings":{"mcp_declared":false,"mcp_live":false,"mcp_auth":false,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":3,"fh":"www.instagram.com","fp_other_host":[]},"derive_v":5,"frame":"top10k","shard":2},{"v":1,"kind":"crawlcheck-census-record","run":18,"domain":"mail.ru","rank":12,"measured_at":"2026-11-13T18:50:46.493Z","home":{"browser":{"status":200,"content_type":"text/html","bytes":425835,"sha256":"d56016e50b49538a8f0c5533150efc737e211dfe384bbc073c1f1d425e5ede74","final_host":"mail.ru","final_path":"/","ms":2245,"identity":"Mobile browser","html":true,"fp":"9089414431008018"},"gptbot":{"status":200,"content_type":"text/html","bytes":936,"sha256":"c9f42ca1bcfd9825fd7aa6a0c278b46f69297e5b36c66b47f5d5263684f64aa4","final_host":"account.mail.ru","final_path":"/login","ms":4677,"identity":"GPTBot","html":true},"claudebot":{"status":200,"content_type":"text/html","bytes":936,"sha256":"daca385569c331c961e478277f8565e2d2e1dcd4a655223f6a353c88b6a14ae1","final_host":"account.mail.ru","final_path":"/login","ms":4621,"identity":"ClaudeBot","html":true},"oaisearch":{"status":200,"content_type":"text/html","bytes":936,"sha256":"8cc2ae19a2ec14b03afdac71e3c84198f8645f49fa0a7f46f1e657ad546260e5","final_host":"account.mail.ru","final_path":"/login","ms":4657,"identity":"OAI-SearchBot","html":true},"perplexity":{"status":200,"content_type":"text/html","bytes":936,"sha256":"8207319e5ccfa2df22d040d6955f4207f88cb86c9ec9963b449ae0cce3fbe264","final_host":"account.mail.ru","final_path":"/login","ms":4620,"identity":"PerplexityBot","html":true}},"robots":{"status":200,"content_type":"text/plain","bytes":433,"sha256":"d3fd6768e203ab4008918459ce8d1ccc9e31d703cc01c9765902bcfc894822bf","final_host":"mail.ru","final_path":"/robots.txt","ms":1613,"html":false,"text":"User-agent: *\nAllow: /*.css\nAllow: /*.jpg\nAllow: /*.gif\nAllow: /*.png\nAllow: /v/j/*.js\nAllow: /$\nDisallow: /\n\nUser-agent: Googlebot\nAllow: /*.css\nAllow: /*.jpg\nAllow: /*.gif\nAllow: /*.png\nAllow: /v/j/*.js\nDisallow: *?\nDisallow: *search\nDisallow: *auth\nDisallow: *api\n\nUser-agent: Yandex\nAllow: /*.css\nAllow: /*.jpg\nAllow: /*.gif\nAllow: /*.png\nAllow: /v/j/*.js\nAllow: /$\nDisallow: /\n\nUser-agent: Twitterbot\nDisallow: /\nAllow: /?logo=\n","agents":{"GPTBot":true,"OAI-SearchBot":true,"ChatGPT-User":true,"ClaudeBot":true,"Claude-SearchBot":true,"PerplexityBot":true,"Google-Extended":true,"Applebot-Extended":true,"CCBot":true,"Bytespider":true,"Meta-ExternalAgent":true,"Amazonbot":true}},"sitemap":{"status":200,"content_type":"text/html","bytes":936,"sha256":"2e612de71e3ae10e147878007a1c0f11541d94c4ca3b475da8a648e5484013d0","final_host":"account.mail.ru","final_path":"/login","ms":1541,"url":"https://mail.ru/sitemap.xml","declared":false,"xml":false},"files":{"llms_txt":{"status":200,"content_type":"text/html","bytes":936,"sha256":"25f9325d76ab56af787c95d723bbfb40de45ceb82f9daf1f46281ae4fcd6becb","final_host":"account.mail.ru","final_path":"/login","ms":1819,"path":"/llms.txt","html":true,"json":null,"present":false},"agents_md":{"status":200,"content_type":"text/html","bytes":936,"sha256":"0da8f79c3fda38ca533074929663e5137a29c2076d053a2ae54e7db891872eab","final_host":"account.mail.ru","final_path":"/login","ms":1738,"path":"/agents.md","html":true,"json":null,"present":false},"agent_skills":{"status":200,"content_type":"text/html","bytes":936,"sha256":"08fec6c83f826ea6b41d2f8536f13526ed5954ff3269959166661ed49effbf39","final_host":"account.mail.ru","final_path":"/login","ms":1737,"path":"/.well-known/agent-skills/index.json","html":true,"json":null,"present":false},"mcp_card":{"status":200,"content_type":"text/html","bytes":936,"sha256":"52a5d4bec14c7f7af90d7b6e8d52c727758eddc27b6d6ac54bd5f6472fd07d3e","final_host":"account.mail.ru","final_path":"/login","ms":1657,"path":"/.well-known/mcp/server-card.json","html":true,"json":null,"present":false},"api_catalog":{"status":200,"content_type":"text/html","bytes":936,"sha256":"77c41e53773c5d6950cdcb44ac4c63eab17dfb0dc824a4f76e4fee73b0919d05","final_host":"account.mail.ru","final_path":"/login","ms":1406,"path":"/.well-known/api-catalog","html":true,"json":null,"present":false},"a2a_card":{"status":200,"content_type":"text/html","bytes":936,"sha256":"29075a11d1aeef69cfdf6995ca863deca29006df6ed624e2de90e48b6ba6d786","final_host":"account.mail.ru","final_path":"/login","ms":1552,"path":"/.well-known/agent-card.json","html":true,"json":null,"present":false}},"rings":{"v":1,"candidates":[],"mcp":[],"a2a":null,"webmcp":{"model_context":false,"bridge":false,"mcp_json":false,"link_rel":false,"any":false},"handshakes":0,"get_path":{"status":200,"content_type":"text/html","ms":987,"error":null},"get_host":{"status":200,"content_type":"text/html","ms":2678,"error":null}},"sf":[-1,0,0,0,0,0,2,0,0],"derived":{"reachable":true,"home_class":"answers","robots_served":true,"robots_sitemap":false,"content_signal":false,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":[],"blocks_any_ai":false,"files":{"llms_txt":false,"agents_md":false,"agent_skills":false,"mcp_card":false,"api_catalog":false,"a2a_card":false},"any_capability":false,"ai_refused":[],"ai_thin":["gptbot","claudebot","oaisearch","perplexity"],"aipref":0,"aipref_measured":true,"sf":[-1,0,0,0,0,0,2,0,0],"rings":{"mcp_declared":false,"mcp_live":false,"mcp_auth":false,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":3,"fh":"mail.ru","fp_other_host":["gptbot","claudebot","oaisearch","perplexity"],"fp":"9089414431008018","fp_bits":{}},"derive_v":5,"frame":"top10k","shard":3},{"v":1,"kind":"crawlcheck-census-record","run":18,"domain":"fbcdn.net","rank":13,"measured_at":"2026-11-13T18:51:40.579Z","home":{"browser":{"status":400,"content_type":"text/html","bytes":3676,"sha256":"8b116ba543cdb38c54776337d9722196efc582419cd2cdcc48568fd49f92a300","final_host":"m.facebook.com","final_path":"/","ms":495,"identity":"Mobile browser","html":true},"gptbot":{"status":200,"content_type":"text/html","bytes":367066,"sha256":"7f7f4ebaddb74908098ed7e32542ae960552b8892e03873826db3a4ec2d3a545","final_host":"www.facebook.com","final_path":"/","ms":497,"identity":"GPTBot","html":true,"fp":"1eb5bc9249c88fab"},"claudebot":{"status":200,"content_type":"text/html","bytes":367026,"sha256":"e7107ede011cff90796181f59c331001b9058bd89be92d0f2f081a7edc0e5506","final_host":"www.facebook.com","final_path":"/","ms":561,"identity":"ClaudeBot","html":true,"fp":"1eb5bc9249c88fab"},"oaisearch":{"status":200,"content_type":"text/html","bytes":366972,"sha256":"1fa6dd9751ea2d29617a7b132169cdf09fa3382a8640e3d1af093548e22c59af","final_host":"www.facebook.com","final_path":"/","ms":509,"identity":"OAI-SearchBot","html":true,"fp":"1eb5bc9249c88fab"},"perplexity":{"status":200,"content_type":"text/html","bytes":367858,"sha256":"4f872d847bfe47ce0ab584b192a1e4fd133041bdce3eab4b762fb07e688d3e32","final_host":"www.facebook.com","final_path":"/","ms":383,"identity":"PerplexityBot","html":true,"fp":"1eb5bc9249c88fab"}},"robots":{"status":200,"content_type":"text/plain","bytes":3400,"sha256":"17d601824568141914b8889d631e92c35aa17dec0be3e567c762cb670202d53f","final_host":"fbcdn.net","final_path":"/robots.txt","ms":143,"html":false,"text":"# Notice: Collection of data on Facebook through automated means is\n# prohibited unless you have express written permission from Facebook\n# and may only be conducted for the limited purpose contained in said\n# permission.\n# All authorized user-agents listed on this page must comply with Meta’s\n# Automated Data Collection Terms available at:\n# https://www.facebook.com/legal/automated_data_collection_terms\nUser-agent: __usage_ai-training\nDisallow: /\n\nUser-agent: Amazonbot\nDisallow: /\n\nUser-agent: Applebot-Extended\nDisallow: /\n\nUser-agent: Brightbot\nDisallow: /\n\nUser-agent: ClaudeBot\nDisallow: /\n\nUser-agent: Google-Extended\nDisallow: /\n\nUser-agent: GPTBot\nDisallow: /\n\nUser-agent: PerplexityBot\nDisallow: /\n\nUser-agent: PetalBot\nDisallow: /\n\nUser-agent: Scrapy\nDisallow: /\n\nUser-agent: uptimerobot\nDisallow: /\n\nUser-agent: viberbot\nDisallow: /\n\nUser-agent: YaK\nDisallow: /\n\nUser-agent: Yandex\nDisallow: /\n\nUser-agent: Yeti\nDisallow: /\n\nUser-agent: Applebot\nDisallow: /\nAllow: /map_tile.php\nAllow: /security/hsts-pixel.gif\nAllow: /static_map.php\n\nUser-agent: Bingbot\nDisallow: /\nAllow: /map_tile.php\nAllow: /security/hsts-pixel.gif\nAllow: /static_map.php\n\nUser-agent: deepcrawl\nDisallow: /\nAllow: /map_tile.php\nAllow: /security/hsts-pixel.gif\nAllow: /static_map.php\n\nUser-agent: Discordbot\nDisallow: /\nAllow: /map_tile.php\nAllow: /security/hsts-pixel.gif\nAllow: /static_map.php\n\nUser-agent: DuckDuckBot\nDisallow: /\nAllow: /map_tile.php\nAllow: /security/hsts-pixel.gif\nAllow: /static_map.php\n\nUser-agent: facebookexternalhit\nDisallow: /\nAllow: /map_tile.php\nAllow: /security/hsts-pixel.gif\nAllow: /static_map.php\n\nUser-agent: Googlebot\nDisallow: /\nAllow: /map_tile.php\nAllow: /security/hsts-pixel.gif\nAllow: /static_map.php\n\nUser-agent: Googlebot-Image\nDisallow: /\nDisallow: /map_tile.php\nDisallow: /static_map.php\nAllow: /map_tile.php\nAllow: /security/hsts-pixel.gif\nAllow: /static_map.php\n\nUser-agent: Googlebot-Video\nDisallow: /\nAllow: /map_tile.php\nAllow: /security/hsts-pixel.gif\nAllow: /static_map.php\n\nUser-agent: Googlebot-News\nDisallow: /\nAllow: /map_tile.php\nAllow: /security/hsts-pixel.gif\nAllow: /static_map.php\n\nUser-agent: Google-InspectionTool\nDisallow: /\nAllow: /map_tile.php\nAllow: /security/hsts-pixel.gif\nAllow: /static_map.php\n\nUser-agent: lumar\nDisallow: /\nAllow: /map_tile.php\nAllow: /security/hsts-pixel.gif\nAllow: /static_map.php\n\nUser-agent: LinkedInBot\nDisallow: /\nAllow: /map_tile.php\nAllow: /security/hsts-pixel.gif\nAllow: /static_map.php\n\nUser-agent: msnbot\nDisallow: /\nAllow: /map_tile.php\nAllow: /security/hsts-pixel.gif\nAllow: /static_map.php\n\nUser-agent: Pinterestbot\nDisallow: /\nAllow: /map_tile.php\nAllow: /security/hsts-pixel.gif\nAllow: /static_map.php\n\nUser-agent: Screaming Frog SEO Spider\nDisallow: /\nAllow: /map_tile.php\nAllow: /security/hsts-pixel.gif\nAllow: /static_map.php\n\nUser-agent: seznambot\nDisallow: /\nAllow: /map_tile.php\nAllow: /security/hsts-pixel.gif\nAllow: /static_map.php\n\nUser-agent: Slurp\nDisallow: /\nAllow: /map_tile.php\nAllow: /security/hsts-pixel.gif\nAllow: /static_map.php\n\nUser-agent: teoma\nDisallow: /\nAllow: /map_tile.php\nAllow: /security/hsts-pixel.gif\nAllow: /static_map.php\n\nUser-agent: TelegramBot\nDisallow: /\nAllow: /map_tile.php\nAllow: /security/hsts-pixel.gif\nAllow: /static_map.php\n\nUser-agent: Twitterbot\nDisallow: /\nAllow: /map_tile.php\nAllow: /security/hsts-pixel.gif\nAllow: /static_map.php\n\nUser-agent: *\nDisallow: /\n\n\n","agents":{"GPTBot":false,"OAI-SearchBot":false,"ChatGPT-User":false,"ClaudeBot":false,"Claude-SearchBot":false,"PerplexityBot":false,"Google-Extended":false,"Applebot-Extended":false,"CCBot":false,"Bytespider":false,"Meta-ExternalAgent":false,"Amazonbot":false}},"sitemap":null,"files":null,"derived":{"reachable":false,"home_class":"refused_us","robots_served":false,"robots_sitemap":false,"content_signal":false,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":[],"blocks_any_ai":false,"files":{"llms_txt":false,"agents_md":false,"agent_skills":false,"mcp_card":false,"api_catalog":false,"a2a_card":false},"any_capability":false,"ai_refused":[],"ai_thin":[],"rings":{"mcp_declared":false,"mcp_live":false,"mcp_auth":false,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":3,"fh":"m.facebook.com","fp_other_host":[]},"derive_v":5,"frame":"top10k","shard":4},{"v":1,"kind":"crawlcheck-census-record","run":18,"domain":"ezviz7.com","rank":14,"measured_at":"2026-11-13T18:52:33.408Z","home":{"browser":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"ezviz7.com","final_path":"/","ms":517,"identity":"Mobile browser","html":false},"gptbot":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"ezviz7.com","final_path":"/","ms":517,"identity":"GPTBot","html":false},"claudebot":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"ezviz7.com","final_path":"/","ms":505,"identity":"ClaudeBot","html":false},"oaisearch":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"ezviz7.com","final_path":"/","ms":495,"identity":"OAI-SearchBot","html":false},"perplexity":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"ezviz7.com","final_path":"/","ms":492,"identity":"PerplexityBot","html":false}},"robots":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"ezviz7.com","final_path":"/robots.txt","ms":407,"html":false,"text":null,"agents":null},"sitemap":null,"files":null,"derived":{"reachable":false,"home_class":"no_site","robots_served":false,"robots_sitemap":false,"content_signal":false,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":[],"blocks_any_ai":false,"files":{"llms_txt":false,"agents_md":false,"agent_skills":false,"mcp_card":false,"api_catalog":false,"a2a_card":false},"any_capability":false,"ai_refused":[],"ai_thin":[],"rings":{"mcp_declared":false,"mcp_live":false,"mcp_auth":false,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":3,"fh":"ezviz7.com","fp_other_host":[]},"derive_v":5,"frame":"top10k","shard":5},{"v":1,"kind":"crawlcheck-census-record","run":18,"domain":"twitter.com","rank":15,"measured_at":"2026-11-13T18:53:15.625Z","home":{"browser":{"status":200,"content_type":"text/html","bytes":46108,"sha256":"d1c6b5cd1f76eca1e9f92b280290df134a2c7d735da0d4b749178dc81b42373e","final_host":"x.com","final_path":"/","ms":111,"identity":"Mobile browser","html":true,"fp":"8b48d3564ad225ec"},"gptbot":{"status":402,"content_type":"application/json","bytes":55,"sha256":"35e642f0e1bd426b9d8b0ee0284aefe801873dc13c18819fbeeaf2bf47136664","final_host":"x.com","final_path":"/","ms":40,"identity":"GPTBot","html":false},"claudebot":{"status":403,"content_type":null,"bytes":0,"sha256":"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855","final_host":"x.com","final_path":"/","ms":126,"identity":"ClaudeBot","html":null},"oaisearch":{"status":402,"content_type":"application/json","bytes":55,"sha256":"35e642f0e1bd426b9d8b0ee0284aefe801873dc13c18819fbeeaf2bf47136664","final_host":"x.com","final_path":"/","ms":85,"identity":"OAI-SearchBot","html":false},"perplexity":{"status":402,"content_type":"application/json","bytes":55,"sha256":"35e642f0e1bd426b9d8b0ee0284aefe801873dc13c18819fbeeaf2bf47136664","final_host":"x.com","final_path":"/","ms":35,"identity":"PerplexityBot","html":false}},"robots":{"status":200,"content_type":"text/plain","bytes":3039,"sha256":"b54dd4fefb8786b6441dc95ab04db1dbe4c6b9fcb5e7bb60d8381ab6e40989ff","final_host":"twitter.com","final_path":"/robots.txt","ms":29,"html":false,"text":"# Google / Bing Search Engine Robots\n# ==================================\n# Shared group so Googlebot and Bingbot always get the same rules.\nUser-agent: Googlebot\nUser-agent: Bingbot\n\nAllow: /*?s=\nAllow: /*?t=\nAllow: /*?ref_src=\nAllow: /hashtag/*?src=\nAllow: /search?q=%23\nAllow: /i/api/\n# Same length as the /*/likes and /*/media rules; RFC 9309 breaks the tie in\n# favor of Allow, so /i/api/ fetches stay crawlable.\nAllow: /i/api/*\nDisallow: /*?lang=en-ss\nAllow: /*?lang=\nDisallow: /search/realtime\nDisallow: /search/users\nDisallow: /search/*/grid\n\nDisallow: /*/analytics\nDisallow: /*/followers\nDisallow: /*/following\nDisallow: /*/verified_followers\n\nDisallow: /account/deactivated\nDisallow: /settings/deactivated\n\n# Only `*` and `$` are metacharacters here (RFC 9309); regex classes match\n# literally. /photo is anchored so /status/*/photo/1 media pages stay open.\nDisallow: /*/likes\nDisallow: /*/likes?\nDisallow: /*/retweets\nDisallow: /*/retweets?\nDisallow: /*/media\nDisallow: /*/media?\nDisallow: /*/photo$\nDisallow: /*/photo?\nAllow: /*?\n\nUser-agent: facebookexternalhit\n\nAllow: /*?lang=\nAllow: /*?s=\nAllow: /*&s=\nAllow: /*?t=\nAllow: /*?ref_src=\nAllow: /hashtag/*?src=\nAllow: /search?*cashtagRestId=\nAllow: /i/api/\nAllow: /i/api/*\nDisallow: /search?q=\nDisallow: /search/realtime\nDisallow: /search/users\nDisallow: /search/*/grid\n\nDisallow: /*?\nDisallow: /*/followers\nDisallow: /*/following\nDisallow: /*/verified_followers\n\nDisallow: /account/deactivated\nDisallow: /settings/deactivated\n\nDisallow: /*/likes\nDisallow: /*/likes?\nDisallow: /*/retweets\nDisallow: /*/retweets?\nDisallow: /*/media\nDisallow: /*/media?\nDisallow: /*/photo$\nDisallow: /*/photo?\n\nUser-Agent: Google-Extended\nDisallow: *\n\nUser-Agent: FacebookBot\nDisallow: *\n\nUser-agent: Discordbot\nDisallow: *\n\n# Meta AI / product crawlers (product tokens; versions like /1.1 are ignored\n# under RFC 9309 token matching).\nUser-agent: meta-webindexer\nDisallow: *\n\nUser-agent: meta-externalagent\nDisallow: *\n\nUser-agent: meta-externalads\nDisallow: *\n\nUser-agent: meta-externalfetcher\nDisallow: *\n\n# Shared AI training preference (IETF draft-kelley-aipref-training-preference-signal),\n# honored by Microsoft, Google, OpenAI, and Anthropic. This is a usage policy\n# token, not a crawler: it does not change crawl access, and `User-agent: *`\n# never matches it.\nUser-agent: __usage_ai-training\nDisallow: /\n\n# Every bot that might possibly read and respect this file\n# ========================================================\nUser-agent: *\nDisallow: /\n\n\n# WHAT-4882 - Keep notification-email links (/i/u) out of search results.\n# Named crawlers stay un-blocked on purpose: they must crawl /i/u to see its\n# X-Robots-Tag noindex (the robots.txt Noindex directive died in 2019).\nDisallow: /i/u\n\n# Wait 1 second between successive requests. See ONBOARD-2698 for details.\nCrawl-delay: 1\n\n# Independent of user agent. Links in the sitemap are full URLs using https://\n# and need to match the protocol of the sitemap.\nSitemap: https://x.com/sitemap.xml\nSitemap: https://x.com/sitemaps/finance0\n","agents":{"GPTBot":false,"OAI-SearchBot":false,"ChatGPT-User":false,"ClaudeBot":false,"Claude-SearchBot":false,"PerplexityBot":false,"Google-Extended":false,"Applebot-Extended":false,"CCBot":false,"Bytespider":false,"Meta-ExternalAgent":false,"Amazonbot":false}},"sitemap":{"status":404,"content_type":"text/html","bytes":46103,"sha256":"66e7a15df5a2e6c858fb9496127cb65ab671f29eedf91011c7a4f305a5d9e5ec","final_host":"x.com","final_path":"/sitemap.xml","ms":90,"url":"https://x.com/sitemap.xml","declared":true,"xml":false},"files":{"llms_txt":{"status":404,"content_type":"text/html","bytes":46081,"sha256":"9fab0f3cc8b3fab5192c6f88e4e183dd650477c3d747d866327cd1b5bf3f7128","final_host":"x.com","final_path":"/llms.txt","ms":173,"path":"/llms.txt","html":true,"json":null,"present":false},"agents_md":{"status":404,"content_type":"text/html","bytes":46089,"sha256":"10e6954bef29e58fe3fb2165796fda36c7e5c1ea43e5701a5a69cb52d031ffed","final_host":"x.com","final_path":"/agents.md","ms":103,"path":"/agents.md","html":true,"json":null,"present":false},"agent_skills":{"status":404,"content_type":"text/html","bytes":46271,"sha256":"463a8bd57038fe119a6c9611b8bd2b1c79f2737249189d0a7084a73e175d0e9c","final_host":"x.com","final_path":"/.well-known/agent-skills/index.json","ms":116,"path":"/.well-known/agent-skills/index.json","html":true,"json":null,"present":false},"mcp_card":{"status":404,"content_type":"text/html","bytes":46254,"sha256":"959b1b678386b46e8af4bb49e35751a0b4c4d9374c35b80f8778dc1fe643c2e9","final_host":"x.com","final_path":"/.well-known/mcp/server-card.json","ms":164,"path":"/.well-known/mcp/server-card.json","html":true,"json":null,"present":false},"api_catalog":{"status":404,"content_type":"text/html","bytes":46188,"sha256":"6bfac0a66d26fb8c8b58de9bc5d1c7c6416aefbed9a08009e10416ea6fb40bdf","final_host":"x.com","final_path":"/.well-known/api-catalog","ms":111,"path":"/.well-known/api-catalog","html":true,"json":null,"present":false},"a2a_card":{"status":404,"content_type":"text/html","bytes":46210,"sha256":"d19fcddf9ec3709b126b6e86aa13a397433258f3af125e8dd915861f86e75889","final_host":"x.com","final_path":"/.well-known/agent-card.json","ms":109,"path":"/.well-known/agent-card.json","html":true,"json":null,"present":false}},"rings":{"v":1,"candidates":[],"mcp":[],"a2a":null,"webmcp":{"model_context":false,"bridge":false,"mcp_json":false,"link_rel":false,"any":false},"handshakes":0,"get_path":{"status":200,"content_type":"text/html","ms":995,"error":null},"get_host":{"status":530,"content_type":"text/plain","ms":85,"error":null}},"sf":[4,0,0,0,0,0,1,0,1],"derived":{"reachable":true,"home_class":"answers","robots_served":true,"robots_sitemap":true,"content_signal":false,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":["GPTBot","OAI-SearchBot","ChatGPT-User","ClaudeBot","Claude-SearchBot","PerplexityBot","Google-Extended","Applebot-Extended","CCBot","Bytespider","Meta-ExternalAgent","Amazonbot"],"blocks_any_ai":true,"files":{"llms_txt":false,"agents_md":false,"agent_skills":false,"mcp_card":false,"api_catalog":false,"a2a_card":false},"any_capability":false,"ai_refused":["gptbot","claudebot","oaisearch","perplexity"],"ai_thin":[],"aipref":2,"aipref_measured":true,"sf":[4,0,0,0,0,0,1,0,1],"rings":{"mcp_declared":false,"mcp_live":false,"mcp_auth":false,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":3,"fh":"x.com","fp_other_host":[],"fp":"8b48d3564ad225ec","fp_bits":{}},"derive_v":5,"frame":"top10k","shard":6},{"v":1,"kind":"crawlcheck-census-record","run":18,"domain":"domaincontrol.com","rank":16,"measured_at":"2026-11-13T18:55:41.951Z","home":{"browser":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"domaincontrol.com","final_path":"/","ms":8,"identity":"Mobile browser","html":false},"gptbot":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"domaincontrol.com","final_path":"/","ms":1,"identity":"GPTBot","html":false},"claudebot":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"domaincontrol.com","final_path":"/","ms":1,"identity":"ClaudeBot","html":false},"oaisearch":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"domaincontrol.com","final_path":"/","ms":1,"identity":"OAI-SearchBot","html":false},"perplexity":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"domaincontrol.com","final_path":"/","ms":0,"identity":"PerplexityBot","html":false}},"robots":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"domaincontrol.com","final_path":"/robots.txt","ms":1,"html":false,"text":null,"agents":null},"sitemap":null,"files":null,"derived":{"reachable":false,"home_class":"no_site","robots_served":false,"robots_sitemap":false,"content_signal":false,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":[],"blocks_any_ai":false,"files":{"llms_txt":false,"agents_md":false,"agent_skills":false,"mcp_card":false,"api_catalog":false,"a2a_card":false},"any_capability":false,"ai_refused":[],"ai_thin":[],"rings":{"mcp_declared":false,"mcp_live":false,"mcp_auth":false,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":3,"fh":"domaincontrol.com","fp_other_host":[]},"derive_v":5,"frame":"top10k","shard":7},{"v":1,"kind":"crawlcheck-census-record","run":18,"domain":"dzen.ru","rank":17,"measured_at":"2026-11-13T18:46:27.088Z","home":{"browser":{"status":200,"content_type":"text/html","bytes":3045,"sha256":"76a0ea096953a780ae5e91ca948c38205ccc95dfd8f4284715bf7ffac21c9eb6","final_host":"sso.passport.yandex.ru","final_path":"/push","ms":3895,"identity":"Mobile browser","html":true},"gptbot":{"status":200,"content_type":"text/html","bytes":2886,"sha256":"668f1e70e508cf98491193974cbb9d59c2f2ea1919ef3a0132d9ac9a8d851eeb","final_host":"sso.passport.yandex.ru","final_path":"/push","ms":3746,"identity":"GPTBot","html":true},"claudebot":{"status":200,"content_type":"text/html","bytes":2886,"sha256":"11262cbced9d95fdc8d3697978b786e9f4582c470d03d7330b7701145294b050","final_host":"sso.passport.yandex.ru","final_path":"/push","ms":3686,"identity":"ClaudeBot","html":true},"oaisearch":{"status":200,"content_type":"text/html","bytes":2883,"sha256":"ce325b289f6b8e5069d18581a92f62d26e74767aae3f9ac4927dd1b72c38ef92","final_host":"sso.passport.yandex.ru","final_path":"/push","ms":5557,"identity":"OAI-SearchBot","html":true},"perplexity":{"status":200,"content_type":"text/html","bytes":2886,"sha256":"6bb9a2b9dacc1fdd3a9b963595f7438a8de3da53e161159f3701034b2bb86c13","final_host":"sso.passport.yandex.ru","final_path":"/push","ms":3494,"identity":"PerplexityBot","html":true}},"robots":{"status":200,"content_type":"text/plain","bytes":9794,"sha256":"a6e7d16a66dcbbfe8d64ec966726922ac7736fd28f3a4b5bede7897992c92086","final_host":"dzen.ru","final_path":"/robots.txt","ms":1437,"html":false,"text":"User-agent: Yandex\nAllow: /\nAllow: /.well-known/assetlinks.json\nAllow: /*?utm_sourse=55\n\nDisallow: /about*?*\nDisallow: /away\nDisallow: /top$\nDisallow: /search\nDisallow: /money\nDisallow: /onboarding\nDisallow: /news/top\nDisallow: /news/smi\nDisallow: /news/search\nDisallow: /t/\nDisallow: /user/\nDisallow: /embed/\nDisallow: /*/url\nDisallow: /video/games/tags/\nDisallow: /*sso_failed=\nDisallow: /*erid=\nDisallow: /*?utm*\nDisallow: /games\nDisallow: /subscriptions-manager?channel=\n\nDisallow: /articles?\nDisallow: /video?\nDisallow: /discussions\n\nClean-param: &_csrf\nClean-param: &action\nClean-param: &amp\nClean-param: &ampplace\nClean-param: &amupr_id\nClean-param: &appsearch_header\nClean-param: &autoplay\nClean-param: &backoffice\nClean-param: &branding\nClean-param: &briefEditorPublicationId\nClean-param: &bx_sender_conversion_id\nClean-param: &clid\nClean-param: &collection\nClean-param: &comment-request\nClean-param: &comments_data\nClean-param: &content\nClean-param: &contour\nClean-param: &country_code\nClean-param: &device_id\nClean-param: &encoded_pulse_user_id\nClean-param: &encoded_pulse_user_info\nClean-param: &epik\nClean-param: &etext=\nClean-param: &experiment\nClean-param: &f\nClean-param: &family\nClean-param: &fan\nClean-param: &favid\nClean-param: &feed_exp\nClean-param: &feed_filter_source\nClean-param: &firstchars\nClean-param: &flight_id\nClean-param: &from\nClean-param: &from_block\nClean-param: &from_cloud\nClean-param: &from_floors_promo\nClean-param: &from_page\nClean-param: &from_parent_id\nClean-param: &from_parent_type\nClean-param: &from_site\nClean-param: &ga_client_id\nClean-param: &hide_interest_header\nClean-param: &id\nClean-param: &integration\nClean-param: &interview_id\nClean-param: &invite\nClean-param: &item_id\nClean-param: &lang\nClean-param: &link\nClean-param: &loop\nClean-param: &lr\nClean-param: &modal\nClean-param: &mt_click_id\nClean-param: &mt_link_id\nClean-param: &mute\nClean-param: &no_ad\nClean-param: &noredir\nClean-param: &nw\nClean-param: &p\nClean-param: &page\nClean-param: &parent_rid\nClean-param: &persistent_id\nClean-param: &place\nClean-param: &play_on_visible\nClean-param: &pulse_user_id\nClean-param: &query\nClean-param: &ref\nClean-param: &referrer\nClean-param: &referrer_clid\nClean-param: &referrer_place\nClean-param: &repostId\nClean-param: &rid\nClean-param: &rubric\nClean-param: &secdata\nClean-param: &service\nClean-param: &share_to\nClean-param: &sid\nClean-param: &sortby\nClean-param: &source\nClean-param: &sso_failed\nClean-param: &stid\nClean-param: &story\nClean-param: &t\nClean-param: &text\nClean-param: &to\nClean-param: &token\nClean-param: &train\nClean-param: &tst\nClean-param: &tt\nClean-param: &tv\nClean-param: &utr\nClean-param: &uuid\nClean-param: &vertical\nClean-param: &via\nClean-param: &vid\nClean-param: &video_id\nClean-param: &wan\nClean-param: &web_uid\nClean-param: &web_yandexuid\nClean-param: &wizard\nClean-param: &yandexClientId\n\nUser-agent: Googlebot\nAllow: /\nAllow: /.well-known/assetlinks.json\nAllow: /list/*\n\nAllow: /a/*_csrf\nAllow: /a/*action\nAllow: /a/*amp\nAllow: /a/*ampplace\nAllow: /a/*amupr_id\nAllow: /a/*appsearch_header=\nAllow: /a/*autoplay=\nAllow: /a/*backoffice=\nAllow: /a/*branding=\nAllow: /a/*briefEditorPublicationId=\nAllow: /a/*bx_sender_conversion_id=\nAllow: /a/*cachebust=\nAllow: /a/*clid=\nAllow: /a/*collection=\nAllow: /a/*country_code=\nAllow: /a/*erid=\nAllow: /a/*experiment=\nAllow: /a/*feed_exp=\nAllow: /a/*from_cloud=\nAllow: /a/*from_page=\nAllow: /a/*invite=\nAllow: /a/*loop=\nAllow: /a/*pulse_user_id=\nAllow: /a/*sso_failed=\nAllow: /a/*t=\nAllow: /a/*to=\nAllow: /a/*token=\nAllow: /a/*utm_referrer=\nAllow: /a/*utm_source=\nAllow: /a/*?etext=\n\nAllow: /b/*_csrf\nAllow: /b/*action\nAllow: /b/*amp\nAllow: /b/*ampplace\nAllow: /b/*amupr_id\nAllow: /b/*appsearch_header=\nAllow: /b/*autoplay=\nAllow: /b/*backoffice=\nAllow: /b/*branding=\nAllow: /b/*briefEditorPublicationId=\nAllow: /b/*bx_sender_conversion_id=\nAllow: /b/*cachebust=\nAllow: /b/*clid=\nAllow: /b/*collection=\nAllow: /b/*country_code=\nAllow: /b/*erid=\nAllow: /b/*experiment=\nAllow: /b/*feed_exp=\nAllow: /b/*from_cloud=\nAllow: /b/*from_page=\nAllow: /b/*invite=\nAllow: /b/*loop=\nAllow: /b/*pulse_user_id=\nAllow: /b/*sso_failed=\nAllow: /b/*t=\nAllow: /b/*to=\nAllow: /b/*token=\nAllow: /b/*utm_referrer=\nAllow: /b/*utm_source=\nAllow: /b/*?etext=\n\nAllow: /list/*_csrf\nAllow: /list/*action\nAllow: /list/*amp\nAllow: /list/*ampplace\nAllow: /list/*amupr_id\nAllow: /list/*appsearch_header=\nAllow: /list/*autoplay=\nAllow: /list/*backoffice=\nAllow: /list/*branding=\nAllow: /list/*briefEditorPublicationId=\nAllow: /list/*bx_sender_conversion_id=\nAllow: /list/*cachebust=\nAllow: /list/*clid=\nAllow: /list/*collection=\nAllow: /list/*country_code=\nAllow: /list/*erid=\nAllow: /list/*experiment=\nAllow: /list/*feed_exp=\nAllow: /list/*from_cloud=\nAllow: /list/*from_page=\nAllow: /list/*invite=\nAllow: /list/*loop=\nAllow: /list/*pulse_user_id=\nAllow: /list/*sso_failed=\nAllow: /list/*t=\nAllow: /list/*to=\nAllow: /list/*token=\nAllow: /list/*utm_referrer=\nAllow: /list/*utm_source=\nAllow: /list/*?etext=\n\nAllow: /video/watch/*_csrf\nAllow: /video/watch/*action\nAllow: /video/watch/*amp\nAllow: /video/watch/*ampplace\nAllow: /video/watch/*amupr_id\nAllow: /video/watch/*appsearch_header=\nAllow: /video/watch/*autoplay=\nAllow: /video/watch/*backoffice=\nAllow: /video/watch/*branding=\nAllow: /video/watch/*briefEditorPublicationId=\nAllow: /video/watch/*bx_sender_conversion_id=\nAllow: /video/watch/*cachebust=\nAllow: /video/watch/*clid=\nAllow: /video/watch/*collection=\nAllow: /video/watch/*country_code=\nAllow: /video/watch/*erid=\nAllow: /video/watch/*experiment=\nAllow: /video/watch/*feed_exp=\nAllow: /video/watch/*from_cloud=\nAllow: /video/watch/*from_page=\nAllow: /video/watch/*invite=\nAllow: /video/watch/*loop=\nAllow: /video/watch/*pulse_user_id=\nAllow: /video/watch/*sso_failed=\nAllow: /video/watch/*t=\nAllow: /video/watch/*to=\nAllow: /video/watch/*token=\nAllow: /video/watch/*utm_referrer=\nAllow: /video/watch/*utm_source=\nAllow: /video/watch/*?etext=\n\nAllow: /shorts/*_csrf\nAllow: /shorts/*action\nAllow: /shorts/*amp\nAllow: /shorts/*ampplace\nAllow: /shorts/*amupr_id\nAllow: /shorts/*appsearch_header=\nAllow: /shorts/*autoplay=\nAllow: /shorts/*backoffice=\nAllow: /shorts/*branding=\nAllow: /shorts/*briefEditorPublicationId=\nAllow: /shorts/*bx_sender_conversion_id=\nAllow: /shorts/*cachebust=\nAllow: /shorts/*clid=\nAllow: /shorts/*collection=\nAllow: /shorts/*country_code=\nAllow: /shorts/*erid=\nAllow: /shorts/*experiment=\nAllow: /shorts/*feed_exp=\nAllow: /shorts/*from_cloud=\nAllow: /shorts/*from_page=\nAllow: /shorts/*invite=\nAllow: /shorts/*loop=\nAllow: /shorts/*pulse_user_id=\nAllow: /shorts/*sso_failed=\nAllow: /shorts/*t=\nAllow: /shorts/*to=\nAllow: /shorts/*token=\nAllow: /shorts/*utm_referrer=\nAllow: /shorts/*utm_source=\nAllow: /shorts/*?etext=\n\nDisallow: /about*?*\nDisallow: /away\nDisallow: /top$\nDisallow: /search\nDisallow: /money\nDisallow: /onboarding\nDisallow: /news/top\nDisallow: /news/smi\nDisallow: /news/search\nDisallow: /t/\nDisallow: /user/\nDisallow: /embed/\nDisallow: /*/url\nDisallow: /video/games/tags/\nDisallow: /*_csrf\nDisallow: /*action\nDisallow: /*amp\nDisallow: /*ampplace\nDisallow: /*amupr_id\nDisallow: /*appsearch_header=\nDisallow: /*autoplay=\nDisallow: /*backoffice=\nDisallow: /*branding=\nDisallow: /*briefEditorPublicationId=\nDisallow: /*bx_sender_conversion_id=\nDisallow: /*cachebust=\nDisallow: /*clid=\nDisallow: /*collection=\nDisallow: /*country_code=\nDisallow: /*erid=\nDisallow: /*experiment=\nDisallow: /*feed_exp=\nDisallow: /*from_cloud=\nDisallow: /*from_page=\nDisallow: /*invite=\nDisallow: /*loop=\nDisallow: /*pulse_user_id=\nDisallow: /*sso_failed=\nDisallow: /*t=\nDisallow: /*to=\nDisallow: /*token=\nDisallow: /*utm_referrer=\nDisallow: /*utm_source=\nDisallow: /games\nDisallow: /*?etext=\nDisallow: /subscriptions-manager?channel=\n\nDisallow: /articles?\nDisallow: /video?\n\nUser-Agent: *\nAllow: /\nAllow: /.well-known/assetlinks.json\nDisallow: /about*?*\nDisallow: /away\nDisallow: /embed/\nDisallow: /games\nDisallow: /money\nDisallow: /news/search\nDisallow: /news/smi\nDisallow: /news/top\nDisallow: /onboarding\nDisallow: /search\nDisallow: /subscriptions-manager?channel=\nDisallow: /t/\nDisallow: /top$\nDisallow: /user/\nDisallow: /*/url\nDisallow: /video/games/tags/\nDisallow: /*erid=\nDisallow: /*?etext=\nDisallow: /*sso_failed=\nDisallow: /*?sso_failed=\nDisallow: /*to=\n\nSitemap: https://dzen.ru/sitemaps/brief_v2/sitemap.xml\nSitemap: https://dzen.ru/sitemaps/channels_v2/sitemap.xml\nSitemap: https://dzen.ru/sitemaps/gif_long_v2/sitemap.xml\nSitemap: https://dzen.ru/sitemaps/gif_short_v2/sitemap.xml\nSitemap: https://dzen.ru/sitemaps/image_v2/sitemap.xml\nSitemap: https://dzen.ru/sitemaps/native_v2/sitemap.xml\nSitemap: https://dzen.ru/sitemaps/slug-factory/sitemap.xml\nSitemap: https://dzen.ru/sitemaps/suite_v2/sitemap.xml\nSitemap: https://dzen.ru/sitemaps/tv/sitemap.xml\nSitemap: https://dzen.ru/sitemaps/news-archive/sitemap.xml\nSitemap: https://dzen.ru/sitemaps/news-region/sitemap.xml\nSitemap: https://dzen.ru/sitemaps/news-rubric/sitemap.xml\nSitemap: https://dzen.ru/sitemaps/meta-channel_v2/sitemap.xml\n\nSitemap: https://dzen.ru/sitemaps/brief_v2/sitemap.xml.gz\nSitemap: https://dzen.ru/sitemaps/channels_v2/sitemap.xml.gz\nSitemap: https://dzen.ru/sitemaps/gif_long_v2/sitemap.xml.gz\nSitemap: https://dzen.ru/sitemaps/gif_short_v2/sitemap.xml.gz\nSitemap: https://dzen.ru/sitemaps/image_v2/sitemap.xml.gz\nSitemap: https://dzen.ru/sitemaps/native_v2/sitemap.xml.gz\nSitemap: https://dzen.ru/sitemaps/slug-factory/sitemap.xml.gz\nSitemap: https://dzen.ru/sitemaps/suite_v2/sitemap.xml.gz\nSitemap: https://dzen.ru/sitemaps/tv/sitemap.xml.gz\nSitemap: https://dzen.ru/sitemaps/news-archive/sitemap.xml.gz\nSitemap: https://dzen.ru/sitemaps/news-region/sitemap.xml.gz\nSitemap: https://dzen.ru/sitemaps/news-rubric/sitemap.xml.gz\nSitemap: https://dzen.ru/sitemaps/meta-channel_v2/sitemap.xml.gz","agents":{"GPTBot":true,"OAI-SearchBot":true,"ChatGPT-User":true,"ClaudeBot":true,"Claude-SearchBot":true,"PerplexityBot":true,"Google-Extended":true,"Applebot-Extended":true,"CCBot":true,"Bytespider":true,"Meta-ExternalAgent":true,"Amazonbot":true}},"sitemap":{"status":200,"content_type":"application/xml","bytes":32999,"sha256":"0a3f87fa6d0ea83757024d95676b150ff575a7f56f4ff62bfcb1a06cb3b34efa","final_host":"dzen.ru","final_path":"/sitemaps/brief_v2/sitemap.xml","ms":320,"url":"https://dzen.ru/sitemaps/brief_v2/sitemap.xml","declared":true,"xml":true},"files":{"llms_txt":{"status":200,"content_type":"text/html","bytes":2890,"sha256":"0084b8d54c7923ebbed23525281e06dbef13cf08391c747dd9f5e5464586e0f1","final_host":"sso.passport.yandex.ru","final_path":"/push","ms":595,"path":"/llms.txt","html":true,"json":null,"present":false},"agents_md":{"status":200,"content_type":"text/html","bytes":2892,"sha256":"621606bf01cf5102b68a0851ffa2ee4037391a801a4fc739f417b239942bcd41","final_host":"sso.passport.yandex.ru","final_path":"/push","ms":624,"path":"/agents.md","html":true,"json":null,"present":false},"agent_skills":{"status":200,"content_type":"text/html","bytes":2926,"sha256":"94601c118947713cb4845d9d83820c354051d9eb64f9f8cd30ec29f85206c020","final_host":"sso.passport.yandex.ru","final_path":"/push","ms":629,"path":"/.well-known/agent-skills/index.json","html":true,"json":null,"present":false},"mcp_card":{"status":200,"content_type":"text/html","bytes":2923,"sha256":"e338723936f3aff5f27204e319fa3b2066c796ffb1244621aba8faed2c431402","final_host":"sso.passport.yandex.ru","final_path":"/push","ms":1158,"path":"/.well-known/mcp/server-card.json","html":true,"json":null,"present":false},"api_catalog":{"status":200,"content_type":"text/html","bytes":2911,"sha256":"3bb0fe65701209b504314dda76002e31f51adaaa905213a49824fbf3f71d63a2","final_host":"sso.passport.yandex.ru","final_path":"/push","ms":1161,"path":"/.well-known/api-catalog","html":true,"json":null,"present":false},"a2a_card":{"status":200,"content_type":"text/html","bytes":2910,"sha256":"153bcccf3c7eb049fa64916adf70fb554e11f696d76fe0fefc00d7fd8f66d375","final_host":"sso.passport.yandex.ru","final_path":"/push","ms":664,"path":"/.well-known/agent-card.json","html":true,"json":null,"present":false}},"rings":{"v":1,"candidates":[],"mcp":[],"a2a":null,"webmcp":{"model_context":false,"bridge":false,"mcp_json":false,"link_rel":false,"any":false},"handshakes":0,"get_path":{"status":200,"content_type":"text/html","ms":622,"error":null},"get_host":{"status":530,"content_type":"text/plain","ms":613,"error":null}},"sf":[4,0,0,0,0,0,0,0,0],"derived":{"reachable":true,"home_class":"answers","robots_served":true,"robots_sitemap":true,"content_signal":false,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":[],"blocks_any_ai":false,"files":{"llms_txt":false,"agents_md":false,"agent_skills":false,"mcp_card":false,"api_catalog":false,"a2a_card":false},"any_capability":false,"ai_refused":[],"ai_thin":[],"aipref":0,"aipref_measured":true,"sf":[4,0,0,0,0,0,0,0,0],"rings":{"mcp_declared":false,"mcp_live":false,"mcp_auth":false,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":3,"fh":"sso.passport.yandex.ru","fp_other_host":[]},"derive_v":5,"frame":"top10k","shard":0},{"v":1,"kind":"crawlcheck-census-record","run":18,"domain":"linkedin.com","rank":18,"measured_at":"2026-11-13T18:48:18.168Z","home":{"browser":{"status":200,"content_type":"text/html","bytes":509861,"sha256":"a006e5e8f245d933ee1ccb7c819d6924571da4e09bbba94421ed63a6186318ef","final_host":"www.linkedin.com","final_path":"/","ms":635,"identity":"Mobile browser","html":true,"fp":"e0d414696e54024d"},"gptbot":{"status":999,"content_type":"text/html","bytes":2109,"sha256":"388be16d000f70286349134384cbd80f7b8eeac84e265029d6a2ab0ab4128ad9","final_host":"linkedin.com","final_path":"/","ms":206,"identity":"GPTBot","html":true},"claudebot":{"status":403,"content_type":"text/html","bytes":5502,"sha256":"4c61392bd4689504b47c7f518be37735d6443f5299800ea7e61cba0382461d72","final_host":"www.linkedin.com","final_path":"/","ms":207,"identity":"ClaudeBot","html":true},"oaisearch":{"status":200,"content_type":"text/html","bytes":141143,"sha256":"4c3b8c81a0df4bfecab622623e5bc509d361b25ca6de5e91eb0f2767fa9b878e","final_host":"www.linkedin.com","final_path":"/","ms":622,"identity":"OAI-SearchBot","html":true,"fp":"e072000f7ba10acd"},"perplexity":{"status":403,"content_type":"text/html","bytes":5695,"sha256":"d1c42e2c4a3b0bb2c773e318a1c5a033dbf005453f4fcd01db636d5be5594769","final_host":"www.linkedin.com","final_path":"/","ms":230,"identity":"PerplexityBot","html":true}},"robots":{"status":200,"content_type":"text/html","bytes":21342,"sha256":"8d301dac976596eaae694b6a7388b9dc8e8f4bbe95f0d13cfd057aaa2aaed39a","final_host":"linkedin.com","final_path":"/robots.txt","ms":40,"html":true,"text":null,"agents":null},"sitemap":{"status":200,"content_type":"text/html","bytes":21342,"sha256":"e324570460c092eab98281c312ea1bbb447c325c7918bad8d5512413f0699ac2","final_host":"linkedin.com","final_path":"/sitemap.xml","ms":25,"url":"https://linkedin.com/sitemap.xml","declared":false,"xml":false},"files":{"llms_txt":{"status":200,"content_type":"text/html","bytes":21334,"sha256":"3feb76980b3cddf7784dd20671c92821560292b941822bb83fb00b61a70a0029","final_host":"linkedin.com","final_path":"/llms.txt","ms":30,"path":"/llms.txt","html":true,"json":null,"present":false},"agents_md":{"status":200,"content_type":"text/html","bytes":21336,"sha256":"3b0f7314e7e0a3d8b9200ef34eecb08effa8dfb57981d02a610039d0950807b2","final_host":"linkedin.com","final_path":"/agents.md","ms":20,"path":"/agents.md","html":true,"json":null,"present":false},"agent_skills":{"status":200,"content_type":"text/html","bytes":21406,"sha256":"46c258e3db83180fc044c37af2e36cc71162a8dbe448fd091f34a6be9394d692","final_host":"linkedin.com","final_path":"/.well-known/agent-skills/index.json","ms":20,"path":"/.well-known/agent-skills/index.json","html":true,"json":null,"present":false},"mcp_card":{"status":200,"content_type":"text/html","bytes":21399,"sha256":"6d2d5aaa3d00a15ab6d028c962e06f92bd643283a53c9108607781c1974fbc59","final_host":"linkedin.com","final_path":"/.well-known/mcp/server-card.json","ms":27,"path":"/.well-known/mcp/server-card.json","html":true,"json":null,"present":false},"api_catalog":{"status":200,"content_type":"text/html","bytes":21375,"sha256":"4ab100d6ebfcb5c05017d06c4f418194e0dd1e786a898a0bbd9b1bf74e736bcc","final_host":"linkedin.com","final_path":"/.well-known/api-catalog","ms":20,"path":"/.well-known/api-catalog","html":true,"json":null,"present":false},"a2a_card":{"status":200,"content_type":"text/html","bytes":21384,"sha256":"2dd2daa631b6eb4ac46985b12ca1b09589f11fc997e672b1143329a1489e2568","final_host":"linkedin.com","final_path":"/.well-known/agent-card.json","ms":27,"path":"/.well-known/agent-card.json","html":true,"json":null,"present":false}},"rings":{"v":1,"candidates":[],"mcp":[],"a2a":null,"webmcp":{"model_context":false,"bridge":false,"mcp_json":false,"link_rel":false,"any":false},"handshakes":0,"get_path":{"status":200,"content_type":"text/html","ms":23,"error":null},"get_host":{"status":404,"content_type":"text/html","ms":218,"error":null}},"sf":[4,0,0,0,0,0,1,0,0],"derived":{"reachable":true,"home_class":"answers","robots_served":false,"robots_sitemap":false,"content_signal":false,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":[],"blocks_any_ai":false,"files":{"llms_txt":false,"agents_md":false,"agent_skills":false,"mcp_card":false,"api_catalog":false,"a2a_card":false},"any_capability":false,"ai_refused":["gptbot","claudebot","perplexity"],"ai_thin":["oaisearch"],"sf":[4,0,0,0,0,0,1,0,0],"rings":{"mcp_declared":false,"mcp_live":false,"mcp_auth":false,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":3,"fh":"www.linkedin.com","fp_other_host":[],"fp":"e0d414696e54024d","fp_bits":{"oaisearch":21}},"derive_v":5,"frame":"top10k","shard":1},{"v":1,"kind":"crawlcheck-census-record","run":18,"domain":"googlevideo.com","rank":19,"measured_at":"2026-11-13T18:49:51.543Z","home":{"browser":{"status":526,"content_type":"text/plain","bytes":16,"sha256":"4c543fe8a4d739c6f622b04303ac9e9d9fafbf00da950fc916ad826da2c0cdac","final_host":"googlevideo.com","final_path":"/","ms":54,"identity":"Mobile browser","html":false},"gptbot":{"status":526,"content_type":"text/plain","bytes":16,"sha256":"4c543fe8a4d739c6f622b04303ac9e9d9fafbf00da950fc916ad826da2c0cdac","final_host":"googlevideo.com","final_path":"/","ms":46,"identity":"GPTBot","html":false},"claudebot":{"status":526,"content_type":"text/plain","bytes":16,"sha256":"4c543fe8a4d739c6f622b04303ac9e9d9fafbf00da950fc916ad826da2c0cdac","final_host":"googlevideo.com","final_path":"/","ms":32,"identity":"ClaudeBot","html":false},"oaisearch":{"status":526,"content_type":"text/plain","bytes":16,"sha256":"4c543fe8a4d739c6f622b04303ac9e9d9fafbf00da950fc916ad826da2c0cdac","final_host":"googlevideo.com","final_path":"/","ms":28,"identity":"OAI-SearchBot","html":false},"perplexity":{"status":526,"content_type":"text/plain","bytes":16,"sha256":"4c543fe8a4d739c6f622b04303ac9e9d9fafbf00da950fc916ad826da2c0cdac","final_host":"googlevideo.com","final_path":"/","ms":31,"identity":"PerplexityBot","html":false}},"robots":{"status":526,"content_type":"text/plain","bytes":16,"sha256":"4c543fe8a4d739c6f622b04303ac9e9d9fafbf00da950fc916ad826da2c0cdac","final_host":"googlevideo.com","final_path":"/robots.txt","ms":54,"html":false,"text":null,"agents":null},"sitemap":null,"files":null,"derived":{"reachable":false,"home_class":"no_site","robots_served":false,"robots_sitemap":false,"content_signal":false,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":[],"blocks_any_ai":false,"files":{"llms_txt":false,"agents_md":false,"agent_skills":false,"mcp_card":false,"api_catalog":false,"a2a_card":false},"any_capability":false,"ai_refused":[],"ai_thin":[],"rings":{"mcp_declared":false,"mcp_live":false,"mcp_auth":false,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":3,"fh":"googlevideo.com","fp_other_host":[]},"derive_v":5,"frame":"top10k","shard":2},{"v":1,"kind":"crawlcheck-census-record","run":18,"domain":"akamaiedge.net","rank":20,"measured_at":"2026-11-13T18:50:46.493Z","home":{"browser":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"akamaiedge.net","final_path":"/","ms":8,"identity":"Mobile browser","html":false},"gptbot":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"akamaiedge.net","final_path":"/","ms":1,"identity":"GPTBot","html":false},"claudebot":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"akamaiedge.net","final_path":"/","ms":1,"identity":"ClaudeBot","html":false},"oaisearch":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"akamaiedge.net","final_path":"/","ms":2,"identity":"OAI-SearchBot","html":false},"perplexity":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"akamaiedge.net","final_path":"/","ms":1,"identity":"PerplexityBot","html":false}},"robots":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"akamaiedge.net","final_path":"/robots.txt","ms":0,"html":false,"text":null,"agents":null},"sitemap":null,"files":null,"derived":{"reachable":false,"home_class":"no_site","robots_served":false,"robots_sitemap":false,"content_signal":false,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":[],"blocks_any_ai":false,"files":{"llms_txt":false,"agents_md":false,"agent_skills":false,"mcp_card":false,"api_catalog":false,"a2a_card":false},"any_capability":false,"ai_refused":[],"ai_thin":[],"rings":{"mcp_declared":false,"mcp_live":false,"mcp_auth":false,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":3,"fh":"akamaiedge.net","fp_other_host":[]},"derive_v":5,"frame":"top10k","shard":3},{"v":1,"kind":"crawlcheck-census-record","run":18,"domain":"googletagmanager.com","rank":21,"measured_at":"2026-11-13T18:51:40.579Z","home":{"browser":{"status":404,"content_type":"text/html","bytes":1561,"sha256":"23926e9185d8d43c02807a838ffb373cc1977726094a4e46807c66ada9dd7660","final_host":"googletagmanager.com","final_path":"/","ms":67,"identity":"Mobile browser","html":true},"gptbot":{"status":404,"content_type":"text/html","bytes":1561,"sha256":"23926e9185d8d43c02807a838ffb373cc1977726094a4e46807c66ada9dd7660","final_host":"googletagmanager.com","final_path":"/","ms":61,"identity":"GPTBot","html":true},"claudebot":{"status":404,"content_type":"text/html","bytes":1561,"sha256":"23926e9185d8d43c02807a838ffb373cc1977726094a4e46807c66ada9dd7660","final_host":"googletagmanager.com","final_path":"/","ms":38,"identity":"ClaudeBot","html":true},"oaisearch":{"status":404,"content_type":"text/html","bytes":1561,"sha256":"23926e9185d8d43c02807a838ffb373cc1977726094a4e46807c66ada9dd7660","final_host":"googletagmanager.com","final_path":"/","ms":34,"identity":"OAI-SearchBot","html":true},"perplexity":{"status":404,"content_type":"text/html","bytes":1561,"sha256":"23926e9185d8d43c02807a838ffb373cc1977726094a4e46807c66ada9dd7660","final_host":"googletagmanager.com","final_path":"/","ms":33,"identity":"PerplexityBot","html":true}},"robots":{"status":404,"content_type":"text/html","bytes":1571,"sha256":"6adee83a3eba87d30fdca3997d27097697f2767805f222a0f78c3f04392004d2","final_host":"googletagmanager.com","final_path":"/robots.txt","ms":71,"html":true,"text":null,"agents":null},"sitemap":null,"files":null,"derived":{"reachable":false,"home_class":"other","robots_served":false,"robots_sitemap":false,"content_signal":false,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":[],"blocks_any_ai":false,"files":{"llms_txt":false,"agents_md":false,"agent_skills":false,"mcp_card":false,"api_catalog":false,"a2a_card":false},"any_capability":false,"ai_refused":[],"ai_thin":[],"rings":{"mcp_declared":false,"mcp_live":false,"mcp_auth":false,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":3,"fh":"googletagmanager.com","fp_other_host":[]},"derive_v":5,"frame":"top10k","shard":4},{"v":1,"kind":"crawlcheck-census-record","run":18,"domain":"office.com","rank":22,"measured_at":"2026-11-13T18:52:33.408Z","home":{"browser":{"status":200,"content_type":"text/html","bytes":313578,"sha256":"f7c1b47a1633f2db6ea5eb3504acf066a3d2cb2386525835896799a0e5faf6aa","final_host":"m365.cloud.microsoft","final_path":"/","ms":383,"identity":"Mobile browser","html":true,"fp":"db47656b07e3c220"},"gptbot":{"status":200,"content_type":"text/html","bytes":334533,"sha256":"142ca9e9c09741165e5e9ad02597a65ad6b18100d9a5362e9c460ddcf397a875","final_host":"m365.cloud.microsoft","final_path":"/","ms":596,"identity":"GPTBot","html":true,"fp":"db47656b27c34a60"},"claudebot":{"status":200,"content_type":"text/html","bytes":334533,"sha256":"142ca9e9c09741165e5e9ad02597a65ad6b18100d9a5362e9c460ddcf397a875","final_host":"m365.cloud.microsoft","final_path":"/","ms":597,"identity":"ClaudeBot","html":true,"fp":"db47656b27c34a60"},"oaisearch":{"status":200,"content_type":"text/html","bytes":334533,"sha256":"142ca9e9c09741165e5e9ad02597a65ad6b18100d9a5362e9c460ddcf397a875","final_host":"m365.cloud.microsoft","final_path":"/","ms":597,"identity":"OAI-SearchBot","html":true,"fp":"db47656b27c34a60"},"perplexity":{"status":200,"content_type":"text/html","bytes":334533,"sha256":"142ca9e9c09741165e5e9ad02597a65ad6b18100d9a5362e9c460ddcf397a875","final_host":"m365.cloud.microsoft","final_path":"/","ms":597,"identity":"PerplexityBot","html":true,"fp":"db47656b27c34a60"}},"robots":{"status":200,"content_type":"text/html","bytes":60407,"sha256":"46ffbfb0c12203ab03c9783a5ecbdd658009f9833a78fc67d208c654a8dbb2fe","final_host":"login.microsoftonline.com","final_path":"/common/oauth2/v2.0/authorize","ms":795,"html":true,"text":null,"agents":null},"sitemap":{"status":200,"content_type":"text/html","bytes":60353,"sha256":"20dec9eec53b69057b8ba41b64a1e4965673f9ec2c3af89c558000326cdcb164","final_host":"login.microsoftonline.com","final_path":"/common/oauth2/v2.0/authorize","ms":565,"url":"https://office.com/sitemap.xml","declared":false,"xml":false},"files":{"llms_txt":{"status":200,"content_type":"text/html","bytes":60343,"sha256":"3750a11d8f34eab4408260f5565313b98af4ef69a488e1ccd718baa1411490ac","final_host":"login.microsoftonline.com","final_path":"/common/oauth2/v2.0/authorize","ms":580,"path":"/llms.txt","html":true,"json":null,"present":false},"agents_md":{"status":200,"content_type":"text/html","bytes":60442,"sha256":"a2f7487d667d2e277f4ab832296dc98aac49a0e872465d8c5f9ebab0326aeb5e","final_host":"login.microsoftonline.com","final_path":"/common/oauth2/v2.0/authorize","ms":556,"path":"/agents.md","html":true,"json":null,"present":false},"agent_skills":{"status":200,"content_type":"text/html","bytes":60639,"sha256":"b8a59b906aa944a22c3fe28c5f8d5bcca839f7d20d4953093d351b5e106fc2d3","final_host":"login.microsoftonline.com","final_path":"/common/oauth2/v2.0/authorize","ms":498,"path":"/.well-known/agent-skills/index.json","html":true,"json":null,"present":false},"mcp_card":{"status":200,"content_type":"text/html","bytes":60640,"sha256":"f0f2de1a9928be525b2e9a2bb6d7908f5c2c72cb5aff8684d2ee860f13708d8c","final_host":"login.microsoftonline.com","final_path":"/common/oauth2/v2.0/authorize","ms":569,"path":"/.well-known/mcp/server-card.json","html":true,"json":null,"present":false},"api_catalog":{"status":200,"content_type":"text/html","bytes":60657,"sha256":"7c5de1f2c4f03022f0755f31a409c70f1495d7960b9e0d28b8ecf53bea2ad130","final_host":"login.microsoftonline.com","final_path":"/common/oauth2/v2.0/authorize","ms":622,"path":"/.well-known/api-catalog","html":true,"json":null,"present":false},"a2a_card":{"status":200,"content_type":"text/html","bytes":60844,"sha256":"aa8b66effef9fb193dfe1f714df43dfa29e38c5cdec8ab89fc3c63a5cec5a141","final_host":"login.microsoftonline.com","final_path":"/common/oauth2/v2.0/authorize","ms":170,"path":"/.well-known/agent-card.json","html":true,"json":null,"present":false}},"rings":{"v":1,"candidates":[],"mcp":[],"a2a":null,"webmcp":{"model_context":false,"bridge":false,"mcp_json":false,"link_rel":false,"any":false},"handshakes":0,"get_path":{"status":200,"content_type":"text/html","ms":226,"error":null},"get_host":{"status":530,"content_type":"text/plain","ms":9,"error":null}},"sf":[4,0,0,0,0,0,2,16,0],"derived":{"reachable":true,"home_class":"answers","robots_served":false,"robots_sitemap":false,"content_signal":false,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":[],"blocks_any_ai":false,"files":{"llms_txt":false,"agents_md":false,"agent_skills":false,"mcp_card":false,"api_catalog":false,"a2a_card":false},"any_capability":false,"ai_refused":[],"ai_thin":[],"sf":[4,0,0,0,0,0,2,16,0],"rings":{"mcp_declared":false,"mcp_live":false,"mcp_auth":false,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":3,"fh":"m365.cloud.microsoft","fp_other_host":[],"fp":"db47656b07e3c220","fp_bits":{"gptbot":5,"claudebot":5,"oaisearch":5,"perplexity":5}},"derive_v":5,"frame":"top10k","shard":5},{"v":1,"kind":"crawlcheck-census-record","run":18,"domain":"live.com","rank":23,"measured_at":"2026-11-13T18:53:15.625Z","home":{"browser":{"status":200,"content_type":"text/html","bytes":13857,"sha256":"ab163c0c7a83b6232a09054b3a216a893f0d259194490320addf936fea702b44","final_host":"outlook.live.com","final_path":"/mail","ms":695,"identity":"Mobile browser","html":true},"gptbot":{"status":200,"content_type":"text/html","bytes":14315,"sha256":"ed990cb9c657277639df370818a926a3d4d146564882993186eb3f6437b4d54e","final_host":"outlook.live.com","final_path":"/mail","ms":762,"identity":"GPTBot","html":true},"claudebot":{"status":200,"content_type":"text/html","bytes":14316,"sha256":"31a65c22ceb52b0ca9cb0f35b4c4b7d724a3482cd9f869d44e5dc12c3d0e06b4","final_host":"outlook.live.com","final_path":"/mail","ms":893,"identity":"ClaudeBot","html":true},"oaisearch":{"status":200,"content_type":"text/html","bytes":14315,"sha256":"8d5df99486452f7b65053efd03486dce963b1ec4f5e7d4111c6a18e856ad2375","final_host":"outlook.live.com","final_path":"/mail","ms":636,"identity":"OAI-SearchBot","html":true},"perplexity":{"status":200,"content_type":"text/html","bytes":14316,"sha256":"a6885ec41a9fdc8bee4b190736b53d9c4c043cd4d27f75971d37da2687818381","final_host":"outlook.live.com","final_path":"/mail","ms":717,"identity":"PerplexityBot","html":true}},"robots":{"status":200,"content_type":"text/html","bytes":14316,"sha256":"69d45aca2cac65268733b4028fb7415594a6105c0e491824a0d37289a876e28e","final_host":"outlook.live.com","final_path":"/mail","ms":588,"html":true,"text":null,"agents":null},"sitemap":{"status":200,"content_type":"text/html","bytes":14316,"sha256":"d71e5642b50056ae672990372cd23dc53c5094ac3232fc62eb1ec5a9a742f081","final_host":"outlook.live.com","final_path":"/mail","ms":217,"url":"https://live.com/sitemap.xml","declared":false,"xml":false},"files":{"llms_txt":{"status":200,"content_type":"text/html","bytes":14316,"sha256":"d7b6bd6b3ebf6d69def85423d349c74288dff9e5117b1390e6d021f54c0daab0","final_host":"outlook.live.com","final_path":"/mail","ms":169,"path":"/llms.txt","html":true,"json":null,"present":false},"agents_md":{"status":200,"content_type":"text/html","bytes":14100,"sha256":"846c0e11df58e8d8a753498cf70e8b328ea5eb84bbf7190a7a356a664e43c613","final_host":"outlook.live.com","final_path":"/mail","ms":173,"path":"/agents.md","html":true,"json":null,"present":false},"agent_skills":{"status":200,"content_type":"text/html","bytes":14313,"sha256":"64f225bdcef0f04b628aab0bffbdcec4430fb074830806cdb6cb5847733a03e2","final_host":"outlook.live.com","final_path":"/mail","ms":263,"path":"/.well-known/agent-skills/index.json","html":true,"json":null,"present":false},"mcp_card":{"status":200,"content_type":"text/html","bytes":14315,"sha256":"f5feae9a5828400416a8210afcbbe516e045c983aa12abf39c2a7720d79b3ddb","final_host":"outlook.live.com","final_path":"/mail","ms":183,"path":"/.well-known/mcp/server-card.json","html":true,"json":null,"present":false},"api_catalog":{"status":200,"content_type":"text/html","bytes":14315,"sha256":"75bb2eb06ed985abf7fdbbc5c48626ac73c15d12aa1d847bab53271ae0bcde9f","final_host":"outlook.live.com","final_path":"/mail","ms":184,"path":"/.well-known/api-catalog","html":true,"json":null,"present":false},"a2a_card":{"status":200,"content_type":"text/html","bytes":14315,"sha256":"38305f63e97f66f354de1c88ceab727328cee783b2dcc4d00aeb0a23a7a118f6","final_host":"outlook.live.com","final_path":"/mail","ms":253,"path":"/.well-known/agent-card.json","html":true,"json":null,"present":false}},"rings":{"v":1,"candidates":[],"mcp":[],"a2a":null,"webmcp":{"model_context":false,"bridge":false,"mcp_json":false,"link_rel":false,"any":false},"handshakes":0,"get_path":{"status":200,"content_type":"text/html","ms":245,"error":null},"get_host":{"status":200,"content_type":"text/html","ms":261,"error":null}},"sf":[4,0,0,0,0,0,1,0,0],"derived":{"reachable":true,"home_class":"answers","robots_served":false,"robots_sitemap":false,"content_signal":false,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":[],"blocks_any_ai":false,"files":{"llms_txt":false,"agents_md":false,"agent_skills":false,"mcp_card":false,"api_catalog":false,"a2a_card":false},"any_capability":false,"ai_refused":[],"ai_thin":[],"sf":[4,0,0,0,0,0,1,0,0],"rings":{"mcp_declared":false,"mcp_live":false,"mcp_auth":false,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":3,"fh":"outlook.live.com","fp_other_host":[]},"derive_v":5,"frame":"top10k","shard":6},{"v":1,"kind":"crawlcheck-census-record","run":18,"domain":"akadns.net","rank":24,"measured_at":"2026-11-13T18:55:41.951Z","home":{"browser":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"akadns.net","final_path":"/","ms":5,"identity":"Mobile browser","html":false},"gptbot":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"akadns.net","final_path":"/","ms":0,"identity":"GPTBot","html":false},"claudebot":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"akadns.net","final_path":"/","ms":1,"identity":"ClaudeBot","html":false},"oaisearch":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"akadns.net","final_path":"/","ms":1,"identity":"OAI-SearchBot","html":false},"perplexity":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"akadns.net","final_path":"/","ms":0,"identity":"PerplexityBot","html":false}},"robots":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"akadns.net","final_path":"/robots.txt","ms":1,"html":false,"text":null,"agents":null},"sitemap":null,"files":null,"derived":{"reachable":false,"home_class":"no_site","robots_served":false,"robots_sitemap":false,"content_signal":false,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":[],"blocks_any_ai":false,"files":{"llms_txt":false,"agents_md":false,"agent_skills":false,"mcp_card":false,"api_catalog":false,"a2a_card":false},"any_capability":false,"ai_refused":[],"ai_thin":[],"rings":{"mcp_declared":false,"mcp_live":false,"mcp_auth":false,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":3,"fh":"akadns.net","fp_other_host":[]},"derive_v":5,"frame":"top10k","shard":7},{"v":1,"kind":"crawlcheck-census-record","run":18,"domain":"hicloudcam.com","rank":25,"measured_at":"2026-11-13T18:46:27.088Z","home":{"browser":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"hicloudcam.com","final_path":"/","ms":475,"identity":"Mobile browser","html":false},"gptbot":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"hicloudcam.com","final_path":"/","ms":0,"identity":"GPTBot","html":false},"claudebot":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"hicloudcam.com","final_path":"/","ms":1,"identity":"ClaudeBot","html":false},"oaisearch":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"hicloudcam.com","final_path":"/","ms":0,"identity":"OAI-SearchBot","html":false},"perplexity":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"hicloudcam.com","final_path":"/","ms":0,"identity":"PerplexityBot","html":false}},"robots":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"hicloudcam.com","final_path":"/robots.txt","ms":1,"html":false,"text":null,"agents":null},"sitemap":null,"files":null,"derived":{"reachable":false,"home_class":"no_site","robots_served":false,"robots_sitemap":false,"content_signal":false,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":[],"blocks_any_ai":false,"files":{"llms_txt":false,"agents_md":false,"agent_skills":false,"mcp_card":false,"api_catalog":false,"a2a_card":false},"any_capability":false,"ai_refused":[],"ai_thin":[],"rings":{"mcp_declared":false,"mcp_live":false,"mcp_auth":false,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":3,"fh":"hicloudcam.com","fp_other_host":[]},"derive_v":5,"frame":"top10k","shard":0},{"v":1,"kind":"crawlcheck-census-record","run":18,"domain":"amazon.com","rank":26,"measured_at":"2026-11-13T18:48:18.168Z","home":{"browser":{"status":202,"content_type":"text/html","bytes":2012,"sha256":"c4f601e3000b53898e8cde24994afb906991991db53293055bfeebdfd9a2015b","final_host":"www.amazon.com","final_path":"/","ms":796,"identity":"Mobile browser","html":true},"gptbot":{"status":202,"content_type":"text/html","bytes":2012,"sha256":"6ec7b46cda8e1d5b7e8c7dc9351076d8bd147acd13cac825e08a49399042b858","final_host":"www.amazon.com","final_path":"/","ms":777,"identity":"GPTBot","html":true},"claudebot":{"status":202,"content_type":"text/html","bytes":2012,"sha256":"bc814ce1173fc4847afb3b956880a9f80ce291d1211097c16bfd3513911be9c7","final_host":"www.amazon.com","final_path":"/","ms":778,"identity":"ClaudeBot","html":true},"oaisearch":{"status":202,"content_type":"text/html","bytes":2012,"sha256":"46dd8c0f341d7bdb8c1f30aeeaffe0f82e5ab8954d60aec6844ba3a6ead6c96f","final_host":"www.amazon.com","final_path":"/","ms":858,"identity":"OAI-SearchBot","html":true},"perplexity":{"status":202,"content_type":"text/html","bytes":2012,"sha256":"b635fdfc2609bbf6e1324c90daca4621e64a5e2287a8e11bda611001b7d775b5","final_host":"www.amazon.com","final_path":"/","ms":762,"identity":"PerplexityBot","html":true}},"robots":{"status":200,"content_type":"text/plain","bytes":7947,"sha256":"74fa27e28cd9d80bdd155ffecf3a033ea2e195a80c83d6fa46acb20665c3cb4d","final_host":"www.amazon.com","final_path":"/robots.txt","ms":1073,"html":false,"text":"User-agent: *\nDisallow: /exec/obidos/account-access-login\nDisallow: /exec/obidos/change-style\nDisallow: /exec/obidos/flex-sign-in\nDisallow: /exec/obidos/handle-buy-box\nDisallow: /exec/obidos/tg/cm/member/\nDisallow: /gp/aw/help/id=sss\nDisallow: /gp/cart\nDisallow: /gp/flex\nDisallow: /gp/product/e-mail-friend\nDisallow: /gp/product/product-availability\nDisallow: /gp/product/rate-this-item\nDisallow: /gp/sign-in\nDisallow: /gp/reader\nDisallow: /gp/sitbv3/reader\nDisallow: /gp/richpub/syltguides/create\nDisallow: /gp/gfix\nDisallow: /gp/associations/wizard.html\nDisallow: /gp/dmusic/order\nDisallow: /gp/legacy-handle-buy-box.html\nDisallow: /gp/aws/ssop\nDisallow: /gp/yourstore\nDisallow: /gp/gift-central/organizer/add-wishlist\nDisallow: /gp/vote\nDisallow: /gp/voting/\nDisallow: /gp/music/wma-pop-up\nDisallow: /gp/customer-images\nDisallow: /gp/richpub/listmania/createpipeline\nDisallow: /gp/content-form\nDisallow: /gp/pdp/invitation/invite\nDisallow: /gp/customer-reviews/common/du\nDisallow: /gp/customer-reviews/write-a-review.html\nDisallow: /gp/associations/wizard.html\nDisallow: /gp/music/clipserve\nDisallow: /gp/customer-media/upload\nDisallow: /gp/history\nDisallow: /gp/item-dispatch\nDisallow: /gp/dmusic/order/handle-buy-box.html\nDisallow: /gp/recsradio\nDisallow: /gp/slredirect\nDisallow: /dp/shipping/\nDisallow: /dp/twister-update/\nDisallow: /dp/manual-submit/\nDisallow: /dp/e-mail-friend/\nDisallow: /dp/product-availability/\nDisallow: /dp/rate-this-item/\nDisallow: /gp/registry/wishlist/*/reserve\nDisallow: /gp/structured-ratings/actions/get-experience.html\nDisallow: /gp/twitter/\nDisallow: /ap/signin\nDisallow: /gp/registry/wishlist/\nDisallow: /wishlist/\nAllow: /wishlist/universal*\nAllow: /wishlist/vendor-button*\nAllow: /wishlist/get-button*\nDisallow: /gp/wishlist/\nAllow: /gp/wishlist/universal*\nAllow: /gp/wishlist/vendor-button*\nAllow: /gp/wishlist/ipad-install*\nDisallow: /registry/wishlist/\nDisallow: /review/common/du\nDisallow: /gp/registry/search.html\nDisallow: /product-reviews/B0069IY63Y\nDisallow: /gp/orc/rml/\nDisallow: */gcrnsts\nDisallow: /gp/gc/widget\nDisallow: /gp/dmusic/mp3/player\nDisallow: /gp/entity-alert/external\nDisallow: /gp/customer-reviews/dynamic/sims-box\nDisallow: /review/dynamic/sims-box\nDisallow: /gp/redirect.html\nDisallow: /gp/twister/ajaxv2\nDisallow: /ss/twister/ajax\nDisallow: /b?*node=7454917011\nDisallow: /b?*node=7454927011\nDisallow: /b?*node=7454939011\nDisallow: /b?*node=7454898011\nDisallow: /gp/customer-media/actions/delete/\nDisallow: /gp/customer-media/actions/edit-caption/\nDisallow: /gp/dmusic/\nAllow: /gp/dmusic/promotions/PrimeMusic\nAllow: /gp/dmusic/promotions/AmazonMusicUnlimited\nDisallow: /gp/offer-listing/\nDisallow: /b?*node=9052533011\nDisallow: /lm/R1XIHQVKXSKBNJ\nDisallow: /lm/R3HQ5WJSZK6QSO\nDisallow: /surprise/\nDisallow: /local/ajax/\nDisallow: /gp/socialmedia/giveaways\nDisallow: /gp/b2b-rd\nDisallow: /gp/aw/so.html\nDisallow: /reviews/iframe\nDisallow: /gp/switch-language\nDisallow: /ga/p/\nDisallow: /gp/profile/\nDisallow: /giveaway/host/setup/\nDisallow: /ss/customer-reviews/lighthouse/\nDisallow: /ospublishing/story/*\nDisallow: /gp/aw/ol/\nDisallow: /gp/promotion/\nDisallow: /hz/leaderboard/top-reviewers/\nDisallow: /creatorhub\nDisallow: /creatorhub/*\nDisallow: /slp/s$\nDisallow: /-/\nAllow: /-/es/\nAllow: /-/en$\nAllow: /-/zh_TW/\nAllow: /-/zh_TW$\nAllow: /-/he/\nAllow: /-/he$\nDisallow: /hz/help/contact/*/message/$\nDisallow: /gp/aw/shoppingAids/\nDisallow: /rss/people/*/reviews\nDisallow: /gp/pdp/rss/*/reviews\nDisallow: /gp/cdp/member-reviews/\nDisallow: /gp/aw/cr/\nAllow: /gp/offer-listing/B000\nAllow: /gp/offer-listing/9000\nDisallow: /gp/aag\nAllow: /gp/aag/main?*seller=ABVFEJU8LS620\nDisallow: /gp/pdp/profile/\nDisallow: /gp/help/customer/express/c2c/\nDisallow: /slp/*/b$\nDisallow: /hz/contact-us/ajax/initiate-trusted-contact/\nDisallow: /hp/video/\nDisallow: /gp/video/api\nDisallow: /gp/video/settings\nDisallow: /gp/video/library\nDisallow: /gp/video/watchlist\nDisallow: /gp/video/mystuff\nDisallow: /gp/video/profiles\nDisallow: /gp/video/search\nDisallow: /gp/video/auth\nDisallow: /gp/video/salp/i\nDisallow: /icdp/\nDisallow: /shoptheshow/*?cat=\nDisallow: /shoptheshow/*&cat=\n\nUser-agent: EtaoSpider\nDisallow: /\n\nUser-agent: GPTBot\nDisallow: /\n\n\nUser-agent: CCBot\nDisallow: /\n\nUser-Agent: PerplexityBot\nDisallow: /\n\nUser-agent: Google-Extended\nDisallow: /\n\nUser-agent: GoogleAgent-Mariner\nDisallow: /\n\nUser-agent: GoogleAgent-Shopping\nDisallow: /\n\nUser-agent: ClaudeBot\nDisallow: /\n\nUser-agent: meta-externalagent\nDisallow: /\n\nUser-agent: Bytespider\nDisallow: /\n\nUser-agent: Scrapy\nDisallow: /\n\nUser-agent: PetalBot\nDisallow: /\n\nUser-agent: Devin\nDisallow: /\n\nUser-agent: omgili\nDisallow: /\n\nUser-agent: AI2Bot\nDisallow: /\n\nUser-agent: Gemini-Deep-Research\nDisallow: /\n\nUser-agent: PanguBot\nDisallow: /\n\nUser-agent: MistralAI-User\nDisallow: /\n\nUser-agent: Diffbot\nDisallow: /\n\nUser-agent: Sidetrade indexer bot\nDisallow: /\n\nUser-agent: Google-NotebookLM\nDisallow: /\n\nUser-agent: DuckAssistBot\nDisallow: /\n\nUser-agent: Copilot\nDisallow: /\n\nUser-agent: Ai2Bot-Dolma\nDisallow: /\n\nUser-agent: Google-CloudVertexBot\nDisallow: /\n\nUser-agent: meta-externalfetcher\nDisallow: /\n\nUser-agent: iaskspider\nDisallow: /\n\nUser-agent: wpbot\nDisallow: /\n\nUser-agent: aiHitBot\nDisallow: /\n\nUser-agent: cohere-ai\nDisallow: /\n\nUser-agent: FriendlyCrawler\nDisallow: /\n\nUser-agent: img2dataset\nDisallow: /\n\nUser-agent: VelenPublicWebCrawler\nDisallow: /\n\nUser-agent: YouBot\nDisallow: /\n\nUser-agent: Brightbot\nDisallow: /\n\nUser-agent: Claude-User\nDisallow: /\n\nUser-agent: ClaudeBot\nDisallow: /\n\nUser-agent: Claude-SearchBot\nDisallow: /\n\nUser-agent: Perplexity-User\nDisallow: /\n\nUser-agent: ISSCyberRiskCrawler\nDisallow: /\n\nUser-agent: CopilotNative\nDisallow: /\n\nUser-agent: CopilotSapphire\nDisallow: /\n\nUser-agent: imgproxy\nDisallow: /\n\nUser-agent: SemrushBot-SWA\nDisallow: /\n\nUser-agent: ChatGPT-User\nDisallow: /\n\nUser-agent: OAI-SearchBot\nDisallow: /\n\nUser-agent: Timpibot\nDisallow: /\n\nUser-agent: Thinkbot\nDisallow: /\n\nUser-agent: AddSearchBot\nDisallow: /\n\nUser-agent: AI2Bot-DeepResearchEval\nDisallow: /\n\nUser-agent: Andibot\nDisallow: /\n\nUser-agent: Anomura\nDisallow: /\n\nUser-agent: atlassian-bot\nDisallow: /\n\nUser-agent: BuddyBot\nDisallow: /\n\nUser-agent: Channel3Bot\nDisallow: /\n\nUser-agent: ChatGLM-Spider\nDisallow: /\n\nUser-agent: Claude-Web\nDisallow: /\n\nUser-agent: Cloudflare-AutoRAG\nDisallow: /\n\nUser-agent: CloudflareBrowserRenderingCrawler\nDisallow: /\n\nUser-agent: cohere-training-data-crawler\nDisallow: /\n\nUser-agent: Cotoyogi\nDisallow: /\n\nUser-agent: Crawl4AI\nDisallow: /\n\nUser-agent: Datenbank Crawler\nDisallow: /\n\nUser-agent: DeepSeekBot\nDisallow: /\n\nUser-agent: Echobot Bot\nDisallow: /\n\nUser-agent: EchoboxBot\nDisallow: /\n\nUser-agent: factset_spyderbot\nDisallow: /\n\nUser-agent: Google-Firebase\nDisallow: /\n\nUser-agent: GoogleOther\nDisallow: /\n\nUser-agent: Grok-DeepSearch\nDisallow: /\n\nUser-agent: GrokBot\nDisallow: /\n\nUser-agent: iAskBot\nDisallow: /\n\nUser-agent: IbouBot\nDisallow: /\n\nUser-agent: ICC-Crawler\nDisallow: /\n\nUser-agent: ImagesiftBot\nDisallow: /\n\nUser-agent: imageSpider\nDisallow: /\n\nUser-agent: Kangaroo Bot\nDisallow: /\n\nUser-agent: KlaviyoAIBot\nDisallow: /\n\nUser-agent: KunatoCrawler\nDisallow: /\n\nUser-agent: laion-huggingface-processor\nDisallow: /\n\nUser-agent: Linguee Bot\nDisallow: /\n\nUser-agent: LinkupBot\nDisallow: /\n\nUser-agent: Manus-User\nDisallow: /\n\nUser-agent: meta-webindexer\nDisallow: /\n\nUser-agent: netEstate Imprint Crawler\nDisallow: /\n\nUser-agent: Panscient\nDisallow: /\n\nUser-agent: panscient.com\nDisallow: /\n\nUser-agent: PhindBot\nDisallow: /\n\nUser-agent: Poggio-Citations\nDisallow: /\n\nUser-agent: QualifiedBot\nDisallow: /\n\nUser-agent: SBIntuitionsBot\nDisallow: /\n\nUser-agent: ShapBot\nDisallow: /\n\nUser-agent: TavilyBot\nDisallow: /\n\nUser-agent: TerraCotta\nDisallow: /\n\nUser-agent: TwinAgent\nDisallow: /\n\nUser-agent: webzio-extended\nDisallow: /\n\nUser-agent: WRTNBot\nDisallow: /\n\nUser-agent: xAI-Grok\nDisallow: /\n\nUser-agent: YaK\nDisallow: /\n\nUser-agent: ZanistaBot\nDisallow: /","agents":{"GPTBot":false,"OAI-SearchBot":false,"ChatGPT-User":false,"ClaudeBot":false,"Claude-SearchBot":false,"PerplexityBot":false,"Google-Extended":false,"Applebot-Extended":true,"CCBot":false,"Bytespider":false,"Meta-ExternalAgent":false,"Amazonbot":true}},"sitemap":{"status":null,"error":"timeout","ms":8000,"url":"https://amazon.com/sitemap.xml","declared":false,"xml":false},"files":{"llms_txt":{"status":404,"content_type":"text/html","bytes":2296,"sha256":"34c92fb4c6351b94e5cc16f68bb483b0fcda57ee7c43887214005b87d1888d3a","final_host":"www.amazon.com","final_path":"/llms.txt","ms":505,"path":"/llms.txt","html":true,"json":null,"present":false},"agents_md":{"status":404,"content_type":"text/html","bytes":2296,"sha256":"34c92fb4c6351b94e5cc16f68bb483b0fcda57ee7c43887214005b87d1888d3a","final_host":"www.amazon.com","final_path":"/agents.md","ms":1000,"path":"/agents.md","html":true,"json":null,"present":false},"agent_skills":{"status":404,"content_type":"text/html","bytes":2296,"sha256":"34c92fb4c6351b94e5cc16f68bb483b0fcda57ee7c43887214005b87d1888d3a","final_host":"www.amazon.com","final_path":"/.well-known/agent-skills/index.json","ms":1019,"path":"/.well-known/agent-skills/index.json","html":true,"json":null,"present":false},"mcp_card":{"status":404,"content_type":"text/html","bytes":2296,"sha256":"34c92fb4c6351b94e5cc16f68bb483b0fcda57ee7c43887214005b87d1888d3a","final_host":"www.amazon.com","final_path":"/.well-known/mcp/server-card.json","ms":513,"path":"/.well-known/mcp/server-card.json","html":true,"json":null,"present":false},"api_catalog":{"status":404,"content_type":"text/html","bytes":2296,"sha256":"34c92fb4c6351b94e5cc16f68bb483b0fcda57ee7c43887214005b87d1888d3a","final_host":"www.amazon.com","final_path":"/.well-known/api-catalog","ms":1011,"path":"/.well-known/api-catalog","html":true,"json":null,"present":false},"a2a_card":{"status":404,"content_type":"text/html","bytes":2296,"sha256":"34c92fb4c6351b94e5cc16f68bb483b0fcda57ee7c43887214005b87d1888d3a","final_host":"www.amazon.com","final_path":"/.well-known/agent-card.json","ms":510,"path":"/.well-known/agent-card.json","html":true,"json":null,"present":false}},"rings":{"v":1,"candidates":[],"mcp":[],"a2a":null,"webmcp":{"model_context":false,"bridge":false,"mcp_json":false,"link_rel":false,"any":false},"handshakes":0,"get_path":{"status":200,"content_type":null,"ms":516,"error":null},"get_host":{"status":530,"content_type":"text/plain","ms":10,"error":null}},"sf":[4,0,0,0,0,0,1,0,0],"derived":{"reachable":false,"home_class":"other","robots_served":false,"robots_sitemap":false,"content_signal":false,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":[],"blocks_any_ai":false,"files":{"llms_txt":false,"agents_md":false,"agent_skills":false,"mcp_card":false,"api_catalog":false,"a2a_card":false},"any_capability":false,"ai_refused":[],"ai_thin":[],"sf":[4,0,0,0,0,0,1,0,0],"rings":{"mcp_declared":false,"mcp_live":false,"mcp_auth":false,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":3,"fh":"www.amazon.com","fp_other_host":[]},"derive_v":5,"frame":"top10k","shard":1},{"v":1,"kind":"crawlcheck-census-record","run":18,"domain":"gtld-servers.net","rank":27,"measured_at":"2026-11-13T18:49:51.543Z","home":{"browser":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"gtld-servers.net","final_path":"/","ms":29,"identity":"Mobile browser","html":false},"gptbot":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"gtld-servers.net","final_path":"/","ms":1,"identity":"GPTBot","html":false},"claudebot":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"gtld-servers.net","final_path":"/","ms":0,"identity":"ClaudeBot","html":false},"oaisearch":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"gtld-servers.net","final_path":"/","ms":1,"identity":"OAI-SearchBot","html":false},"perplexity":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"gtld-servers.net","final_path":"/","ms":1,"identity":"PerplexityBot","html":false}},"robots":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"gtld-servers.net","final_path":"/robots.txt","ms":2,"html":false,"text":null,"agents":null},"sitemap":null,"files":null,"derived":{"reachable":false,"home_class":"no_site","robots_served":false,"robots_sitemap":false,"content_signal":false,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":[],"blocks_any_ai":false,"files":{"llms_txt":false,"agents_md":false,"agent_skills":false,"mcp_card":false,"api_catalog":false,"a2a_card":false},"any_capability":false,"ai_refused":[],"ai_thin":[],"rings":{"mcp_declared":false,"mcp_live":false,"mcp_auth":false,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":3,"fh":"gtld-servers.net","fp_other_host":[]},"derive_v":5,"frame":"top10k","shard":2},{"v":1,"kind":"crawlcheck-census-record","run":18,"domain":"azure.com","rank":28,"measured_at":"2026-11-13T18:50:46.493Z","home":{"browser":{"status":200,"content_type":"text/html","bytes":463665,"sha256":"f1e8849a5607dcd78d3d7d41edd1902b3b7bec11dd9700559863275bdbabf02a","final_host":"azure.microsoft.com","final_path":"/en-us","ms":996,"identity":"Mobile browser","html":true,"fp":"da5efe5944800e3c"},"gptbot":{"status":200,"content_type":"text/html","bytes":463664,"sha256":"ae3e80f500d54252e19d991b79c26649c989771aafdf8de1014a5d9fdd24a676","final_host":"azure.microsoft.com","final_path":"/en-us","ms":3431,"identity":"GPTBot","html":true,"fp":"da5efe5944800e3c"},"claudebot":{"status":200,"content_type":"text/html","bytes":463741,"sha256":"48e348bdc1e17f0a92b35dd42a88070b6d05374d2d22036ca9a81c786d193fa3","final_host":"azure.microsoft.com","final_path":"/en-us","ms":3178,"identity":"ClaudeBot","html":true,"fp":"da5efe5944800f3c"},"oaisearch":{"status":200,"content_type":"text/html","bytes":463741,"sha256":"d4db3a2c063f07141ed7ec7da69138c5e9d3f7b8774588374ab8cbffd855fd83","final_host":"azure.microsoft.com","final_path":"/en-us","ms":4467,"identity":"OAI-SearchBot","html":true,"fp":"da5efe5944800e3c"},"perplexity":{"status":200,"content_type":"text/html","bytes":463664,"sha256":"5615539cf7b875b77b5dff5de611bfab16bd40a12a573c946fad81572c8cf9a1","final_host":"azure.microsoft.com","final_path":"/en-us","ms":6234,"identity":"PerplexityBot","html":true,"fp":"da5efe5944800f3c"}},"robots":{"status":200,"content_type":"text/html","bytes":463741,"sha256":"5c08d1e73abe3f28339079a35417e87a644c6fa371e3af63a06904530692cb84","final_host":"azure.microsoft.com","final_path":"/en-us","ms":569,"html":true,"text":null,"agents":null},"sitemap":{"status":200,"content_type":"text/html","bytes":463741,"sha256":"e22e63716e35a9250681cdc279b057fb367b540dbb9b45073937c5604f9a40ad","final_host":"azure.microsoft.com","final_path":"/en-us","ms":248,"url":"https://azure.com/sitemap.xml","declared":false,"xml":false},"files":{"llms_txt":{"status":404,"content_type":"text/html","bytes":94649,"sha256":"18f56bb66013dfbf60965ff0c515841b3b01bf885aa69ba501f6fbbb7d9fa7c8","final_host":"azure.microsoft.com","final_path":"/llms.txt","ms":590,"path":"/llms.txt","html":true,"json":null,"present":false},"agents_md":{"status":200,"content_type":"text/html","bytes":463664,"sha256":"e28291b6b46f52cc60555e601d955ee20567c72a00a708e05b068d5e0761ee11","final_host":"azure.microsoft.com","final_path":"/en-us","ms":709,"path":"/agents.md","html":true,"json":null,"present":false},"agent_skills":{"status":200,"content_type":"text/html","bytes":463664,"sha256":"5fa0441c07b87182a0223af9aced54e5b863048f7e8cf4610b95e63666428ac8","final_host":"azure.microsoft.com","final_path":"/en-us","ms":309,"path":"/.well-known/agent-skills/index.json","html":true,"json":null,"present":false},"mcp_card":{"status":200,"content_type":"text/html","bytes":463664,"sha256":"0030378524783d9a5e91fb148b770f2616e45d0cfaf819c3c556806903b2c0a8","final_host":"azure.microsoft.com","final_path":"/en-us","ms":376,"path":"/.well-known/mcp/server-card.json","html":true,"json":null,"present":false},"api_catalog":{"status":200,"content_type":"text/html","bytes":463741,"sha256":"4a38901fd72ffea2441494445d732a85d977458c2dc3a2b9944f3bbbaab79127","final_host":"azure.microsoft.com","final_path":"/en-us","ms":510,"path":"/.well-known/api-catalog","html":true,"json":null,"present":false},"a2a_card":{"status":200,"content_type":"text/html","bytes":463741,"sha256":"a9c4924b233ea9703fc3533458b13e01eb59c1432748d81cb46e61e01808ee1b","final_host":"azure.microsoft.com","final_path":"/en-us","ms":582,"path":"/.well-known/agent-card.json","html":true,"json":null,"present":false}},"rings":{"v":1,"candidates":[],"mcp":[],"a2a":null,"webmcp":{"model_context":false,"bridge":false,"mcp_json":false,"link_rel":false,"any":false},"handshakes":0,"get_path":{"status":200,"content_type":"text/html","ms":399,"error":null},"get_host":{"status":530,"content_type":"text/plain","ms":25,"error":null}},"sf":[4,0,0,0,0,0,4,0,0],"derived":{"reachable":true,"home_class":"answers","robots_served":false,"robots_sitemap":false,"content_signal":false,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":[],"blocks_any_ai":false,"files":{"llms_txt":false,"agents_md":false,"agent_skills":false,"mcp_card":false,"api_catalog":false,"a2a_card":false},"any_capability":false,"ai_refused":[],"ai_thin":[],"sf":[4,0,0,0,0,0,4,0,0],"rings":{"mcp_declared":false,"mcp_live":false,"mcp_auth":false,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":3,"fh":"azure.microsoft.com","fp_other_host":[],"fp":"da5efe5944800e3c","fp_bits":{"gptbot":0,"claudebot":1,"oaisearch":0,"perplexity":1}},"derive_v":5,"frame":"top10k","shard":3},{"v":1,"kind":"crawlcheck-census-record","run":18,"domain":"github.com","rank":29,"measured_at":"2026-11-13T18:51:40.579Z","home":{"browser":{"status":200,"content_type":"text/html","bytes":526331,"sha256":"44cd220312256b32699cc09300a6dd46a67097807e344aaac6a72071d29cbae0","final_host":"github.com","final_path":"/","ms":41,"identity":"Desktop browser (device re-check)","html":true,"fp":"0da570e962416fa7"},"gptbot":{"status":200,"content_type":"text/html","bytes":525842,"sha256":"017ab32fef145701498f41d8a28f7adcbc14a7bc8eb8c88c112aba1f67f34bec","final_host":"github.com","final_path":"/","ms":160,"identity":"GPTBot","html":true,"fp":"09b770e94061c6d5"},"claudebot":{"status":200,"content_type":"text/html","bytes":526331,"sha256":"9dda324c0e2ec2997977f1cdd6850853e223796c9bf15b4d2f122c9d9b3d509d","final_host":"github.com","final_path":"/","ms":99,"identity":"ClaudeBot","html":true,"fp":"0da570e962416fa7"},"oaisearch":{"status":200,"content_type":"text/html","bytes":526331,"sha256":"9af686bd56dcf5ade81ba2cd807b748c45ac4162ab80677e002df4678270cfec","final_host":"github.com","final_path":"/","ms":121,"identity":"OAI-SearchBot","html":true,"fp":"0da570e962416fa7"},"perplexity":{"status":200,"content_type":"text/html","bytes":526331,"sha256":"6da0f057ff692ebdce3333fc75aa8b3e91b27bc2200a57594b26e0a32c6f7abc","final_host":"github.com","final_path":"/","ms":112,"identity":"PerplexityBot","html":true,"fp":"0da570e962416fa7"},"browser_mobile":{"status":200,"content_type":"text/html","bytes":525997,"sha256":"b34dc14971db4b11cea48cd90faa84c777752c3d56f515d8442087895d4b6f86","final_host":"github.com","final_path":"/","ms":333,"identity":"Mobile browser","html":true,"fp":"19b770e96261c697"}},"robots":{"status":200,"content_type":"text/plain","bytes":29821,"sha256":"8b8e46edf4593a7e23fbf207472b2b56601bfae79eff6d9e17f990a6db39d5d3","final_host":"github.com","final_path":"/robots.txt","ms":123,"html":false,"text":"# If you would like to crawl GitHub contact us via https://support.github.com?tags=dotcom-robots\n# We also provide an extensive API: https://docs.github.com\nSitemap: https://github.com/sitemap_marketing.xml\n\nUser-agent: GPTBot\nUser-agent: OAI-SearchBot\nUser-agent: ChatGPT-User\nUser-agent: ClaudeBot\nUser-agent: Claude-SearchBot\nUser-agent: Claude-User\nUser-agent: PerplexityBot\nUser-agent: Perplexity-User\nCrawl-delay: 1\nAllow: /*?tab=achievements&achievement=*\nAllow: /marketplace/*?tab=readme$\nAllow: /marketplace/*?tab=transparency$\nAllow: /enterprise/contact?*\nAllow: /enterprise/contact/data-residency?*\nAllow: /features/codespaces?*\nAllow: /features/copilot/plans?*\nAllow: /security/advanced-security/demo?*\nAllow: /security/contact-sales?*\nAllow: /security/plans?*\nAllow: /topics/*?q=*\nAllow: /topics/*?*&q=*\nAllow: /orgs/*/repositories?q=*\nAllow: /orgs/*/repositories?*&q=*\nDisallow: /account-login\nDisallow: /copilot/\nDisallow: /copilot/c/\nDisallow: /ekansa/Open-Context-Data\nDisallow: /ekansa/opencontext-*\nDisallow: /Explodingstuff/\nDisallow: /gist/\nDisallow: /search/advanced\nDisallow: /search$\nDisallow: /*/*/pulse_committer_data.json$\nDisallow: /*/*/pulse_committer_data.json?*\nDisallow: /*/archive/\nDisallow: /*/blame/\nDisallow: /*/cache/\nDisallow: /*/raw/\nDisallow: /*/tree/\nDisallow: */tarball/\nDisallow: */zipball/\nDisallow: /.git/\nDisallow: */.git/\nDisallow: /*.git$\nDisallow: /*.atom$\nDisallow: /*report-abuse?report=*\nDisallow: /*/*/commits/\nDisallow: /marketplace/*/*?tab=readme$\nDisallow: /marketplace/*/*?tab=transparency$\nDisallow: /marketplace/*?*?tab=readme$\nDisallow: /marketplace/*?*?tab=transparency$\nDisallow: /marketplace/actions?tab=readme$\nDisallow: /marketplace/actions?tab=transparency$\nDisallow: /marketplace/free-trials?tab=readme$\nDisallow: /marketplace/free-trials?tab=transparency$\nDisallow: /marketplace/manage?tab=readme$\nDisallow: /marketplace/manage?tab=transparency$\nDisallow: /marketplace/models?tab=readme$\nDisallow: /marketplace/models?tab=transparency$\nDisallow: /marketplace/new?tab=readme$\nDisallow: /marketplace/new?tab=transparency$\nDisallow: /marketplace/publishers?tab=readme$\nDisallow: /marketplace/publishers?tab=transparency$\nDisallow: /marketplace/recently-added?tab=readme$\nDisallow: /marketplace/recently-added?tab=transparency$\nDisallow: /topics/*/*?q=*\nDisallow: /topics/*/*?*&q=*\nDisallow: /topics/*?*?q=*\nDisallow: /topics/*?*?*&q=*\nDisallow: /orgs/*/*/repositories?q=*\nDisallow: /orgs/*/*/repositories?*&q=*\nDisallow: /orgs/*?*/repositories?q=*\nDisallow: /orgs/*?*/repositories?*&q=*\nDisallow: /topics/*?source=*&q=*\nDisallow: /topics/*?*&source=*&q=*\nDisallow: /topics/*?q=*&source=*\nDisallow: /topics/*?*&q=*&source=*\nDisallow: /topics/*?ref_cta=*&q=*\nDisallow: /topics/*?*&ref_cta=*&q=*\nDisallow: /topics/*?q=*&ref_cta=*\nDisallow: /topics/*?*&q=*&ref_cta=*\nDisallow: /topics/*?plan=*&q=*\nDisallow: /topics/*?*&plan=*&q=*\nDisallow: /topics/*?q=*&plan=*\nDisallow: /topics/*?*&q=*&plan=*\nDisallow: /topics/*?return_to=*&q=*\nDisallow: /topics/*?*&return_to=*&q=*\nDisallow: /topics/*?q=*&return_to=*\nDisallow: /topics/*?*&q=*&return_to=*\nDisallow: /topics/*?ref_loc=*&q=*\nDisallow: /topics/*?*&ref_loc=*&q=*\nDisallow: /topics/*?q=*&ref_loc=*\nDisallow: /topics/*?*&q=*&ref_loc=*\nDisallow: /topics/*?setup_organization=*&q=*\nDisallow: /topics/*?*&setup_organization=*&q=*\nDisallow: /topics/*?q=*&setup_organization=*\nDisallow: /topics/*?*&q=*&setup_organization=*\nDisallow: /topics/*?source_repo=*&q=*\nDisallow: /topics/*?*&source_repo=*&q=*\nDisallow: /topics/*?q=*&source_repo=*\nDisallow: /topics/*?*&q=*&source_repo=*\nDisallow: /topics/*?ref_page=*&q=*\nDisallow: /topics/*?*&ref_page=*&q=*\nDisallow: /topics/*?q=*&ref_page=*\nDisallow: /topics/*?*&q=*&ref_page=*\nDisallow: /topics/*?referrer=*&q=*\nDisallow: /topics/*?*&referrer=*&q=*\nDisallow: /topics/*?q=*&referrer=*\nDisallow: /topics/*?*&q=*&referrer=*\nDisallow: /topics/*?report=*&q=*\nDisallow: /topics/*?*&report=*&q=*\nDisallow: /topics/*?q=*&report=*\nDisallow: /topics/*?*&q=*&report=*\nDisallow: /topics/*?author=*&q=*\nDisallow: /topics/*?*&author=*&q=*\nDisallow: /topics/*?q=*&author=*\nDisallow: /topics/*?*&q=*&author=*\nDisallow: /topics/*?since=*&q=*\nDisallow: /topics/*?*&since=*&q=*\nDisallow: /topics/*?q=*&since=*\nDisallow: /topics/*?*&q=*&since=*\nDisallow: /topics/*?until=*&q=*\nDisallow: /topics/*?*&until=*&q=*\nDisallow: /topics/*?q=*&until=*\nDisallow: /topics/*?*&q=*&until=*\nDisallow: /topics/*?tab=*&q=*\nDisallow: /topics/*?*&tab=*&q=*\nDisallow: /topics/*?q=*&tab=*\nDisallow: /topics/*?*&q=*&tab=*\nDisallow: /orgs/*/repositories?source=*&q=*\nDisallow: /orgs/*/repositories?*&source=*&q=*\nDisallow: /orgs/*/repositories?q=*&source=*\nDisallow: /orgs/*/repositories?*&q=*&source=*\nDisallow: /orgs/*/repositories?ref_cta=*&q=*\nDisallow: /orgs/*/repositories?*&ref_cta=*&q=*\nDisallow: /orgs/*/repositories?q=*&ref_cta=*\nDisallow: /orgs/*/repositories?*&q=*&ref_cta=*\nDisallow: /orgs/*/repositories?plan=*&q=*\nDisallow: /orgs/*/repositories?*&plan=*&q=*\nDisallow: /orgs/*/repositories?q=*&plan=*\nDisallow: /orgs/*/repositories?*&q=*&plan=*\nDisallow: /orgs/*/repositories?return_to=*&q=*\nDisallow: /orgs/*/repositories?*&return_to=*&q=*\nDisallow: /orgs/*/repositories?q=*&return_to=*\nDisallow: /orgs/*/repositories?*&q=*&return_to=*\nDisallow: /orgs/*/repositories?ref_loc=*&q=*\nDisallow: /orgs/*/repositories?*&ref_loc=*&q=*\nDisallow: /orgs/*/repositories?q=*&ref_loc=*\nDisallow: /orgs/*/repositories?*&q=*&ref_loc=*\nDisallow: /orgs/*/repositories?setup_organization=*&q=*\nDisallow: /orgs/*/repositories?*&setup_organization=*&q=*\nDisallow: /orgs/*/repositories?q=*&setup_organization=*\nDisallow: /orgs/*/repositories?*&q=*&setup_organization=*\nDisallow: /orgs/*/repositories?source_repo=*&q=*\nDisallow: /orgs/*/repositories?*&source_repo=*&q=*\nDisallow: /orgs/*/repositories?q=*&source_repo=*\nDisallow: /orgs/*/repositories?*&q=*&source_repo=*\nDisallow: /orgs/*/repositories?ref_page=*&q=*\nDisallow: /orgs/*/repositories?*&ref_page=*&q=*\nDisallow: /orgs/*/repositories?q=*&ref_page=*\nDisallow: /orgs/*/repositories?*&q=*&ref_page=*\nDisallow: /orgs/*/repositories?referrer=*&q=*\nDisallow: /orgs/*/repositories?*&referrer=*&q=*\nDisallow: /orgs/*/repositories?q=*&referrer=*\nDisallow: /orgs/*/repositories?*&q=*&referrer=*\nDisallow: /orgs/*/repositories?report=*&q=*\nDisallow: /orgs/*/repositories?*&report=*&q=*\nDisallow: /orgs/*/repositories?q=*&report=*\nDisallow: /orgs/*/repositories?*&q=*&report=*\nDisallow: /orgs/*/repositories?author=*&q=*\nDisallow: /orgs/*/repositories?*&author=*&q=*\nDisallow: /orgs/*/repositories?q=*&author=*\nDisallow: /orgs/*/repositories?*&q=*&author=*\nDisallow: /orgs/*/repositories?since=*&q=*\nDisallow: /orgs/*/repositories?*&since=*&q=*\nDisallow: /orgs/*/repositories?q=*&since=*\nDisallow: /orgs/*/repositories?*&q=*&since=*\nDisallow: /orgs/*/repositories?until=*&q=*\nDisallow: /orgs/*/repositories?*&until=*&q=*\nDisallow: /orgs/*/repositories?q=*&until=*\nDisallow: /orgs/*/repositories?*&q=*&until=*\nDisallow: /orgs/*/repositories?tab=*&q=*\nDisallow: /orgs/*/repositories?*&tab=*&q=*\nDisallow: /orgs/*/repositories?q=*&tab=*\nDisallow: /orgs/*/repositories?*&q=*&tab=*\nDisallow: /biztools$\nDisallow: /biztools?*\nDisallow: /biztools/*\nDisallow: /businesses$\nDisallow: /businesses?*\nDisallow: /businesses/*\nDisallow: /enterprises$\nDisallow: /enterprises?*\nDisallow: /enterprises/*\nDisallow: /organizations$\nDisallow: /organizations?*\nDisallow: /organizations/*\nDisallow: /settings$\nDisallow: /settings?*\nDisallow: /settings/*\nDisallow: /stafftools$\nDisallow: /stafftools?*\nDisallow: /stafftools/*\nDisallow: /users/*/billing$\nDisallow: /users/*/billing?*\nDisallow: /users/*/billing/*\nDisallow: /writing$\nDisallow: /writing?*\nDisallow: /writing/*\nDisallow: /*/*/branches$\nDisallow: /*/*/branches?*\nDisallow: /*/*/branches/*\nDisallow: /*/*/comments$\nDisallow: /*/*/comments?*\nDisallow: /*/*/comments/*\nDisallow: /*/*/compare$\nDisallow: /*/*/compare?*\nDisallow: /*/*/compare/*\nDisallow: /*/*/contributors$\nDisallow: /*/*/contributors?*\nDisallow: /*/*/contributors/*\nDisallow: /*/*/contributors_list$\nDisallow: /*/*/contributors_list?*\nDisallow: /*/*/contributors_list/*\nDisallow: /*/*/contributors-list$\nDisallow: /*/*/contributors-list?*\nDisallow: /*/*/contributors-list/*\nDisallow: /*/*/download$\nDisallow: /*/*/download?*\nDisallow: /*/*/download/*\nDisallow: /*/*/forks$\nDisallow: /*/*/forks?*\nDisallow: /*/*/forks/*\nDisallow: /*/*/graphs$\nDisallow: /*/*/graphs?*\nDisallow: /*/*/graphs/*\nDisallow: /*/*/issues/new$\nDisallow: /*/*/issues/new?*\nDisallow: /*/*/issues/new/*\nDisallow: /*/*/issues/search$\nDisallow: /*/*/issues/search?*\nDisallow: /*/*/issues/search/*\nDisallow: /*/*/milestones/new$\nDisallow: /*/*/milestones/new?*\nDisallow: /*/*/milestones/new/*\nDisallow: /*/*/network$\nDisallow: /*/*/network?*\nDisallow: /*/*/network/*\nDisallow: /*/*/projects$\nDisallow: /*/*/projects?*\nDisallow: /*/*/projects/*\nDisallow: /*/*/pulse$\nDisallow: /*/*/pulse?*\nDisallow: /*/*/pulse/*\nDisallow: /*/*/pulse-new$\nDisallow: /*/*/pulse-new?*\nDisallow: /*/*/pulse-new/*\nDisallow: /*/*/pulse_committer_data$\nDisallow: /*/*/pulse_committer_data?*\nDisallow: /*/*/pulse_committer_data/*\nDisallow: /*/*/pulse_diffstat_summary$\nDisallow: /*/*/pulse_diffstat_summary?*\nDisallow: /*/*/pulse_diffstat_summary/*\nDisallow: /*/*/revisions$\nDisallow: /*/*/revisions?*\nDisallow: /*/*/revisions/*\nDisallow: /*/*/stargazers$\nDisallow: /*/*/stargazers?*\nDisallow: /*/*/stargazers/*\nDisallow: /*/*/tags$\nDisallow: /*/*/tags?*\nDisallow: /*/*/tags/*\nDisallow: /*/*/watchers$\nDisallow: /*/*/watchers?*\nDisallow: /*/*/watchers/*\nDisallow: /*?source=*\nDisallow: /*&source=*\nDisallow: /*?ref_cta=*\nDisallow: /*&ref_cta=*\nDisallow: /*?plan=*\nDisallow: /*&plan=*\nDisallow: /*?return_to=*\nDisallow: /*&return_to=*\nDisallow: /*?ref_loc=*\nDisallow: /*&ref_loc=*\nDisallow: /*?setup_organization=*\nDisallow: /*&setup_organization=*\nDisallow: /*?source_repo=*\nDisallow: /*&source_repo=*\nDisallow: /*?ref_page=*\nDisallow: /*&ref_page=*\nDisallow: /*?referrer=*\nDisallow: /*&referrer=*\nDisallow: /*?report=*\nDisallow: /*&report=*\nDisallow: /*?author=*\nDisallow: /*&author=*\nDisallow: /*?since=*\nDisallow: /*&since=*\nDisallow: /*?until=*\nDisallow: /*&until=*\nDisallow: /*?tab=*\nDisallow: /*&tab=*\nDisallow: /*?q=*\nDisallow: /*&q=*\n\nUser-agent: Bytespider\nDisallow: /\n\nUser-agent: AdsBot-Google\nUser-agent: AdsBot-Google-Mobile\nDisallow: /account-login\nDisallow: /copilot/\nDisallow: /copilot/c/\nDisallow: /ekansa/Open-Context-Data\nDisallow: /ekansa/opencontext-*\nDisallow: /Explodingstuff/\nDisallow: /gist/\nDisallow: /search/advanced\nDisallow: /search$\nDisallow: /*/*/pulse_committer_data.json$\nDisallow: /*/*/pulse_committer_data.json?*\nDisallow: /*/archive/\nDisallow: /*/blame/\nDisallow: /*/cache/\nDisallow: /*/raw/\nDisallow: /*/tree/\nDisallow: */tarball/\nDisallow: */zipball/\nDisallow: /.git/\nDisallow: */.git/\nDisallow: /*.git$\nDisallow: /*.atom$\nDisallow: /*report-abuse?report=*\nDisallow: /*/*/commits/\nDisallow: /marketplace/*/*?tab=readme$\nDisallow: /marketplace/*/*?tab=transparency$\nDisallow: /marketplace/*?*?tab=readme$\nDisallow: /marketplace/*?*?tab=transparency$\nDisallow: /marketplace/actions?tab=readme$\nDisallow: /marketplace/actions?tab=transparency$\nDisallow: /marketplace/free-trials?tab=readme$\nDisallow: /marketplace/free-trials?tab=transparency$\nDisallow: /marketplace/manage?tab=readme$\nDisallow: /marketplace/manage?tab=transparency$\nDisallow: /marketplace/models?tab=readme$\nDisallow: /marketplace/models?tab=transparency$\nDisallow: /marketplace/new?tab=readme$\nDisallow: /marketplace/new?tab=transparency$\nDisallow: /marketplace/publishers?tab=readme$\nDisallow: /marketplace/publishers?tab=transparency$\nDisallow: /marketplace/recently-added?tab=readme$\nDisallow: /marketplace/recently-added?tab=transparency$\nDisallow: /topics/*/*?q=*\nDisallow: /topics/*/*?*&q=*\nDisallow: /topics/*?*?q=*\nDisallow: /topics/*?*?*&q=*\nDisallow: /orgs/*/*/repositories?q=*\nDisallow: /orgs/*/*/repositories?*&q=*\nDisallow: /orgs/*?*/repositories?q=*\nDisallow: /orgs/*?*/repositories?*&q=*\nDisallow: /topics/*?source=*&q=*\nDisallow: /topics/*?*&source=*&q=*\nDisallow: /topics/*?q=*&source=*\nDisallow: /topics/*?*&q=*&source=*\nDisallow: /topics/*?ref_cta=*&q=*\nDisallow: /topics/*?*&ref_cta=*&q=*\nDisallow: /topics/*?q=*&ref_cta=*\nDisallow: /topics/*?*&q=*&ref_cta=*\nDisallow: /topics/*?plan=*&q=*\nDisallow: /topics/*?*&plan=*&q=*\nDisallow: /topics/*?q=*&plan=*\nDisallow: /topics/*?*&q=*&plan=*\nDisallow: /topics/*?return_to=*&q=*\nDisallow: /topics/*?*&return_to=*&q=*\nDisallow: /topics/*?q=*&return_to=*\nDisallow: /topics/*?*&q=*&return_to=*\nDisallow: /topics/*?ref_loc=*&q=*\nDisallow: /topics/*?*&ref_loc=*&q=*\nDisallow: /topics/*?q=*&ref_loc=*\nDisallow: /topics/*?*&q=*&ref_loc=*\nDisallow: /topics/*?setup_organization=*&q=*\nDisallow: /topics/*?*&setup_organization=*&q=*\nDisallow: /topics/*?q=*&setup_organization=*\nDisallow: /topics/*?*&q=*&setup_organization=*\nDisallow: /topics/*?source_repo=*&q=*\nDisallow: /topics/*?*&source_repo=*&q=*\nDisallow: /topics/*?q=*&source_repo=*\nDisallow: /topics/*?*&q=*&source_repo=*\nDisallow: /topics/*?ref_page=*&q=*\nDisallow: /topics/*?*&ref_page=*&q=*\nDisallow: /topics/*?q=*&ref_page=*\nDisallow: /topics/*?*&q=*&ref_page=*\nDisallow: /topics/*?referrer=*&q=*\nDisallow: /topics/*?*&referrer=*&q=*\nDisallow: /topics/*?q=*&referrer=*\nDisallow: /topics/*?*&q=*&referrer=*\nDisallow: /topics/*?report=*&q=*\nDisallow: /topics/*?*&report=*&q=*\nDisallow: /topics/*?q=*&report=*\nDisallow: /topics/*?*&q=*&report=*\nDisallow: /topics/*?author=*&q=*\nDisallow: /topics/*?*&author=*&q=*\nDisallow: /topics/*?q=*&author=*\nDisallow: /topics/*?*&q=*&author=*\nDisallow: /topics/*?since=*&q=*\nDisallow: /topics/*?*&since=*&q=*\nDisallow: /topics/*?q=*&since=*\nDisallow: /topics/*?*&q=*&since=*\nDisallow: /topics/*?until=*&q=*\nDisallow: /topics/*?*&until=*&q=*\nDisallow: /topics/*?q=*&until=*\nDisallow: /topics/*?*&q=*&until=*\nDisallow: /topics/*?tab=*&q=*\nDisallow: /topics/*?*&tab=*&q=*\nDisallow: /topics/*?q=*&tab=*\nDisallow: /topics/*?*&q=*&tab=*\nDisallow: /orgs/*/repositories?source=*&q=*\nDisallow: /orgs/*/repositories?*&source=*&q=*\nDisallow: /orgs/*/repositories?q=*&source=*\nDisallow: /orgs/*/repositories?*&q=*&source=*\nDisallow: /orgs/*/repositories?ref_cta=*&q=*\nDisallow: /orgs/*/repositories?*&ref_cta=*&q=*\nDisallow: /orgs/*/repositories?q=*&ref_cta=*\nDisallow: /orgs/*/repositories?*&q=*&ref_cta=*\nDisallow: /orgs/*/repositories?plan=*&q=*\nDisallow: /orgs/*/repositories?*&plan=*&q=*\nDisallow: /orgs/*/repositories?q=*&plan=*\nDisallow: /orgs/*/repositories?*&q=*&plan=*\nDisallow: /orgs/*/repositories?return_to=*&q=*\nDisallow: /orgs/*/repositories?*&return_to=*&q=*\nDisallow: /orgs/*/repositories?q=*&return_to=*\nDisallow: /orgs/*/repositories?*&q=*&return_to=*\nDisallow: /orgs/*/repositories?ref_loc=*&q=*\nDisallow: /orgs/*/repositories?*&ref_loc=*&q=*\nDisallow: /orgs/*/repositories?q=*&ref_loc=*\nDisallow: /orgs/*/repositories?*&q=*&ref_loc=*\nDisallow: /orgs/*/repositories?setup_organization=*&q=*\nDisallow: /orgs/*/repositories?*&setup_organization=*&q=*\nDisallow: /orgs/*/repositories?q=*&setup_organization=*\nDisallow: /orgs/*/repositories?*&q=*&setup_organization=*\nDisallow: /orgs/*/repositories?source_repo=*&q=*\nDisallow: /orgs/*/repositories?*&source_repo=*&q=*\nDisallow: /orgs/*/repositories?q=*&source_repo=*\nDisallow: /orgs/*/repositories?*&q=*&source_repo=*\nDisallow: /orgs/*/repositories?ref_page=*&q=*\nDisallow: /orgs/*/repositories?*&ref_page=*&q=*\nDisallow: /orgs/*/repositories?q=*&ref_page=*\nDisallow: /orgs/*/repositories?*&q=*&ref_page=*\nDisallow: /orgs/*/repositories?referrer=*&q=*\nDisallow: /orgs/*/repositories?*&referrer=*&q=*\nDisallow: /orgs/*/repositories?q=*&referrer=*\nDisallow: /orgs/*/repositories?*&q=*&referrer=*\nDisallow: /orgs/*/repositories?report=*&q=*\nDisallow: /orgs/*/repositories?*&report=*&q=*\nDisallow: /orgs/*/repositories?q=*&report=*\nDisallow: /orgs/*/repositories?*&q=*&report=*\nDisallow: /orgs/*/repositories?author=*&q=*\nDisallow: /orgs/*/repositories?*&author=*&q=*\nDisallow: /orgs/*/repositories?q=*&author=*\nDisallow: /orgs/*/repositories?*&q=*&author=*\nDisallow: /orgs/*/repositories?since=*&q=*\nDisallow: /orgs/*/repositories?*&since=*&q=*\nDisallow: /orgs/*/repositories?q=*&since=*\nDisallow: /orgs/*/repositories?*&q=*&since=*\nDisallow: /orgs/*/repositories?until=*&q=*\nDisallow: /orgs/*/repositories?*&until=*&q=*\nDisallow: /orgs/*/repositories?q=*&until=*\nDisallow: /orgs/*/repositories?*&q=*&until=*\nDisallow: /orgs/*/repositories?tab=*&q=*\nDisallow: /orgs/*/repositories?*&tab=*&q=*\nDisallow: /orgs/*/repositories?q=*&tab=*\nDisallow: /orgs/*/repositories?*&q=*&tab=*\nDisallow: /biztools$\nDisallow: /biztools?*\nDisallow: /biztools/*\nDisallow: /businesses$\nDisallow: /businesses?*\nDisallow: /businesses/*\nDisallow: /enterprises$\nDisallow: /enterprises?*\nDisallow: /enterprises/*\nDisallow: /organizations$\nDisallow: /organizations?*\nDisallow: /organizations/*\nDisallow: /settings$\nDisallow: /settings?*\nDisallow: /settings/*\nDisallow: /stafftools$\nDisallow: /stafftools?*\nDisallow: /stafftools/*\nDisallow: /users/*/billing$\nDisallow: /users/*/billing?*\nDisallow: /users/*/billing/*\nDisallow: /writing$\nDisallow: /writing?*\nDisallow: /writing/*\nDisallow: /*/*/branches$\nDisallow: /*/*/branches?*\nDisallow: /*/*/branches/*\nDisallow: /*/*/comments$\nDisallow: /*/*/comments?*\nDisallow: /*/*/comments/*\nDisallow: /*/*/compare$\nDisallow: /*/*/compare?*\nDisallow: /*/*/compare/*\nDisallow: /*/*/contributors$\nDisallow: /*/*/contributors?*\nDisallow: /*/*/contributors/*\nDisallow: /*/*/contributors_list$\nDisallow: /*/*/contributors_list?*\nDisallow: /*/*/contributors_list/*\nDisallow: /*/*/contributors-list$\nDisallow: /*/*/contributors-list?*\nDisallow: /*/*/contributors-list/*\nDisallow: /*/*/download$\nDisallow: /*/*/download?*\nDisallow: /*/*/download/*\nDisallow: /*/*/forks$\nDisallow: /*/*/forks?*\nDisallow: /*/*/forks/*\nDisallow: /*/*/graphs$\nDisallow: /*/*/graphs?*\nDisallow: /*/*/graphs/*\nDisallow: /*/*/issues/new$\nDisallow: /*/*/issues/new?*\nDisallow: /*/*/issues/new/*\nDisallow: /*/*/issues/search$\nDisallow: /*/*/issues/search?*\nDisallow: /*/*/issues/search/*\nDisallow: /*/*/milestones/new$\nDisallow: /*/*/milestones/new?*\nDisallow: /*/*/milestones/new/*\nDisallow: /*/*/network$\nDisallow: /*/*/network?*\nDisallow: /*/*/network/*\nDisallow: /*/*/projects$\nDisallow: /*/*/projects?*\nDisallow: /*/*/projects/*\nDisallow: /*/*/pulse$\nDisallow: /*/*/pulse?*\nDisallow: /*/*/pulse/*\nDisallow: /*/*/pulse-new$\nDisallow: /*/*/pulse-new?*\nDisallow: /*/*/pulse-new/*\nDisallow: /*/*/pulse_committer_data$\nDisallow: /*/*/pulse_committer_data?*\nDisallow: /*/*/pulse_committer_data/*\nDisallow: /*/*/pulse_diffstat_summary$\nDisallow: /*/*/pulse_diffstat_summary?*\nDisallow: /*/*/pulse_diffstat_summary/*\nDisallow: /*/*/revisions$\nDisallow: /*/*/revisions?*\nDisallow: /*/*/revisions/*\nDisallow: /*/*/stargazers$\nDisallow: /*/*/stargazers?*\nDisallow: /*/*/stargazers/*\nDisallow: /*/*/tags$\nDisallow: /*/*/tags?*\nDisallow: /*/*/tags/*\nDisallow: /*/*/watchers$\nDisallow: /*/*/watchers?*\nDisallow: /*/*/watchers/*\nDisallow: /*?source=*\nDisallow: /*&source=*\nDisallow: /*?ref_cta=*\nDisallow: /*&ref_cta=*\nDisallow: /*?plan=*\nDisallow: /*&plan=*\nDisallow: /*?return_to=*\nDisallow: /*&return_to=*\nDisallow: /*?ref_loc=*\nDisallow: /*&ref_loc=*\nDisallow: /*?setup_organization=*\nDisallow: /*&setup_organization=*\nDisallow: /*?source_repo=*\nDisallow: /*&source_repo=*\nDisallow: /*?ref_page=*\nDisallow: /*&ref_page=*\nDisallow: /*?referrer=*\nDisallow: /*&referrer=*\nDisallow: /*?report=*\nDisallow: /*&report=*\nDisallow: /*?author=*\nDisallow: /*&author=*\nDisallow: /*?since=*\nDisallow: /*&since=*\nDisallow: /*?until=*\nDisallow: /*&until=*\nDisallow: /*?tab=*\nDisallow: /*&tab=*\nDisallow: /*?q=*\nDisallow: /*&q=*\n\nUser-agent: bingbot\nUser-agent: adidxbot\nUser-agent: BingPreview\nDisallow: /ekansa/Open-Context-Data\nDisallow: /ekansa/opencontext-*\nDisallow: /account-login\nDisallow: */tarball/\nDisallow: */zipball/\nDisallow: /Explodingstuff/\nDisallow: /copilot/\nDisallow: /copilot/c/\n\nUser-agent: baidu\ncrawl-delay: 1\n\nUser-agent: *\nAllow: /*?tab=achievements&achievement=*\nAllow: /marketplace/*?tab=readme$\nAllow: /marketplace/*?tab=transparency$\nAllow: /enterprise/contact?*\nAllow: /enterprise/contact/data-residency?*\nAllow: /features/codespaces?*\nAllow: /features/copilot/plans?*\nAllow: /security/advanced-security/demo?*\nAllow: /security/contact-sales?*\nAllow: /security/plans?*\nAllow: /topics/*?q=*\nAllow: /topics/*?*&q=*\nAllow: /orgs/*/repositories?q=*\nAllow: /orgs/*/repositories?*&q=*\nDisallow: /account-login\nDisallow: /copilot/\nDisallow: /copilot/c/\nDisallow: /ekansa/Open-Context-Data\nDisallow: /ekansa/opencontext-*\nDisallow: /Explodingstuff/\nDisallow: /gist/\nDisallow: /search/advanced\nDisallow: /search$\nDisallow: /*/*/pulse_committer_data.json$\nDisallow: /*/*/pulse_committer_data.json?*\nDisallow: /*/archive/\nDisallow: /*/blame/\nDisallow: /*/cache/\nDisallow: /*/raw/\nDisallow: /*/tree/\nDisallow: */tarball/\nDisallow: */zipball/\nDisallow: /.git/\nDisallow: */.git/\nDisallow: /*.git$\nDisallow: /*.atom$\nDisallow: /*report-abuse?report=*\nDisallow: /*/*/commits/\nDisallow: /marketplace/*/*?tab=readme$\nDisallow: /marketplace/*/*?tab=transparency$\nDisallow: /marketplace/*?*?tab=readme$\nDisallow: /marketplace/*?*?tab=transparency$\nDisallow: /marketplace/actions?tab=readme$\nDisallow: /marketplace/actions?tab=transparency$\nDisallow: /marketplace/free-trials?tab=readme$\nDisallow: /marketplace/free-trials?tab=transparency$\nDisallow: /marketplace/manage?tab=readme$\nDisallow: /marketplace/manage?tab=transparency$\nDisallow: /marketplace/models?tab=readme$\nDisallow: /marketplace/models?tab=transparency$\nDisallow: /marketplace/new?tab=readme$\nDisallow: /marketplace/new?tab=transparency$\nDisallow: /marketplace/publishers?tab=readme$\nDisallow: /marketplace/publishers?tab=transparency$\nDisallow: /marketplace/recently-added?tab=readme$\nDisallow: /marketplace/recently-added?tab=transparency$\nDisallow: /topics/*/*?q=*\nDisallow: /topics/*/*?*&q=*\nDisallow: /topics/*?*?q=*\nDisallow: /topics/*?*?*&q=*\nDisallow: /orgs/*/*/repositories?q=*\nDisallow: /orgs/*/*/repositories?*&q=*\nDisallow: /orgs/*?*/repositories?q=*\nDisallow: /orgs/*?*/repositories?*&q=*\nDisallow: /topics/*?source=*&q=*\nDisallow: /topics/*?*&source=*&q=*\nDisallow: /topics/*?q=*&source=*\nDisallow: /topics/*?*&q=*&source=*\nDisallow: /topics/*?ref_cta=*&q=*\nDisallow: /topics/*?*&ref_cta=*&q=*\nDisallow: /topics/*?q=*&ref_cta=*\nDisallow: /topics/*?*&q=*&ref_cta=*\nDisallow: /topics/*?plan=*&q=*\nDisallow: /topics/*?*&plan=*&q=*\nDisallow: /topics/*?q=*&plan=*\nDisallow: /topics/*?*&q=*&plan=*\nDisallow: /topics/*?return_to=*&q=*\nDisallow: /topics/*?*&return_to=*&q=*\nDisallow: /topics/*?q=*&return_to=*\nDisallow: /topics/*?*&q=*&return_to=*\nDisallow: /topics/*?ref_loc=*&q=*\nDisallow: /topics/*?*&ref_loc=*&q=*\nDisallow: /topics/*?q=*&ref_loc=*\nDisallow: /topics/*?*&q=*&ref_loc=*\nDisallow: /topics/*?setup_organization=*&q=*\nDisallow: /topics/*?*&setup_organization=*&q=*\nDisallow: /topics/*?q=*&setup_organization=*\nDisallow: /topics/*?*&q=*&setup_organization=*\nDisallow: /topics/*?source_repo=*&q=*\nDisallow: /topics/*?*&source_repo=*&q=*\nDisallow: /topics/*?q=*&source_repo=*\nDisallow: /topics/*?*&q=*&source_repo=*\nDisallow: /topics/*?ref_page=*&q=*\nDisallow: /topics/*?*&ref_page=*&q=*\nDisallow: /topics/*?q=*&ref_page=*\nDisallow: /topics/*?*&q=*&ref_page=*\nDisallow: /topics/*?referrer=*&q=*\nDisallow: /topics/*?*&referrer=*&q=*\nDisallow: /topics/*?q=*&referrer=*\nDisallow: /topics/*?*&q=*&referrer=*\nDisallow: /topics/*?report=*&q=*\nDisallow: /topics/*?*&report=*&q=*\nDisallow: /topics/*?q=*&report=*\nDisallow: /topics/*?*&q=*&report=*\nDisallow: /topics/*?author=*&q=*\nDisallow: /topics/*?*&author=*&q=*\nDisallow: /topics/*?q=*&author=*\nDisallow: /topics/*?*&q=*&author=*\nDisallow: /topics/*?since=*&q=*\nDisallow: /topics/*?*&since=*&q=*\nDisallow: /topics/*?q=*&since=*\nDisallow: /topics/*?*&q=*&since=*\nDisallow: /topics/*?until=*&q=*\nDisallow: /topics/*?*&until=*&q=*\nDisallow: /topics/*?q=*&until=*\nDisallow: /topics/*?*&q=*&until=*\nDisallow: /topics/*?tab=*&q=*\nDisallow: /topics/*?*&tab=*&q=*\nDisallow: /topics/*?q=*&tab=*\nDisallow: /topics/*?*&q=*&tab=*\nDisallow: /orgs/*/repositories?source=*&q=*\nDisallow: /orgs/*/repositories?*&source=*&q=*\nDisallow: /orgs/*/repositories?q=*&source=*\nDisallow: /orgs/*/repositories?*&q=*&source=*\nDisallow: /orgs/*/repositories?ref_cta=*&q=*\nDisallow: /orgs/*/repositories?*&ref_cta=*&q=*\nDisallow: /orgs/*/repositories?q=*&ref_cta=*\nDisallow: /orgs/*/repositories?*&q=*&ref_cta=*\nDisallow: /orgs/*/repositories?plan=*&q=*\nDisallow: /orgs/*/repositories?*&plan=*&q=*\nDisallow: /orgs/*/repositories?q=*&plan=*\nDisallow: /orgs/*/repositories?*&q=*&plan=*\nDisallow: /orgs/*/repositories?return_to=*&q=*\nDisallow: /orgs/*/repositories?*&return_to=*&q=*\nDisallow: /orgs/*/repositories?q=*&return_to=*\nDisallow: /orgs/*/repositories?*&q=*&return_to=*\nDisallow: /orgs/*/repositories?ref_loc=*&q=*\nDisallow: /orgs/*/repositories?*&ref_loc=*&q=*\nDisallow: /orgs/*/repositories?q=*&ref_loc=*\nDisallow: /orgs/*/repositories?*&q=*&ref_loc=*\nDisallow: /orgs/*/repositories?setup_organization=*&q=*\nDisallow: /orgs/*/repositories?*&setup_organization=*&q=*\nDisallow: /orgs/*/repositories?q=*&setup_organization=*\nDisallow: /orgs/*/repositories?*&q=*&setup_organization=*\nDisallow: /orgs/*/repositories?source_repo=*&q=*\nDisallow: /orgs/*/repositories?*&source_repo=*&q=*\nDisallow: /orgs/*/repositories?q=*&source_repo=*\nDisallow: /orgs/*/repositories?*&q=*&source_repo=*\nDisallow: /orgs/*/repositories?ref_page=*&q=*\nDisallow: /orgs/*/repositories?*&ref_page=*&q=*\nDisallow: /orgs/*/repositories?q=*&ref_page=*\nDisallow: /orgs/*/repositories?*&q=*&ref_page=*\nDisallow: /orgs/*/repositories?referrer=*&q=*\nDisallow: /orgs/*/repositories?*&referrer=*&q=*\nDisallow: /orgs/*/repositories?q=*&referrer=*\nDisallow: /orgs/*/repositories?*&q=*&referrer=*\nDisallow: /orgs/*/repositories?report=*&q=*\nDisallow: /orgs/*/repositories?*&report=*&q=*\nDisallow: /orgs/*/repositories?q=*&report=*\nDisallow: /orgs/*/repositories?*&q=*&report=*\nDisallow: /orgs/*/repositories?author=*&q=*\nDisallow: /orgs/*/repositories?*&author=*&q=*\nDisallow: /orgs/*/repositories?q=*&author=*\nDisallow: /orgs/*/repositories?*&q=*&author=*\nDisallow: /orgs/*/repositories?since=*&q=*\nDisallow: /orgs/*/repositories?*&since=*&q=*\nDisallow: /orgs/*/repositories?q=*&since=*\nDisallow: /orgs/*/repositories?*&q=*&since=*\nDisallow: /orgs/*/repositories?until=*&q=*\nDisallow: /orgs/*/repositories?*&until=*&q=*\nDisallow: /orgs/*/repositories?q=*&until=*\nDisallow: /orgs/*/repositories?*&q=*&until=*\nDisallow: /orgs/*/repositories?tab=*&q=*\nDisallow: /orgs/*/repositories?*&tab=*&q=*\nDisallow: /orgs/*/repositories?q=*&tab=*\nDisallow: /orgs/*/repositories?*&q=*&tab=*\nDisallow: /biztools$\nDisallow: /biztools?*\nDisallow: /biztools/*\nDisallow: /businesses$\nDisallow: /businesses?*\nDisallow: /businesses/*\nDisallow: /enterprises$\nDisallow: /enterprises?*\nDisallow: /enterprises/*\nDisallow: /organizations$\nDisallow: /organizations?*\nDisallow: /organizations/*\nDisallow: /settings$\nDisallow: /settings?*\nDisallow: /settings/*\nDisallow: /stafftools$\nDisallow: /stafftools?*\nDisallow: /stafftools/*\nDisallow: /users/*/billing$\nDisallow: /users/*/billing?*\nDisallow: /users/*/billing/*\nDisallow: /writing$\nDisallow: /writing?*\nDisallow: /writing/*\nDisallow: /*/*/branches$\nDisallow: /*/*/branches?*\nDisallow: /*/*/branches/*\nDisallow: /*/*/comments$\nDisallow: /*/*/comments?*\nDisallow: /*/*/comments/*\nDisallow: /*/*/compare$\nDisallow: /*/*/compare?*\nDisallow: /*/*/compare/*\nDisallow: /*/*/contributors$\nDisallow: /*/*/contributors?*\nDisallow: /*/*/contributors/*\nDisallow: /*/*/contributors_list$\nDisallow: /*/*/contributors_list?*\nDisallow: /*/*/contributors_list/*\nDisallow: /*/*/contributors-list$\nDisallow: /*/*/contributors-list?*\nDisallow: /*/*/contributors-list/*\nDisallow: /*/*/download$\nDisallow: /*/*/download?*\nDisallow: /*/*/download/*\nDisallow: /*/*/forks$\nDisallow: /*/*/forks?*\nDisallow: /*/*/forks/*\nDisallow: /*/*/graphs$\nDisallow: /*/*/graphs?*\nDisallow: /*/*/graphs/*\nDisallow: /*/*/issues/new$\nDisallow: /*/*/issues/new?*\nDisallow: /*/*/issues/new/*\nDisallow: /*/*/issues/search$\nDisallow: /*/*/issues/search?*\nDisallow: /*/*/issues/search/*\nDisallow: /*/*/milestones/new$\nDisallow: /*/*/milestones/new?*\nDisallow: /*/*/milestones/new/*\nDisallow: /*/*/network$\nDisallow: /*/*/network?*\nDisallow: /*/*/network/*\nDisallow: /*/*/projects$\nDisallow: /*/*/projects?*\nDisallow: /*/*/projects/*\nDisallow: /*/*/pulse$\nDisallow: /*/*/pulse?*\nDisallow: /*/*/pulse/*\nDisallow: /*/*/pulse-new$\nDisallow: /*/*/pulse-new?*\nDisallow: /*/*/pulse-new/*\nDisallow: /*/*/pulse_committer_data$\nDisallow: /*/*/pulse_committer_data?*\nDisallow: /*/*/pulse_committer_data/*\nDisallow: /*/*/pulse_diffstat_summary$\nDisallow: /*/*/pulse_diffstat_summary?*\nDisallow: /*/*/pulse_diffstat_summary/*\nDisallow: /*/*/revisions$\nDisallow: /*/*/revisions?*\nDisallow: /*/*/revisions/*\nDisallow: /*/*/stargazers$\nDisallow: /*/*/stargazers?*\nDisallow: /*/*/stargazers/*\nDisallow: /*/*/tags$\nDisallow: /*/*/tags?*\nDisallow: /*/*/tags/*\nDisallow: /*/*/watchers$\nDisallow: /*/*/watchers?*\nDisallow: /*/*/watchers/*\nDisallow: /*?source=*\nDisallow: /*&source=*\nDisallow: /*?ref_cta=*\nDisallow: /*&ref_cta=*\nDisallow: /*?plan=*\nDisallow: /*&plan=*\nDisallow: /*?return_to=*\nDisallow: /*&return_to=*\nDisallow: /*?ref_loc=*\nDisallow: /*&ref_loc=*\nDisallow: /*?setup_organization=*\nDisallow: /*&setup_organization=*\nDisallow: /*?source_repo=*\nDisallow: /*&source_repo=*\nDisallow: /*?ref_page=*\nDisallow: /*&ref_page=*\nDisallow: /*?referrer=*\nDisallow: /*&referrer=*\nDisallow: /*?report=*\nDisallow: /*&report=*\nDisallow: /*?author=*\nDisallow: /*&author=*\nDisallow: /*?since=*\nDisallow: /*&since=*\nDisallow: /*?until=*\nDisallow: /*&until=*\nDisallow: /*?tab=*\nDisallow: /*&tab=*\nDisallow: /*?q=*\nDisallow: /*&q=*\n","agents":{"GPTBot":true,"OAI-SearchBot":true,"ChatGPT-User":true,"ClaudeBot":true,"Claude-SearchBot":true,"PerplexityBot":true,"Google-Extended":true,"Applebot-Extended":true,"CCBot":true,"Bytespider":false,"Meta-ExternalAgent":true,"Amazonbot":true}},"sitemap":{"status":200,"content_type":"application/xml","bytes":32446,"sha256":"ebdf1e30684d4ee14dc69515bef0381e652011e0c782f5e8caf5882306db10cd","final_host":"github.com","final_path":"/sitemap_marketing.xml","ms":156,"url":"https://github.com/sitemap_marketing.xml","declared":true,"xml":true},"files":{"llms_txt":{"status":200,"content_type":"text/plain","bytes":28658,"sha256":"71d217ec3186152052ca1050932c0e6e48e766d3bc5a1359b50aa140436d297f","final_host":"github.com","final_path":"/llms.txt","ms":79,"path":"/llms.txt","html":false,"json":null,"present":true},"agents_md":{"status":406,"content_type":"text/html","bytes":0,"sha256":"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855","final_host":"github.com","final_path":"/agents.md","ms":90,"path":"/agents.md","html":true,"json":null,"present":false},"agent_skills":{"status":404,"content_type":"application/json","bytes":21,"sha256":"e28bf7d9fcc22cdfde0c7c8f31e3648a4847e7bda9cb69f309f24257eee3dd41","final_host":"github.com","final_path":"/.well-known/agent-skills/index.json","ms":55,"path":"/.well-known/agent-skills/index.json","html":false,"json":null,"present":false},"mcp_card":{"status":404,"content_type":"application/json","bytes":21,"sha256":"e28bf7d9fcc22cdfde0c7c8f31e3648a4847e7bda9cb69f309f24257eee3dd41","final_host":"github.com","final_path":"/.well-known/mcp/server-card.json","ms":54,"path":"/.well-known/mcp/server-card.json","html":false,"json":null,"present":false},"api_catalog":{"status":404,"content_type":"text/html","bytes":224756,"sha256":"29a22e734dbf02a9d8ec5ff964c4cbc81fb26be327ef60db6feaa888c66ac602","final_host":"github.com","final_path":"/.well-known/api-catalog","ms":246,"path":"/.well-known/api-catalog","html":true,"json":null,"present":false},"a2a_card":{"status":404,"content_type":"application/json","bytes":21,"sha256":"e28bf7d9fcc22cdfde0c7c8f31e3648a4847e7bda9cb69f309f24257eee3dd41","final_host":"github.com","final_path":"/.well-known/agent-card.json","ms":151,"path":"/.well-known/agent-card.json","html":false,"json":null,"present":false}},"device_variant":true,"rings":{"v":1,"candidates":[],"mcp":[],"a2a":null,"webmcp":{"model_context":false,"bridge":false,"mcp_json":false,"link_rel":false,"any":false},"handshakes":0,"get_path":{"status":200,"content_type":"text/html","ms":415,"error":null},"get_host":{"status":530,"content_type":"text/plain","ms":8,"error":null}},"sf":[4,0,0,0,0,0,1,0,1],"derived":{"reachable":true,"home_class":"answers","robots_served":true,"robots_sitemap":true,"content_signal":false,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":["Bytespider"],"blocks_any_ai":true,"files":{"llms_txt":true,"agents_md":false,"agent_skills":false,"mcp_card":false,"api_catalog":false,"a2a_card":false},"any_capability":false,"ai_refused":[],"ai_thin":[],"aipref":0,"aipref_measured":true,"sf":[4,0,0,0,0,0,1,0,1],"rings":{"mcp_declared":false,"mcp_live":false,"mcp_auth":false,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":3,"fh":"github.com","fp_other_host":[],"fp":"0da570e962416fa7","fp_bits":{"gptbot":14,"claudebot":0,"oaisearch":0,"perplexity":0}},"derive_v":5,"frame":"top10k","shard":4},{"v":1,"kind":"crawlcheck-census-record","run":18,"domain":"wikipedia.org","rank":30,"measured_at":"2026-11-13T18:52:33.408Z","home":{"browser":{"status":200,"content_type":"text/html","bytes":122506,"sha256":"0d267456014b06bd08d6e50a554fb32abd30407b5f56dee637c26614bac05349","final_host":"www.wikipedia.org","final_path":"/","ms":1320,"identity":"Mobile browser","html":true,"fp":"1f628783c51e34fe"},"gptbot":{"status":200,"content_type":"text/html","bytes":122506,"sha256":"0d267456014b06bd08d6e50a554fb32abd30407b5f56dee637c26614bac05349","final_host":"www.wikipedia.org","final_path":"/","ms":1117,"identity":"GPTBot","html":true,"fp":"1f628783c51e34fe"},"claudebot":{"status":403,"content_type":"text/plain","bytes":82,"sha256":"41609a6edee1e154f2092d6b2c64e26c5aa5a21d445ae636fe5a1f24b1ebc905","final_host":"wikipedia.org","final_path":"/","ms":349,"identity":"ClaudeBot","html":false},"oaisearch":{"status":200,"content_type":"text/html","bytes":122506,"sha256":"0d267456014b06bd08d6e50a554fb32abd30407b5f56dee637c26614bac05349","final_host":"www.wikipedia.org","final_path":"/","ms":1116,"identity":"OAI-SearchBot","html":true,"fp":"1f628783c51e34fe"},"perplexity":{"status":200,"content_type":"text/html","bytes":122506,"sha256":"0d267456014b06bd08d6e50a554fb32abd30407b5f56dee637c26614bac05349","final_host":"www.wikipedia.org","final_path":"/","ms":1116,"identity":"PerplexityBot","html":true,"fp":"1f628783c51e34fe"}},"robots":{"status":200,"content_type":"text/plain","bytes":28275,"sha256":"48b98d80c8444d37cdf50cc9a4d2257b66039477e581066d2356062a4cbd678f","final_host":"en.wikipedia.org","final_path":"/robots.txt","ms":1453,"html":false,"text":"# robots.txt for http://www.wikipedia.org/ and friends\n#\n# Please note: There are a lot of pages on this site, and there are\n# some misbehaved spiders out there that go _way_ too fast. If you're\n# irresponsible, your access to the site may be blocked.\n#\n\n# Observed spamming large amounts of https://en.wikipedia.org/?curid=NNNNNN\n# and ignoring 429 ratelimit responses, claims to respect robots:\n# http://mj12bot.com/\nUser-agent: MJ12bot\nDisallow: /\n\n# advertising-related bots:\nUser-agent: Mediapartners-Google*\nDisallow: /\n\n# Wikipedia work bots:\nUser-agent: IsraBot\nDisallow:\n\nUser-agent: Orthogaffe\nDisallow:\n\n# Crawlers that are kind enough to obey, but which we'd rather not have\n# unless they're feeding search engines.\nUser-agent: UbiCrawler\nDisallow: /\n\nUser-agent: DOC\nDisallow: /\n\nUser-agent: Zao\nDisallow: /\n\n# Some bots are known to be trouble, particularly those designed to copy\n# entire sites. Please obey robots.txt.\nUser-agent: sitecheck.internetseer.com\nDisallow: /\n\nUser-agent: Zealbot\nDisallow: /\n\nUser-agent: MSIECrawler\nDisallow: /\n\nUser-agent: SiteSnagger\nDisallow: /\n\nUser-agent: WebStripper\nDisallow: /\n\nUser-agent: WebCopier\nDisallow: /\n\nUser-agent: Fetch\nDisallow: /\n\nUser-agent: Offline Explorer\nDisallow: /\n\nUser-agent: Teleport\nDisallow: /\n\nUser-agent: TeleportPro\nDisallow: /\n\nUser-agent: WebZIP\nDisallow: /\n\nUser-agent: linko\nDisallow: /\n\nUser-agent: HTTrack\nDisallow: /\n\nUser-agent: Microsoft.URL.Control\nDisallow: /\n\nUser-agent: Xenu\nDisallow: /\n\nUser-agent: larbin\nDisallow: /\n\nUser-agent: libwww\nDisallow: /\n\nUser-agent: ZyBORG\nDisallow: /\n\nUser-agent: Download Ninja\nDisallow: /\n\n# Misbehaving, requests much too fast\nUser-agent: fast\nDisallow: /\n\n# Sorry, wget in its recursive mode is a frequent problem.\n# Please read the man page and use it properly; there is a\n# --wait option you can use to set the delay between hits,\n# for instance.\nUser-agent: wget\nDisallow: /\n\n# The 'grub' distributed client has been *very* poorly behaved.\nUser-agent: grub-client\nDisallow: /\n\n# Doesn't follow robots.txt anyway, but...\nUser-agent: k2spider\nDisallow: /\n\n# Hits many times per second, not acceptable\n# http://www.nameprotect.com/botinfo.html\nUser-agent: NPBot\nDisallow: /\n\n# A capture bot, downloads gazillions of pages with no public benefit\n# http://www.webreaper.net/\nUser-agent: WebReaper\nDisallow: /\n\n# Per their statement, semrushbot respects crawl-delay directives\n# We want them to overall stay within reasonable request rates to\n# the backend (20 rps); keeping in mind that the crawl-delay will\n# be applied by site and not globally by the bot, 5 seconds seem\n# like a reasonable approximation\nUser-agent: SemrushBot\nCrawl-delay: 5\n\n#\n# Friendly, low-speed bots are welcome viewing article pages, but not\n# dynamically-generated pages please.\n#\n# Inktomi's \"Slurp\" can read a minimum delay between hits; if your\n# bot supports such a thing using the 'Crawl-delay' or another\n# instruction, please let us know.\n#\n# There is a special exception for API mobileview to allow dynamic\n# mobile web & app views to load section content.\n# These views aren't HTTP-cached but use parser cache aggressively\n# and don't expose Special-pages and other uncached content.\n#\n# Another exception is for REST API documentation, located at\n# /api/rest_v1/?doc.\n#\nUser-agent: *\nAllow: /w/api.php?action=mobileview&\nAllow: /w/load.php?\nAllow: /api/rest_v1/?doc\nAllow: /w/rest.php/site/v1/sitemap\nDisallow: /w/\nDisallow: /api/\nDisallow: /trap/\nDisallow: /wiki/Special:\nDisallow: /wiki/Spezial:\nDisallow: /wiki/Spesial:\nDisallow: /wiki/Special%3A\nDisallow: /wiki/Spezial%3A\nDisallow: /wiki/Spesial%3A\n#\n# ar:\nDisallow: /wiki/%D8%AE%D8%A7%D8%B5:Search\nDisallow: /wiki/%D8%AE%D8%A7%D8%B5%3ASearch\n#\n# dewiki:\n# T6937\n# sensible deletion and meta user discussion pages:\nDisallow: /wiki/Wikipedia:L%C3%B6schkandidaten/\nDisallow: /wiki/Wikipedia:Löschkandidaten/\nDisallow: /wiki/Wikipedia:Vandalensperrung/\nDisallow: /wiki/Wikipedia:Benutzersperrung/\nDisallow: /wiki/Wikipedia:Vermittlungsausschuss/\nDisallow: /wiki/Wikipedia:Administratoren/Probleme/\nDisallow: /wiki/Wikipedia:Adminkandidaturen/\nDisallow: /wiki/Wikipedia:Qualitätssicherung/\nDisallow: /wiki/Wikipedia:Qualit%C3%A4tssicherung/\n# 4937#5\nDisallow: /wiki/Wikipedia:Vandalismusmeldung/\nDisallow: /wiki/Wikipedia:Gesperrte_Lemmata/\nDisallow: /wiki/Wikipedia:Löschprüfung/\nDisallow: /wiki/Wikipedia:L%C3%B6schprüfung/\nDisallow: /wiki/Wikipedia:Administratoren/Notizen/\nDisallow: /wiki/Wikipedia:Schiedsgericht/Anfragen/\nDisallow: /wiki/Wikipedia:L%C3%B6schpr%C3%BCfung/\n# T14111\nDisallow: /wiki/Wikipedia:Checkuser/\nDisallow: /wiki/Wikipedia_Diskussion:Checkuser/\nDisallow: /wiki/Wikipedia_Diskussion:Adminkandidaturen/\n# T15961\nDisallow: /wiki/Wikipedia:Spam-Blacklist-Log\nDisallow: /wiki/Wikipedia%3ASpam-Blacklist-Log\nDisallow: /wiki/Wikipedia_Diskussion:Spam-Blacklist-Log\nDisallow: /wiki/Wikipedia_Diskussion%3ASpam-Blacklist-Log\n#\n# enwiki:\n# Folks get annoyed when VfD discussions end up the number 1 google hit for\n# their name. See T6776\nDisallow: /wiki/Wikipedia:Articles_for_deletion/\nDisallow: /wiki/Wikipedia%3AArticles_for_deletion/\nDisallow: /wiki/Wikipedia:Votes_for_deletion/\nDisallow: /wiki/Wikipedia%3AVotes_for_deletion/\nDisallow: /wiki/Wikipedia:Pages_for_deletion/\nDisallow: /wiki/Wikipedia%3APages_for_deletion/\nDisallow: /wiki/Wikipedia:Miscellany_for_deletion/\nDisallow: /wiki/Wikipedia%3AMiscellany_for_deletion/\nDisallow: /wiki/Wikipedia:Miscellaneous_deletion/\nDisallow: /wiki/Wikipedia%3AMiscellaneous_deletion/\nDisallow: /wiki/Wikipedia:Copyright_problems\nDisallow: /wiki/Wikipedia%3ACopyright_problems\nDisallow: /wiki/Wikipedia:Protected_titles/\nDisallow: /wiki/Wikipedia%3AProtected_titles/\n# T15398\nDisallow: /wiki/Wikipedia:WikiProject_Spam/\nDisallow: /wiki/Wikipedia%3AWikiProject_Spam/\n# T16075\nDisallow: /wiki/MediaWiki:Spam-blacklist\nDisallow: /wiki/MediaWiki%3ASpam-blacklist\nDisallow: /wiki/MediaWiki_talk:Spam-blacklist\nDisallow: /wiki/MediaWiki_talk%3ASpam-blacklist\n# T13261\nDisallow: /wiki/Wikipedia:Requests_for_arbitration/\nDisallow: /wiki/Wikipedia%3ARequests_for_arbitration/\nDisallow: /wiki/Wikipedia:Requests_for_comment/\nDisallow: /wiki/Wikipedia%3ARequests_for_comment/\nDisallow: /wiki/Wikipedia:Requests_for_adminship/\nDisallow: /wiki/Wikipedia%3ARequests_for_adminship/\n# T12288\nDisallow: /wiki/Wikipedia_talk:Articles_for_deletion/\nDisallow: /wiki/Wikipedia_talk%3AArticles_for_deletion/\nDisallow: /wiki/Wikipedia_talk:Votes_for_deletion/\nDisallow: /wiki/Wikipedia_talk%3AVotes_for_deletion/\nDisallow: /wiki/Wikipedia_talk:Pages_for_deletion/\nDisallow: /wiki/Wikipedia_talk%3APages_for_deletion/\nDisallow: /wiki/Wikipedia_talk:Miscellany_for_deletion/\nDisallow: /wiki/Wikipedia_talk%3AMiscellany_for_deletion/\nDisallow: /wiki/Wikipedia_talk:Miscellaneous_deletion/\nDisallow: /wiki/Wikipedia_talk%3AMiscellaneous_deletion/\n# T16793\nDisallow: /wiki/Wikipedia:Changing_username\nDisallow: /wiki/Wikipedia%3AChanging_username\nDisallow: /wiki/Wikipedia:Changing_username/\nDisallow: /wiki/Wikipedia%3AChanging_username/\nDisallow: /wiki/Wikipedia_talk:Changing_username\nDisallow: /wiki/Wikipedia_talk%3AChanging_username\nDisallow: /wiki/Wikipedia_talk:Changing_username/\nDisallow: /wiki/Wikipedia_talk%3AChanging_username/\n#\n# eswiki:\n# T8746\nDisallow: /wiki/Wikipedia:Consultas_de_borrado/\nDisallow: /wiki/Wikipedia%3AConsultas_de_borrado/\n#\n# fiwiki:\n# T10695\nDisallow: /wiki/Wikipedia:Poistettavat_sivut\nDisallow: /wiki/K%C3%A4ytt%C3%A4j%C3%A4:\nDisallow: /wiki/Käyttäjä:\nDisallow: /wiki/Keskustelu_k%C3%A4ytt%C3%A4j%C3%A4st%C3%A4:\nDisallow: /wiki/Keskustelu_käyttäjästä:\nDisallow: /wiki/Wikipedia:Yll%C3%A4pit%C3%A4j%C3%A4t/\nDisallow: /wiki/Wikipedia:Ylläpitäjät/\n#\n# hewiki:\nDisallow: /wiki/%D7%9E%D7%99%D7%95%D7%97%D7%93:Search\nDisallow: /wiki/%D7%9E%D7%99%D7%95%D7%97%D7%93%3ASearch\n#T11517\nDisallow: /wiki/ויקיפדיה:רשימת_מועמדים_למחיקה/\nDisallow: /wiki/ויקיפדיה%3Aרשימת_מועמדים_למחיקה/\nDisallow: /wiki/%D7%95%D7%99%D7%A7%D7%99%D7%A4%D7%93%D7%99%D7%94:%D7%A8%D7%A9%D7%99%D7%9E%D7%AA_%D7%9E%D7%95%D7%A2%D7%9E%D7%93%D7%99%D7%9D_%D7%9C%D7%9E%D7%97%D7%99%D7%A7%D7%94/\nDisallow: /wiki/%D7%95%D7%99%D7%A7%D7%99%D7%A4%D7%93%D7%99%D7%94%3A%D7%A8%D7%A9%D7%99%D7%9E%D7%AA_%D7%9E%D7%95%D7%A2%D7%9E%D7%93%D7%99%D7%9D_%D7%9C%D7%9E%D7%97%D7%99%D7%A7%D7%94/\nDisallow: /wiki/ויקיפדיה:ערכים_לא_קיימים_ומוגנים\nDisallow: /wiki/ויקיפדיה%3Aערכים_לא_קיימים_ומוגנים\nDisallow: /wiki/%D7%95%D7%99%D7%A7%D7%99%D7%A4%D7%93%D7%99%D7%94:%D7%A2%D7%A8%D7%9B%D7%99%D7%9D_%D7%9C%D7%90_%D7%A7%D7%99%D7%99%D7%9E%D7%99%D7%9D_%D7%95%D7%9E%D7%95%D7%92%D7%A0%D7%99%D7%9D\nDisallow: /wiki/%D7%95%D7%99%D7%A7%D7%99%D7%A4%D7%93%D7%99%D7%94%3A%D7%A2%D7%A8%D7%9B%D7%99%D7%9D_%D7%9C%D7%90_%D7%A7%D7%99%D7%99%D7%9E%D7%99%D7%9D_%D7%95%D7%9E%D7%95%D7%92%D7%A0%D7%99%D7%9D\nDisallow: /wiki/ויקיפדיה:דפים_לא_קיימים_ומוגנים\nDisallow: /wiki/ויקיפדיה%3Aדפים_לא_קיימים_ומוגנים\nDisallow: /wiki/%D7%95%D7%99%D7%A7%D7%99%D7%A4%D7%93%D7%99%D7%94:%D7%93%D7%A4%D7%99%D7%9D_%D7%9C%D7%90_%D7%A7%D7%99%D7%99%D7%9E%D7%99%D7%9D_%D7%95%D7%9E%D7%95%D7%92%D7%A0%D7%99%D7%9D\nDisallow: /wiki/%D7%95%D7%99%D7%A7%D7%99%D7%A4%D7%93%D7%99%D7%94%3A%D7%93%D7%A4%D7%99%D7%9D_%D7%9C%D7%90_%D7%A7%D7%99%D7%99%D7%9E%D7%99%D7%9D_%D7%95%D7%9E%D7%95%D7%92%D7%A0%D7%99%D7%9D\n#\n# huwiki:\nDisallow: /wiki/Speci%C3%A1lis:Search\nDisallow: /wiki/Speci%C3%A1lis%3ASearch\n#\n# itwiki:\n# T7545\nDisallow: /wiki/Wikipedia:Pagine_da_cancellare\nDisallow: /wiki/Wikipedia%3APagine_da_cancellare\nDisallow: /wiki/Wikipedia:Utenti_problematici\nDisallow: /wiki/Wikipedia%3AUtenti_problematici\nDisallow: /wiki/Wikipedia:Vandalismi_in_corso\nDisallow: /wiki/Wikipedia%3AVandalismi_in_corso\nDisallow: /wiki/Wikipedia:Amministratori\nDisallow: /wiki/Wikipedia%3AAmministratori\nDisallow: /wiki/Wikipedia:Proposte_di_cancellazione_semplificata\nDisallow: /wiki/Wikipedia%3AProposte_di_cancellazione_semplificata\nDisallow: /wiki/Categoria:Da_cancellare_subito\nDisallow: /wiki/Categoria%3ADa_cancellare_subito\nDisallow: /wiki/Wikipedia:Sospette_violazioni_di_copyright\nDisallow: /wiki/Wikipedia%3ASospette_violazioni_di_copyright\nDisallow: /wiki/Categoria:Da_controllare_per_copyright\nDisallow: /wiki/Categoria%3ADa_controllare_per_copyright\nDisallow: /wiki/Progetto:Rimozione_contributi_sospetti\nDisallow: /wiki/Progetto%3ARimozione_contributi_sospetti\nDisallow: /wiki/Categoria:Da_cancellare_subito_per_violazione_integrale_copyright\nDisallow: /wiki/Categoria%3ADa_cancellare_subito_per_violazione_integrale_copyright\nDisallow: /wiki/Progetto:Cococo\nDisallow: /wiki/Progetto%3ACococo\nDisallow: /wiki/Discussioni_progetto:Cococo\nDisallow: /wiki/Discussioni_progetto%3ACococo\n#\n# jawiki\nDisallow: /wiki/%E7%89%B9%E5%88%A5:Search\nDisallow: /wiki/%E7%89%B9%E5%88%A5%3ASearch\n# T7239\nDisallow: /wiki/Wikipedia:%E5%89%8A%E9%99%A4%E4%BE%9D%E9%A0%BC/\nDisallow: /wiki/Wikipedia%3A%E5%89%8A%E9%99%A4%E4%BE%9D%E9%A0%BC/\nDisallow: /wiki/Wikipedia:%E5%88%A9%E7%94%A8%E8%80%85%E3%83%9A%E3%83%BC%E3%82%B8%E3%81%AE%E5%89%8A%E9%99%A4%E4%BE%9D%E9%A0%BC\nDisallow: /wiki/Wikipedia%3A%E5%88%A9%E7%94%A8%E8%80%85%E3%83%9A%E3%83%BC%E3%82%B8%E3%81%AE%E5%89%8A%E9%99%A4%E4%BE%9D%E9%A0%BC\n# nowiki\n# T13432\nDisallow: /wiki/Bruker:\nDisallow: /wiki/Bruker%3A\nDisallow: /wiki/Brukerdiskusjon\nDisallow: /wiki/Wikipedia:Administratorer\nDisallow: /wiki/Wikipedia%3AAdministratorer\nDisallow: /wiki/Wikipedia-diskusjon:Administratorer\nDisallow: /wiki/Wikipedia-diskusjon%3AAdministratorer\nDisallow: /wiki/Wikipedia:Sletting\nDisallow: /wiki/Wikipedia%3ASletting\nDisallow: /wiki/Wikipedia-diskusjon:Sletting\nDisallow: /wiki/Wikipedia-diskusjon%3ASletting\n#\n# plwiki\n# T10067\nDisallow: /wiki/Wikipedia:Strony_do_usuni%C4%99cia\nDisallow: /wiki/Wikipedia%3AStrony_do_usuni%C4%99cia\nDisallow: /wiki/Wikipedia:Do_usuni%C4%99cia\nDisallow: /wiki/Wikipedia%3ADo_usuni%C4%99cia\nDisallow: /wiki/Wikipedia:SDU/\nDisallow: /wiki/Wikipedia%3ASDU/\nDisallow: /wiki/Wikipedia:Strony_podejrzane_o_naruszenie_praw_autorskich\nDisallow: /wiki/Wikipedia%3AStrony_podejrzane_o_naruszenie_praw_autorskich\n#\n# ptwiki:\n# T7394\nDisallow: /wiki/Wikipedia:Páginas_para_eliminar/\nDisallow: /wiki/Wikipedia:P%C3%A1ginas_para_eliminar/\nDisallow: /wiki/Wikipedia%3AP%C3%A1ginas_para_eliminar/\nDisallow: /wiki/Wikipedia_Discussão:Páginas_para_eliminar/\nDisallow: /wiki/Wikipedia_Discuss%C3%A3o:P%C3%A1ginas_para_eliminar/\nDisallow: /wiki/Wikipedia_Discuss%C3%A3o%3AP%C3%A1ginas_para_eliminar/\n#\n# rowiki:\n# T14546\nDisallow: /wiki/Wikipedia:Pagini_de_%C5%9Fters\nDisallow: /wiki/Wikipedia%3APagini_de_%C5%9Fters\nDisallow: /wiki/Discu%C5%A3ie_Wikipedia:Pagini_de_%C5%9Fters\nDisallow: /wiki/Discu%C5%A3ie_Wikipedia%3APagini_de_%C5%9Fters\n#\n# ruwiki:\nDisallow: /wiki/%D0%A1%D0%BF%D0%B5%D1%86%D0%B8%D0%B0%D0%BB%D1%8C%D0%BD%D1%8B%D0%B5:Search\nDisallow: /wiki/%D0%A1%D0%BF%D0%B5%D1%86%D0%B8%D0%B0%D0%BB%D1%8C%D0%BD%D1%8B%D0%B5%3ASearch\n#\n# svwiki:\n# T12229\nDisallow: /wiki/Wikipedia%3ASidor_f%C3%B6reslagna_f%C3%B6r_radering\nDisallow: /wiki/Wikipedia:Sidor_f%C3%B6reslagna_f%C3%B6r_radering\nDisallow: /wiki/Wikipedia:Sidor_föreslagna_för_radering\nDisallow: /wiki/Användare\nDisallow: /wiki/Anv%C3%A4ndare\nDisallow: /wiki/Användardiskussion\nDisallow: /wiki/Anv%C3%A4ndardiskussion\nDisallow: /wiki/Wikipedia:Skyddade_sidnamn\nDisallow: /wiki/Wikipedia%3ASkyddade_sidnamn\n# T13291\nDisallow: /wiki/Wikipedia:Sidor_som_bör_raderas\nDisallow: /wiki/Wikipedia:Sidor_som_b%C3%B6r_raderas\nDisallow: /wiki/Wikipedia%3ASidor_som_b%C3%B6r_raderas\n#\n# zhwiki:\n# T7104\nDisallow: /wiki/Wikipedia:删除投票/侵权\nDisallow: /wiki/Wikipedia:%E5%88%A0%E9%99%A4%E6%8A%95%E7%A5%A8/%E4%BE%B5%E6%9D%83\nDisallow: /wiki/Wikipedia:删除投票和请求\nDisallow: /wiki/Wikipedia:%E5%88%A0%E9%99%A4%E6%8A%95%E7%A5%A8%E5%92%8C%E8%AF%B7%E6%B1%82\nDisallow: /wiki/Category:快速删除候选\nDisallow: /wiki/Category:%E5%BF%AB%E9%80%9F%E5%88%A0%E9%99%A4%E5%80%99%E9%80%89\nDisallow: /wiki/Category:维基百科需要翻译的文章\nDisallow: /wiki/Category:%E7%BB%B4%E5%9F%BA%E7%99%BE%E7%A7%91%E9%9C%80%E8%A6%81%E7%BF%BB%E8%AF%91%E7%9A%84%E6%96%87%E7%AB%A0\n#\n# sister projects\n#\n# enwikinews:\n# T7340\nDisallow: /wiki/Portal:Prepared_stories/\nDisallow: /wiki/Portal%3APrepared_stories/\n#\n# itwikinews\n# T11138\nDisallow: /wiki/Wikinotizie:Richieste_di_cancellazione\nDisallow: /wiki/Wikinotizie:Sospette_violazioni_di_copyright\nDisallow: /wiki/Categoria:Da_cancellare_subito\nDisallow: /wiki/Categoria:Da_cancellare_subito_per_violazione_integrale_copyright\nDisallow: /wiki/Wikinotizie:Storie_in_preparazione\n#\n# enwikiquote:\n# T17095\nDisallow: /wiki/Wikiquote:Votes_for_deletion/\nDisallow: /wiki/Wikiquote%3AVotes_for_deletion/\nDisallow: /wiki/Wikiquote_talk:Votes_for_deletion/\nDisallow: /wiki/Wikiquote_talk%3AVotes_for_deletion/\nDisallow: /wiki/Wikiquote:Votes_for_deletion_archive/\nDisallow: /wiki/Wikiquote%3AVotes_for_deletion_archive/\nDisallow: /wiki/Wikiquote_talk:Votes_for_deletion_archive/\nDisallow: /wiki/Wikiquote_talk%3AVotes_for_deletion_archive/\n#\n# enwikibooks\nDisallow: /wiki/Wikibooks:Votes_for_deletion\n#\n\n\nSitemap: https://en.wikipedia.org/w/rest.php/site/v1/sitemap/0\n\n#\n#\n#----------------------------------------------------------#\n#\n#\n#\n#\n# Localisable part of robots.txt for en.wikipedia.org\n#\n# Edit at https://en.wikipedia.org/w/index.php?title=MediaWiki:Robots.txt&action=edit\n# Don't add newlines here. All rules set here are active for every user-agent.\n#\n# Please check any changes using a syntax validator\n# Enter https://en.wikipedia.org/robots.txt as the URL to check.\n#\n# https://phabricator.wikimedia.org/T16075\nDisallow: /wiki/MediaWiki:Spam-blacklist\nDisallow: /wiki/MediaWiki%3ASpam-blacklist\nDisallow: /wiki/MediaWiki_talk:Spam-blacklist\nDisallow: /wiki/MediaWiki_talk%3ASpam-blacklist\nDisallow: /wiki/Wikipedia:WikiProject_Spam\nDisallow: /wiki/Wikipedia_talk:WikiProject_Spam\n#\n# Folks get annoyed when XfD discussions end up the number 1 google hit for\n# their name. \n# https://phabricator.wikimedia.org/T16075\nDisallow: /wiki/Wikipedia:Articles_for_deletion\nDisallow: /wiki/Wikipedia%3AArticles_for_deletion\nDisallow: /wiki/Wikipedia:Votes_for_deletion\nDisallow: /wiki/Wikipedia%3AVotes_for_deletion\nDisallow: /wiki/Wikipedia:Pages_for_deletion\nDisallow: /wiki/Wikipedia%3APages_for_deletion\nDisallow: /wiki/Wikipedia:Miscellany_for_deletion\nDisallow: /wiki/Wikipedia%3AMiscellany_for_deletion\nDisallow: /wiki/Wikipedia:Miscellaneous_deletion\nDisallow: /wiki/Wikipedia%3AMiscellaneous_deletion\nDisallow: /wiki/Wikipedia:Categories_for_discussion\nDisallow: /wiki/Wikipedia%3ACategories_for_discussion\nDisallow: /wiki/Wikipedia:Templates_for_deletion\nDisallow: /wiki/Wikipedia%3ATemplates_for_deletion\nDisallow: /wiki/Wikipedia:Redirects_for_discussion\nDisallow: /wiki/Wikipedia%3ARedirects_for_discussion\nDisallow: /wiki/Wikipedia:Deletion_review\nDisallow: /wiki/Wikipedia%3ADeletion_review\nDisallow: /wiki/Wikipedia:WikiProject_Deletion_sorting\nDisallow: /wiki/Wikipedia%3AWikiProject_Deletion_sorting\nDisallow: /wiki/Wikipedia:Files_for_deletion\nDisallow: /wiki/Wikipedia%3AFiles_for_deletion\nDisallow: /wiki/Wikipedia:Files_for_discussion\nDisallow: /wiki/Wikipedia%3AFiles_for_discussion\nDisallow: /wiki/Wikipedia:Possibly_unfree_files\nDisallow: /wiki/Wikipedia%3APossibly_unfree_files\n#\n# https://phabricator.wikimedia.org/T12288\nDisallow: /wiki/Wikipedia_talk:Articles_for_deletion\nDisallow: /wiki/Wikipedia_talk%3AArticles_for_deletion\nDisallow: /wiki/Wikipedia_talk:Votes_for_deletion\nDisallow: /wiki/Wikipedia_talk%3AVotes_for_deletion\nDisallow: /wiki/Wikipedia_talk:Pages_for_deletion\nDisallow: /wiki/Wikipedia_talk%3APages_for_deletion\nDisallow: /wiki/Wikipedia_talk:Miscellany_for_deletion\nDisallow: /wiki/Wikipedia_talk%3AMiscellany_for_deletion\nDisallow: /wiki/Wikipedia_talk:Miscellaneous_deletion\nDisallow: /wiki/Wikipedia_talk%3AMiscellaneous_deletion\nDisallow: /wiki/Wikipedia_talk:Templates_for_deletion\nDisallow: /wiki/Wikipedia_talk%3ATemplates_for_deletion\nDisallow: /wiki/Wikipedia_talk:Categories_for_discussion\nDisallow: /wiki/Wikipedia_talk%3ACategories_for_discussion\nDisallow: /wiki/Wikipedia_talk:Deletion_review\nDisallow: /wiki/Wikipedia_talk%3ADeletion_review\nDisallow: /wiki/Wikipedia_talk:WikiProject_Deletion_sorting\nDisallow: /wiki/Wikipedia_talk%3AWikiProject_Deletion_sorting\nDisallow: /wiki/Wikipedia_talk:Files_for_deletion\nDisallow: /wiki/Wikipedia_talk%3AFiles_for_deletion\nDisallow: /wiki/Wikipedia_talk:Files_for_discussion\nDisallow: /wiki/Wikipedia_talk%3AFiles_for_discussion\nDisallow: /wiki/Wikipedia_talk:Possibly_unfree_files\nDisallow: /wiki/Wikipedia_talk%3APossibly_unfree_files\n#\nDisallow: /wiki/Wikipedia:Copyright_problems\nDisallow: /wiki/Wikipedia%3ACopyright_problems\nDisallow: /wiki/Wikipedia_talk:Copyright_problems\nDisallow: /wiki/Wikipedia_talk%3ACopyright_problems\nDisallow: /wiki/Wikipedia:Suspected_copyright_violations\nDisallow: /wiki/Wikipedia%3ASuspected_copyright_violations\nDisallow: /wiki/Wikipedia_talk:Suspected_copyright_violations\nDisallow: /wiki/Wikipedia_talk%3ASuspected_copyright_violations\nDisallow: /wiki/Wikipedia:Contributor_copyright_investigations\nDisallow: /wiki/Wikipedia%3AContributor_copyright_investigations\nDisallow: /wiki/Wikipedia:Contributor_copyright_investigations\nDisallow: /wiki/Wikipedia%3AContributor_copyright_investigations\nDisallow: /wiki/Wikipedia_talk:Contributor_copyright_investigations\nDisallow: /wiki/Wikipedia_talk%3AContributor_copyright_investigations\nDisallow: /wiki/Wikipedia_talk:Contributor_copyright_investigations\nDisallow: /wiki/Wikipedia_talk%3AContributor_copyright_investigations\nDisallow: /wiki/Wikipedia:Protected_titles\nDisallow: /wiki/Wikipedia%3AProtected_titles\nDisallow: /wiki/Wikipedia_talk:Protected_titles\nDisallow: /wiki/Wikipedia_talk%3AProtected_titles\nDisallow: /wiki/Wikipedia:Articles_for_creation\nDisallow: /wiki/Wikipedia%3AArticles_for_creation\nDisallow: /wiki/Wikipedia_talk:Articles_for_creation\nDisallow: /wiki/Wikipedia_talk%3AArticles_for_creation\nDisallow: /wiki/Wikipedia_talk:Article_wizard\nDisallow: /wiki/Wikipedia_talk%3AArticle_wizard\n#\n# https://phabricator.wikimedia.org/T13261\nDisallow: /wiki/Wikipedia:Requests_for_arbitration\nDisallow: /wiki/Wikipedia%3ARequests_for_arbitration\nDisallow: /wiki/Wikipedia_talk:Requests_for_arbitration\nDisallow: /wiki/Wikipedia_talk%3ARequests_for_arbitration\nDisallow: /wiki/Wikipedia:Requests_for_comment\nDisallow: /wiki/Wikipedia%3ARequests_for_comment\nDisallow: /wiki/Wikipedia_talk:Requests_for_comment\nDisallow: /wiki/Wikipedia_talk%3ARequests_for_comment\nDisallow: /wiki/Wikipedia:Requests_for_adminship\nDisallow: /wiki/Wikipedia%3ARequests_for_adminship\nDisallow: /wiki/Wikipedia_talk:Requests_for_adminship\nDisallow: /wiki/Wikipedia_talk%3ARequests_for_adminship\n#\n# https://phabricator.wikimedia.org/T14111\nDisallow: /wiki/Wikipedia:Requests_for_checkuser\nDisallow: /wiki/Wikipedia%3ARequests_for_checkuser\nDisallow: /wiki/Wikipedia_talk:Requests_for_checkuser\nDisallow: /wiki/Wikipedia_talk%3ARequests_for_checkuser\n#\n# https://phabricator.wikimedia.org/T15398\nDisallow: /wiki/Wikipedia:WikiProject_Spam\nDisallow: /wiki/Wikipedia%3AWikiProject_Spam\n#\n# https://phabricator.wikimedia.org/T16793\nDisallow: /wiki/Wikipedia:Changing_username\nDisallow: /wiki/Wikipedia%3AChanging_username\nDisallow: /wiki/Wikipedia:Changing_username\nDisallow: /wiki/Wikipedia%3AChanging_username\nDisallow: /wiki/Wikipedia_talk:Changing_username\nDisallow: /wiki/Wikipedia_talk%3AChanging_username\nDisallow: /wiki/Wikipedia_talk:Changing_username\nDisallow: /wiki/Wikipedia_talk%3AChanging_username\n#\nDisallow: /wiki/Wikipedia:Administrators%27_noticeboard\nDisallow: /wiki/Wikipedia%3AAdministrators%27_noticeboard\nDisallow: /wiki/Wikipedia_talk:Administrators%27_noticeboard\nDisallow: /wiki/Wikipedia_talk%3AAdministrators%27_noticeboard\nDisallow: /wiki/Wikipedia:Community_sanction_noticeboard\nDisallow: /wiki/Wikipedia%3ACommunity_sanction_noticeboard\nDisallow: /wiki/Wikipedia_talk:Community_sanction_noticeboard\nDisallow: /wiki/Wikipedia_talk%3ACommunity_sanction_noticeboard\nDisallow: /wiki/Wikipedia:Bureaucrats%27_noticeboard\nDisallow: /wiki/Wikipedia%3ABureaucrats%27_noticeboard\nDisallow: /wiki/Wikipedia_talk:Bureaucrats%27_noticeboard\nDisallow: /wiki/Wikipedia_talk%3ABureaucrats%27_noticeboard\n#\nDisallow: /wiki/Wikipedia:Sockpuppet_investigations\nDisallow: /wiki/Wikipedia%3ASockpuppet_investigations\nDisallow: /wiki/Wikipedia_talk:Sockpuppet_investigations\nDisallow: /wiki/Wikipedia_talk%3ASockpuppet_investigations\n#\nDisallow: /wiki/Wikipedia:Neutral_point_of_view/Noticeboard\nDisallow: /wiki/Wikipedia%3ANeutral_point_of_view/Noticeboard\nDisallow: /wiki/Wikipedia_talk:Neutral_point_of_view/Noticeboard\nDisallow: /wiki/Wikipedia_talk%3ANeutral_point_of_view/Noticeboard\n#\nDisallow: /wiki/Wikipedia:No_original_research/noticeboard\nDisallow: /wiki/Wikipedia%3ANo_original_research/noticeboard\nDisallow: /wiki/Wikipedia_talk:No_original_research/noticeboard\nDisallow: /wiki/Wikipedia_talk%3ANo_original_research/noticeboard\n#\nDisallow: /wiki/Wikipedia:Fringe_theories/Noticeboard\nDisallow: /wiki/Wikipedia%3AFringe_theories/Noticeboard\nDisallow: /wiki/Wikipedia_talk:Fringe_theories/Noticeboard\nDisallow: /wiki/Wikipedia_talk%3AFringe_theories/Noticeboard\n#\nDisallow: /wiki/Wikipedia:Conflict_of_interest/Noticeboard\nDisallow: /wiki/Wikipedia%3AConflict_of_interest/Noticeboard\nDisallow: /wiki/Wikipedia_talk:Conflict_of_interest/Noticeboard\nDisallow: /wiki/Wikipedia_talk%3AConflict_of_interest/Noticeboard\n#\nDisallow: /wiki/Wikipedia:Long-term_abuse\nDisallow: /wiki/Wikipedia%3ALong-term_abuse\nDisallow: /wiki/Wikipedia_talk:Long-term_abuse\nDisallow: /wiki/Wikipedia_talk%3ALong-term_abuse\nDisallow: /wiki/Wikipedia:Long_term_abuse\nDisallow: /wiki/Wikipedia%3ALong_term_abuse\nDisallow: /wiki/Wikipedia_talk:Long_term_abuse\nDisallow: /wiki/Wikipedia_talk%3ALong_term_abuse\n#\nDisallow: /wiki/Wikipedia:Wikiquette_assistance\nDisallow: /wiki/Wikipedia%3AWikiquette_assistance\n#\nDisallow: /wiki/Wikipedia:Abuse_reports\nDisallow: /wiki/Wikipedia%3AAbuse_reports\nDisallow: /wiki/Wikipedia_talk:Abuse_reports\nDisallow: /wiki/Wikipedia_talk%3AAbuse_reports\nDisallow: /wiki/Wikipedia:Abuse_response\nDisallow: /wiki/Wikipedia%3AAbuse_response\nDisallow: /wiki/Wikipedia_talk:Abuse_response\nDisallow: /wiki/Wikipedia_talk%3AAbuse_response\n#\nDisallow: /wiki/Wikipedia:Reliable_sources/Noticeboard\nDisallow: /wiki/Wikipedia%3AReliable_sources/Noticeboard\nDisallow: /wiki/Wikipedia_talk:Reliable_sources/Noticeboard\nDisallow: /wiki/Wikipedia_talk%3AReliable_sources/Noticeboard\n#\nDisallow: /wiki/Wikipedia:Suspected_sock_puppets\nDisallow: /wiki/Wikipedia%3ASuspected_sock_puppets\nDisallow: /wiki/Wikipedia_talk:Suspected_sock_puppets\nDisallow: /wiki/Wikipedia_talk%3ASuspected_sock_puppets\n#\nDisallow: /wiki/Wikipedia:Biographies_of_living_persons/Noticeboard\nDisallow: /wiki/Wikipedia%3ABiographies_of_living_persons/Noticeboard\nDisallow: /wiki/Wikipedia_talk:Biographies_of_living_persons/Noticeboard\nDisallow: /wiki/Wikipedia_talk%3ABiographies_of_living_persons/Noticeboard\nDisallow: /wiki/Wikipedia:Biographies_of_living_persons%2FNoticeboard\nDisallow: /wiki/Wikipedia%3ABiographies_of_living_persons%2FNoticeboard\nDisallow: /wiki/Wikipedia_talk:Biographies_of_living_persons%2FNoticeboard\nDisallow: /wiki/Wikipedia_talk%3ABiographies_of_living_persons%2FNoticeboard\n#\nDisallow: /wiki/Wikipedia:Content_noticeboard\nDisallow: /wiki/Wikipedia%3AContent_noticeboard\nDisallow: /wiki/Wikipedia_talk:Content_noticeboard\nDisallow: /wiki/Wikipedia_talk%3AContent_noticeboard\n#\nDisallow: /wiki/Template:Editnotices\nDisallow: /wiki/Template%3AEditnotices\n#\nDisallow: /wiki/Wikipedia:Arbitration\nDisallow: /wiki/Wikipedia%3AArbitration\nDisallow: /wiki/Wikipedia_talk:Arbitration\nDisallow: /wiki/Wikipedia_talk%3AArbitration\n#\nDisallow: /wiki/Wikipedia:Arbitration_Committee\nDisallow: /wiki/Wikipedia%3AArbitration_Committee\nDisallow: /wiki/Wikipedia_talk:Arbitration_Committee\nDisallow: /wiki/Wikipedia_talk%3AArbitration_Committee\n#\nDisallow: /wiki/Wikipedia:Arbitration_Committee_Elections\nDisallow: /wiki/Wikipedia%3AArbitration_Committee_Elections\nDisallow: /wiki/Wikipedia_talk:Arbitration_Committee_Elections\nDisallow: /wiki/Wikipedia_talk%3AArbitration_Committee_Elections\n#\nDisallow: /wiki/Wikipedia:Mediation_Committee\nDisallow: /wiki/Wikipedia%3AMediation_Committee\nDisallow: /wiki/Wikipedia_talk:Mediation_Committee\nDisallow: /wiki/Wikipedia_talk%3AMediation_Committee\n#\nDisallow: /wiki/Wikipedia:Mediation_Cabal/Cases\nDisallow: /wiki/Wikipedia%3AMediation_Cabal/Cases\n#\nDisallow: /wiki/Wikipedia:Requests_for_bureaucratship\nDisallow: /wiki/Wikipedia%3ARequests_for_bureaucratship\nDisallow: /wiki/Wikipedia_talk:Requests_for_bureaucratship\nDisallow: /wiki/Wikipedia_talk%3ARequests_for_bureaucratship\n#\nDisallow: /wiki/Wikipedia:Administrator_review\nDisallow: /wiki/Wikipedia%3AAdministrator_review\nDisallow: /wiki/Wikipedia_talk:Administrator_review\nDisallow: /wiki/Wikipedia_talk%3AAdministrator_review\n#\nDisallow: /wiki/Wikipedia:Editor_review\nDisallow: /wiki/Wikipedia%3AEditor_review\nDisallow: /wiki/Wikipedia_talk:Editor_review\nDisallow: /wiki/Wikipedia_talk%3AEditor_review\n#\nDisallow: /wiki/Wikipedia:Article_Incubator\nDisallow: /wiki/Wikipedia%3AArticle_Incubator\nDisallow: /wiki/Wikipedia_talk:Article_Incubator\nDisallow: /wiki/Wikipedia_talk%3AArticle_Incubator\n#\nDisallow: /wiki/Category:Noindexed_pages\nDisallow: /wiki/Category%3ANoindexed_pages\n#\n# User sandboxes for modules and Template Styles are placed in these subpages for testing\n#\nDisallow: /wiki/Module:Sandbox\nDisallow: /wiki/Module%3ASandbox\nDisallow: /wiki/Template:TemplateStyles_sandbox\nDisallow: /wiki/Template%3ATemplateStyles_sandbox\n# Adminship and arbitration discussions are blocked from search engines, so blocking newer adminship processes (as of 2024) as well\nDisallow: /wiki/Wikipedia:Administrator_recall\nDisallow: /wiki/Wikipedia%3AAdministrator_recall\nDisallow: /wiki/Wikipedia_talk:Administrator_recall\nDisallow: /wiki/Wikipedia_talk%3AAdministrator_recall\nDisallow: /wiki/Wikipedia:Administrator_elections\nDisallow: /wiki/Wikipedia%3AAdministrator_elections\nDisallow: /wiki/Wikipedia_talk:Administrator_elections\nDisallow: /wiki/Wikipedia_talk%3AAdministrator_elections","agents":{"GPTBot":true,"OAI-SearchBot":true,"ChatGPT-User":true,"ClaudeBot":true,"Claude-SearchBot":true,"PerplexityBot":true,"Google-Extended":true,"Applebot-Extended":true,"CCBot":true,"Bytespider":true,"Meta-ExternalAgent":true,"Amazonbot":true}},"sitemap":{"status":403,"content_type":"text/html","bytes":3358,"sha256":"ce8abc2c6d08184cb7c63e77e3ebaad153a14a6ec49cf95a0eff7e6c53b434c2","final_host":"en.wikipedia.org","final_path":"/w/rest.php/site/v1/sitemap/0","ms":104,"url":"https://en.wikipedia.org/w/rest.php/site/v1/sitemap/0","declared":true,"xml":false},"files":{"llms_txt":{"status":404,"content_type":"text/html","bytes":2692,"sha256":"ee1efb6d6f284ee741b66f61d5baac16f8034d67feef6c51681982ab60a06a9f","final_host":"en.wikipedia.org","final_path":"/llms.txt","ms":656,"path":"/llms.txt","html":true,"json":null,"present":false},"agents_md":{"status":404,"content_type":"text/html","bytes":2695,"sha256":"2e8e528146a7f212165514f7c0b51994a33c8bc22cd69a2e129679b193bc83aa","final_host":"en.wikipedia.org","final_path":"/agents.md","ms":678,"path":"/agents.md","html":true,"json":null,"present":false},"agent_skills":{"status":404,"content_type":"text/html","bytes":2773,"sha256":"c7b071b55ca051a0fde81d87b070a8b1b66fa71f36f63fd136614db7db68d17b","final_host":"en.wikipedia.org","final_path":"/.well-known/agent-skills/index.json","ms":665,"path":"/.well-known/agent-skills/index.json","html":true,"json":null,"present":false},"mcp_card":{"status":404,"content_type":"text/html","bytes":2764,"sha256":"65a730402eeab88bdb2e21843c2c9f2a59fbe90d9dbdbedb7bdb56ec0a20a62c","final_host":"en.wikipedia.org","final_path":"/.well-known/mcp/server-card.json","ms":873,"path":"/.well-known/mcp/server-card.json","html":true,"json":null,"present":false},"api_catalog":{"status":404,"content_type":"text/html","bytes":2737,"sha256":"9321a2060e71ab7f661cea22389c9453d798be8fdddd854ad3fb10b1ccc2aff0","final_host":"en.wikipedia.org","final_path":"/.well-known/api-catalog","ms":857,"path":"/.well-known/api-catalog","html":true,"json":null,"present":false},"a2a_card":{"status":404,"content_type":"text/html","bytes":2749,"sha256":"de9924b979bbdf4135e5a534f74db7b6c3b93ee915160ec2e1571c0b3e9dac14","final_host":"en.wikipedia.org","final_path":"/.well-known/agent-card.json","ms":863,"path":"/.well-known/agent-card.json","html":true,"json":null,"present":false}},"rings":{"v":1,"candidates":[],"mcp":[],"a2a":null,"webmcp":{"model_context":false,"bridge":false,"mcp_json":false,"link_rel":false,"any":false},"handshakes":0,"get_path":{"status":404,"content_type":"text/html","ms":669,"error":null},"get_host":{"status":530,"content_type":"text/plain","ms":166,"error":null}},"sf":[4,0,0,0,0,0,0,0,1],"derived":{"reachable":true,"home_class":"answers","robots_served":true,"robots_sitemap":true,"content_signal":false,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":[],"blocks_any_ai":false,"files":{"llms_txt":false,"agents_md":false,"agent_skills":false,"mcp_card":false,"api_catalog":false,"a2a_card":false},"any_capability":false,"ai_refused":["claudebot"],"ai_thin":[],"aipref":0,"aipref_measured":true,"sf":[4,0,0,0,0,0,0,0,1],"rings":{"mcp_declared":false,"mcp_live":false,"mcp_auth":false,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":3,"fh":"www.wikipedia.org","fp_other_host":[],"fp":"1f628783c51e34fe","fp_bits":{"gptbot":0,"oaisearch":0,"perplexity":0}},"derive_v":5,"frame":"top10k","shard":5},{"v":1,"kind":"crawlcheck-census-record","run":18,"domain":"appsflyersdk.com","rank":31,"measured_at":"2026-11-13T18:53:15.625Z","home":{"browser":{"status":403,"content_type":"text/xml","bytes":146,"sha256":"95769eb326e23b8cd1b7a9b82d884a713a9724f49b2794ac5b967d8cd706de83","final_host":"appsflyersdk.com","final_path":"/","ms":73,"identity":"Mobile browser","html":false},"gptbot":{"status":403,"content_type":"text/xml","bytes":146,"sha256":"95769eb326e23b8cd1b7a9b82d884a713a9724f49b2794ac5b967d8cd706de83","final_host":"appsflyersdk.com","final_path":"/","ms":66,"identity":"GPTBot","html":false},"claudebot":{"status":403,"content_type":"text/xml","bytes":146,"sha256":"95769eb326e23b8cd1b7a9b82d884a713a9724f49b2794ac5b967d8cd706de83","final_host":"appsflyersdk.com","final_path":"/","ms":63,"identity":"ClaudeBot","html":false},"oaisearch":{"status":403,"content_type":"text/xml","bytes":146,"sha256":"95769eb326e23b8cd1b7a9b82d884a713a9724f49b2794ac5b967d8cd706de83","final_host":"appsflyersdk.com","final_path":"/","ms":13,"identity":"OAI-SearchBot","html":false},"perplexity":{"status":403,"content_type":"text/xml","bytes":146,"sha256":"95769eb326e23b8cd1b7a9b82d884a713a9724f49b2794ac5b967d8cd706de83","final_host":"appsflyersdk.com","final_path":"/","ms":5,"identity":"PerplexityBot","html":false}},"robots":{"status":403,"content_type":"text/xml","bytes":146,"sha256":"95769eb326e23b8cd1b7a9b82d884a713a9724f49b2794ac5b967d8cd706de83","final_host":"appsflyersdk.com","final_path":"/robots.txt","ms":64,"html":false,"text":null,"agents":null},"sitemap":null,"files":null,"derived":{"reachable":false,"home_class":"refused_us","robots_served":false,"robots_sitemap":false,"content_signal":false,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":[],"blocks_any_ai":false,"files":{"llms_txt":false,"agents_md":false,"agent_skills":false,"mcp_card":false,"api_catalog":false,"a2a_card":false},"any_capability":false,"ai_refused":[],"ai_thin":[],"rings":{"mcp_declared":false,"mcp_live":false,"mcp_auth":false,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":3,"fh":"appsflyersdk.com","fp_other_host":[]},"derive_v":5,"frame":"top10k","shard":6},{"v":1,"kind":"crawlcheck-census-record","run":18,"domain":"bing.com","rank":32,"measured_at":"2026-11-13T18:55:41.951Z","home":{"browser":{"status":200,"content_type":"text/html","bytes":53486,"sha256":"263e30c8e600482ea634f1af4898a51aff024a4921f253b9a47fbdaa629e50ac","final_host":"www.bing.com","final_path":"/","ms":90,"identity":"Mobile browser","html":true},"gptbot":{"status":200,"content_type":"text/html","bytes":78883,"sha256":"354ff2b186525abe604bb96619fa8e94ed6b408dd3d3720cdde24cb820b7c432","final_host":"www.bing.com","final_path":"/","ms":77,"identity":"GPTBot","html":true},"claudebot":{"status":200,"content_type":"text/html","bytes":79090,"sha256":"f7ed825930930ca795a43b9a984d2743e1a7050e67c3b2f5233cb0d48251ce3a","final_host":"www.bing.com","final_path":"/","ms":72,"identity":"ClaudeBot","html":true},"oaisearch":{"status":200,"content_type":"text/html","bytes":79371,"sha256":"356cfa3e6b26f86ce09e7bde2233f0793b0fcbab69cdbb6e77c6b4a1e6318fa3","final_host":"www.bing.com","final_path":"/","ms":87,"identity":"OAI-SearchBot","html":true},"perplexity":{"status":200,"content_type":"text/html","bytes":78654,"sha256":"69963296ebbdd8e30b7af1f6f73671becbce4ba223e26d82c4015bcbd8a2ff91","final_host":"www.bing.com","final_path":"/","ms":94,"identity":"PerplexityBot","html":true}},"robots":{"status":200,"content_type":"text/plain","bytes":2676,"sha256":"40d2da70c3b443225d5b01cc7b05117944980347637cc95ab285c6e4f5548235","final_host":"www.bing.com","final_path":"/robots.txt","ms":99,"html":false,"text":"User-agent: msnbot-media\r\nDisallow: /\r\nAllow: /th?\r\n\r\nUser-agent: Twitterbot\r\nDisallow:\r\n\r\nUser-agent: *\r\nDisallow: /academic/profile\r\nDisallow: /academic/search\r\nDisallow: /account/\r\nDisallow: /aclick\r\nDisallow: /alink\r\nDisallow: /amp/\r\nAllow: /api/maps/\r\nDisallow: /api/\r\nDisallow: /bfp/search\r\nDisallow: /bing-site-safety\r\nDisallow: /blogs/search/\r\nDisallow: /ck/\r\nDisallow: /copilotsearch?q=\r\nDisallow: /cr$\r\nDisallow: /cr?\r\nDisallow: /entities/search\r\nDisallow: /entityexplore$\r\nDisallow: /entityexplore?\r\nDisallow: /fd/\r\nDisallow: /fun/api/\r\nDisallow: /fun/g/\r\nDisallow: /history\r\nDisallow: /hotel/reviews?\r\nDisallow: /hotels/search\r\nDisallow: /HpImageArchive.aspx\r\nDisallow: /hpm\r\nDisallow: /hpmob\r\nDisallow: /images/async/?\r\nDisallow: /images/async?\r\nDisallow: /images/hosted/?\r\nDisallow: /images/hosted/search?\r\nDisallow: /images/hosted?\r\nDisallow: /images/search%3F\r\nDisallow: /images/search%5C\r\nDisallow: /images/search/?\r\nDisallow: /images/search?\r\nDisallow: /images/searchbyimage\r\nDisallow: /images?\r\nDisallow: /inboxcommerce\r\nDisallow: /local\r\nDisallow: /maps/adsendpoint\r\nDisallow: /merchant/reviews?\r\nDisallow: /notifications/\r\nDisallow: /offers/proxy/dealsserver/api/log\r\nDisallow: /offers/proxy/dealsserver/buy\r\nDisallow: /ping\r\nDisallow: /product/reviews?\r\nDisallow: /profile/history?\r\nDisallow: /proFile/history?\r\nDisallow: /Proxy.ashx\r\nDisallow: /results\r\nDisallow: /rewardsapp/\r\nDisallow: /search\r\nDisallow: /Search\r\nDisallow: /settings\r\nDisallow: /shenghuo\r\nAllow: /shop$\r\nAllow: /shop/\r\nDisallow: /shop/collections\r\nDisallow: /shop/productpage\r\nDisallow: /shop/topics\r\nDisallow: /shop?*\r\nDisallow: /social/search?\r\nDisallow: /spbasic\r\nDisallow: /spresults\r\nDisallow: /static/\r\nDisallow: /th$\r\nDisallow: /th?\r\nDisallow: /translator/?ref=\r\nDisallow: /travel/api\r\nDisallow: /travel/css\r\nDisallow: /travel/flight/flightSearch\r\nDisallow: /travel/flight/flightSearchAction\r\nDisallow: /travel/flight/search/?\r\nDisallow: /travel/flight/search?\r\nDisallow: /travel/flights\r\nDisallow: /travel/flight-search\r\nDisallow: /travel/hotel/hotelMiniSearchRequest\r\nDisallow: /travel/hotel/hotelSearch\r\nDisallow: /travel/hotels/checkout\r\nDisallow: /travel/hotels/detail\r\nDisallow: /travel/hotels/search/?\r\nDisallow: /travel/hotels/search?\r\nDisallow: /travel/hotel-search\r\nDisallow: /travel/scripts\r\nDisallow: /travel/secure\r\nDisallow: /url\r\nDisallow: /videos/browsing\r\nDisallow: /videos/explore\r\nDisallow: /videos/favorites\r\nDisallow: /videos/feed\r\nDisallow: /videos/music\r\nDisallow: /videos/trending\r\nDisallow: /widget/cr\r\nDisallow: /widget/entity/search/?\r\nDisallow: /widget/render\r\nDisallow: /widget/snapshot\r\nDisallow: /work\r\n\r\nSitemap: https://www.bing.com/sitemap.xml","agents":{"GPTBot":true,"OAI-SearchBot":true,"ChatGPT-User":true,"ClaudeBot":true,"Claude-SearchBot":true,"PerplexityBot":true,"Google-Extended":true,"Applebot-Extended":true,"CCBot":true,"Bytespider":true,"Meta-ExternalAgent":true,"Amazonbot":true}},"sitemap":{"status":200,"content_type":"text/xml","bytes":1187,"sha256":"b01f48436b21eb3993fa19ff9244e963061c9be6b2fa159d18de0d8995334c13","final_host":"www.bing.com","final_path":"/sitemap.xml","ms":53,"url":"https://www.bing.com/sitemap.xml","declared":true,"xml":true},"files":{"llms_txt":{"status":404,"content_type":"text/html","bytes":0,"sha256":"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855","final_host":"www.bing.com","final_path":"/llms.txt","ms":125,"path":"/llms.txt","html":true,"json":null,"present":false},"agents_md":{"status":404,"content_type":"text/html","bytes":76156,"sha256":"a0df105ae73c2760c58827c0a38ecefc62c75b2d179f6d8b1f1ff63b27f0d028","final_host":"www.bing.com","final_path":"/agents.md","ms":168,"path":"/agents.md","html":true,"json":null,"present":false},"agent_skills":{"status":404,"content_type":"text/html","bytes":0,"sha256":"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855","final_host":"www.bing.com","final_path":"/.well-known/agent-skills/index.json","ms":81,"path":"/.well-known/agent-skills/index.json","html":true,"json":null,"present":false},"mcp_card":{"status":404,"content_type":"text/html","bytes":0,"sha256":"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855","final_host":"www.bing.com","final_path":"/.well-known/mcp/server-card.json","ms":125,"path":"/.well-known/mcp/server-card.json","html":true,"json":null,"present":false},"api_catalog":{"status":404,"content_type":"text/html","bytes":78607,"sha256":"549ecfd607bd3deed0c02595e24650b5ccdafac90804c943b22293608531faf4","final_host":"www.bing.com","final_path":"/.well-known/api-catalog","ms":98,"path":"/.well-known/api-catalog","html":true,"json":null,"present":false},"a2a_card":{"status":404,"content_type":"text/html","bytes":0,"sha256":"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855","final_host":"www.bing.com","final_path":"/.well-known/agent-card.json","ms":83,"path":"/.well-known/agent-card.json","html":true,"json":null,"present":false}},"rings":{"v":1,"candidates":[],"mcp":[],"a2a":null,"webmcp":{"model_context":false,"bridge":false,"mcp_json":false,"link_rel":false,"any":false},"handshakes":0,"get_path":{"status":404,"content_type":"text/html","ms":150,"error":null},"get_host":{"status":200,"content_type":"text/html","ms":157,"error":null}},"sf":[1,0,0,0,0,0,0,0,1],"derived":{"reachable":true,"home_class":"answers","robots_served":true,"robots_sitemap":true,"content_signal":false,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":[],"blocks_any_ai":false,"files":{"llms_txt":false,"agents_md":false,"agent_skills":false,"mcp_card":false,"api_catalog":false,"a2a_card":false},"any_capability":false,"ai_refused":[],"ai_thin":[],"aipref":0,"aipref_measured":true,"sf":[1,0,0,0,0,0,0,0,1],"rings":{"mcp_declared":false,"mcp_live":false,"mcp_auth":false,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":3,"fh":"www.bing.com","fp_other_host":[]},"derive_v":5,"frame":"top10k","shard":7},{"v":1,"kind":"crawlcheck-census-record","run":18,"domain":"whatsapp.net","rank":33,"measured_at":"2026-11-13T18:46:27.088Z","home":{"browser":{"status":400,"content_type":"text/html","bytes":6278,"sha256":"375af8de622bf68dd654ddd422ab227da311590cf576d939efd88425432bccc3","final_host":"www.whatsapp.com","final_path":"/","ms":1586,"identity":"Mobile browser","html":true},"gptbot":{"status":200,"content_type":"text/html","bytes":266260,"sha256":"0bc34b8cd9bd8f52fd2a97ec3788b44bcd90bcd5def0351e9d4e6ff5f0260e4c","final_host":"www.whatsapp.com","final_path":"/","ms":950,"identity":"GPTBot","html":true,"fp":"031ccaf7ab074b16"},"claudebot":{"status":200,"content_type":"text/html","bytes":266204,"sha256":"c2415048da16104b91ae778acb3ead086a05390bd0d81fd86f0a3c26ba0361c4","final_host":"www.whatsapp.com","final_path":"/","ms":1103,"identity":"ClaudeBot","html":true,"fp":"031ccaf7ab074b16"},"oaisearch":{"status":200,"content_type":"text/html","bytes":266542,"sha256":"cba40c16d53a65f2e789862b4a9b5651c136da848b6a755e81269688998c2aa5","final_host":"www.whatsapp.com","final_path":"/","ms":987,"identity":"OAI-SearchBot","html":true,"fp":"031ccaf7ab074b16"},"perplexity":{"status":200,"content_type":"text/html","bytes":281991,"sha256":"5807e9e06bf121fb93c7fb3744e4a0f07bcf3a7700f872c033297e50cd85dcc3","final_host":"www.whatsapp.com","final_path":"/","ms":988,"identity":"PerplexityBot","html":true,"fp":"219a4af7ab0f4b16"}},"robots":{"status":200,"content_type":"text/plain","bytes":1225,"sha256":"6d6febc5de572ecaca22e0169030eb4e4cd41ecaf12af30d64f2cdd9c000d0cb","final_host":"www.whatsapp.com","final_path":"/robots.txt","ms":998,"html":false,"text":"# Notice: Collection of data on Facebook through automated means is\n# prohibited unless you have express written permission from Facebook\n# and may only be conducted for the limited purpose contained in said\n# permission.\n# See: http://www.facebook.com/apps/site_scraping_tos_terms.php\n\nUser-agent: __usage_ai-training\nDisallow: /\n\nUser-agent: Amazonbot\nDisallow: /\n\nUser-agent: Applebot-Extended\nDisallow: /\n\nUser-agent: Brightbot\nDisallow: /\n\nUser-agent: ClaudeBot\nDisallow: /\n\nUser-agent: Google-Extended\nDisallow: /\n\nUser-agent: GPTBot\nDisallow: /\n\nUser-agent: PerplexityBot\nDisallow: /\n\nUser-agent: PetalBot\nDisallow: /\n\nUser-agent: Scrapy\nDisallow: /\n\nUser-agent: uptimerobot\nDisallow: /\n\nUser-agent: viberbot\nDisallow: /\n\nUser-agent: YaK\nDisallow: /\n\nUser-agent: Yandex\nDisallow: /\n\nUser-agent: Yeti\nDisallow: /\n\nUser-agent: facebookexternalhit\nAllow: *\n\nUser-agent: meta-externalads\nAllow: *\n\nUser-agent: *\nDisallow: /*cursor=\nDisallow: /*fb_comment_id=\nDisallow: /ajax/\nDisallow: /tealium/\nDisallow: /intern/\nDisallow: /internal/\nDisallow: /login/\nDisallow: /oidc/callback/\nDisallow: /*.php\nDisallow: /android/third_party\nDisallow: /settings\n\nSitemap: https://www.whatsapp.com/sitemap/www_whatsapp_com_sitemap.xml.gz","agents":{"GPTBot":false,"OAI-SearchBot":true,"ChatGPT-User":true,"ClaudeBot":false,"Claude-SearchBot":true,"PerplexityBot":false,"Google-Extended":false,"Applebot-Extended":false,"CCBot":true,"Bytespider":true,"Meta-ExternalAgent":true,"Amazonbot":false}},"sitemap":null,"files":null,"derived":{"reachable":false,"home_class":"refused_us","robots_served":false,"robots_sitemap":false,"content_signal":false,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":[],"blocks_any_ai":false,"files":{"llms_txt":false,"agents_md":false,"agent_skills":false,"mcp_card":false,"api_catalog":false,"a2a_card":false},"any_capability":false,"ai_refused":[],"ai_thin":[],"rings":{"mcp_declared":false,"mcp_live":false,"mcp_auth":false,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":3,"fh":"www.whatsapp.com","fp_other_host":[]},"derive_v":5,"frame":"top10k","shard":0},{"v":1,"kind":"crawlcheck-census-record","run":18,"domain":"apple-dns.net","rank":34,"measured_at":"2026-11-13T18:48:18.168Z","home":{"browser":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"apple-dns.net","final_path":"/","ms":9,"identity":"Mobile browser","html":false},"gptbot":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"apple-dns.net","final_path":"/","ms":0,"identity":"GPTBot","html":false},"claudebot":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"apple-dns.net","final_path":"/","ms":1,"identity":"ClaudeBot","html":false},"oaisearch":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"apple-dns.net","final_path":"/","ms":0,"identity":"OAI-SearchBot","html":false},"perplexity":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"apple-dns.net","final_path":"/","ms":1,"identity":"PerplexityBot","html":false}},"robots":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"apple-dns.net","final_path":"/robots.txt","ms":0,"html":false,"text":null,"agents":null},"sitemap":null,"files":null,"derived":{"reachable":false,"home_class":"no_site","robots_served":false,"robots_sitemap":false,"content_signal":false,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":[],"blocks_any_ai":false,"files":{"llms_txt":false,"agents_md":false,"agent_skills":false,"mcp_card":false,"api_catalog":false,"a2a_card":false},"any_capability":false,"ai_refused":[],"ai_thin":[],"rings":{"mcp_declared":false,"mcp_live":false,"mcp_auth":false,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":3,"fh":"apple-dns.net","fp_other_host":[]},"derive_v":5,"frame":"top10k","shard":1},{"v":1,"kind":"crawlcheck-census-record","run":18,"domain":"aaplimg.com","rank":35,"measured_at":"2026-11-13T18:49:51.543Z","home":{"browser":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"aaplimg.com","final_path":"/","ms":10,"identity":"Mobile browser","html":false},"gptbot":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"aaplimg.com","final_path":"/","ms":2,"identity":"GPTBot","html":false},"claudebot":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"aaplimg.com","final_path":"/","ms":0,"identity":"ClaudeBot","html":false},"oaisearch":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"aaplimg.com","final_path":"/","ms":1,"identity":"OAI-SearchBot","html":false},"perplexity":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"aaplimg.com","final_path":"/","ms":1,"identity":"PerplexityBot","html":false}},"robots":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"aaplimg.com","final_path":"/robots.txt","ms":1,"html":false,"text":null,"agents":null},"sitemap":null,"files":null,"derived":{"reachable":false,"home_class":"no_site","robots_served":false,"robots_sitemap":false,"content_signal":false,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":[],"blocks_any_ai":false,"files":{"llms_txt":false,"agents_md":false,"agent_skills":false,"mcp_card":false,"api_catalog":false,"a2a_card":false},"any_capability":false,"ai_refused":[],"ai_thin":[],"rings":{"mcp_declared":false,"mcp_live":false,"mcp_auth":false,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":3,"fh":"aaplimg.com","fp_other_host":[]},"derive_v":5,"frame":"top10k","shard":2},{"v":1,"kind":"crawlcheck-census-record","run":18,"domain":"fastly.net","rank":36,"measured_at":"2026-11-13T18:50:46.493Z","home":{"browser":{"status":200,"content_type":"text/html","bytes":227467,"sha256":"3ba2910e15ee3879cd1c68292a769be2bbc7733dc84c26f2c36adfb8128dd57b","final_host":"www.fastly.com","final_path":"/","ms":87,"identity":"Mobile browser","html":true,"fp":"d749449900b33e78"},"gptbot":{"status":200,"content_type":"text/html","bytes":227467,"sha256":"3ba2910e15ee3879cd1c68292a769be2bbc7733dc84c26f2c36adfb8128dd57b","final_host":"www.fastly.com","final_path":"/","ms":64,"identity":"GPTBot","html":true,"fp":"d749449900b33e78"},"claudebot":{"status":200,"content_type":"text/html","bytes":227467,"sha256":"3ba2910e15ee3879cd1c68292a769be2bbc7733dc84c26f2c36adfb8128dd57b","final_host":"www.fastly.com","final_path":"/","ms":117,"identity":"ClaudeBot","html":true,"fp":"d749449900b33e78"},"oaisearch":{"status":200,"content_type":"text/html","bytes":227467,"sha256":"3ba2910e15ee3879cd1c68292a769be2bbc7733dc84c26f2c36adfb8128dd57b","final_host":"www.fastly.com","final_path":"/","ms":65,"identity":"OAI-SearchBot","html":true,"fp":"d749449900b33e78"},"perplexity":{"status":200,"content_type":"text/html","bytes":227467,"sha256":"3ba2910e15ee3879cd1c68292a769be2bbc7733dc84c26f2c36adfb8128dd57b","final_host":"www.fastly.com","final_path":"/","ms":88,"identity":"PerplexityBot","html":true,"fp":"d749449900b33e78"}},"robots":{"status":200,"content_type":"text/plain","bytes":398,"sha256":"23afd0d2868ab27736e15dd91bfa2b061fff83da62d60d131f335d21311dc6f3","final_host":"www.fastly.com","final_path":"/robots.txt","ms":176,"html":false,"text":"User-agent: *\nDisallow: /ja/*\nDisallow: /labs/*\nDisallow: /production-2019/*\nDisallow: /cassets/*\nDisallow: /resources?*\nDisallow: /altitude/2020\nDisallow: /astro-header/*\nDisallow: /*/astro-header/*\nDisallow: /feature-o11y/*\nDisallow: /*/feature-o11y/*\nDisallow: /learning/fastly-research-methodology-glossary\nSitemap: https://www.fastly.com/sitemap/sitemap-index.xml\nHost: https://www.fastly.com\n","agents":{"GPTBot":true,"OAI-SearchBot":true,"ChatGPT-User":true,"ClaudeBot":true,"Claude-SearchBot":true,"PerplexityBot":true,"Google-Extended":true,"Applebot-Extended":true,"CCBot":true,"Bytespider":true,"Meta-ExternalAgent":true,"Amazonbot":true}},"sitemap":{"status":200,"content_type":"application/xml","bytes":380,"sha256":"d4951d59091004f4b2646cfdce10fcca2264b3a7a4c3e6737af619655d916450","final_host":"www.fastly.com","final_path":"/sitemap/sitemap-index.xml","ms":28,"url":"https://www.fastly.com/sitemap/sitemap-index.xml","declared":true,"xml":true},"files":{"llms_txt":{"status":200,"content_type":"text/plain","bytes":16618,"sha256":"7d44aa6ffeddff002cd037eae424224165bc5c5c669c58e9c2e7497d58e2e028","final_host":"www.fastly.com","final_path":"/llms.txt","ms":403,"path":"/llms.txt","html":false,"json":null,"present":true},"agents_md":{"status":404,"content_type":"text/html","bytes":108841,"sha256":"ce0c37c10b2bb1f5da306f9fa16766177cb14dedb080de657415d82e86c715f8","final_host":"www.fastly.com","final_path":"/agents.md","ms":395,"path":"/agents.md","html":true,"json":null,"present":false},"agent_skills":{"status":404,"content_type":"text/html","bytes":108841,"sha256":"ce0c37c10b2bb1f5da306f9fa16766177cb14dedb080de657415d82e86c715f8","final_host":"www.fastly.com","final_path":"/.well-known/agent-skills/index.json","ms":309,"path":"/.well-known/agent-skills/index.json","html":true,"json":null,"present":false},"mcp_card":{"status":404,"content_type":"text/html","bytes":108841,"sha256":"ce0c37c10b2bb1f5da306f9fa16766177cb14dedb080de657415d82e86c715f8","final_host":"www.fastly.com","final_path":"/.well-known/mcp/server-card.json","ms":318,"path":"/.well-known/mcp/server-card.json","html":true,"json":null,"present":false},"api_catalog":{"status":404,"content_type":"text/html","bytes":108841,"sha256":"ce0c37c10b2bb1f5da306f9fa16766177cb14dedb080de657415d82e86c715f8","final_host":"www.fastly.com","final_path":"/.well-known/api-catalog","ms":385,"path":"/.well-known/api-catalog","html":true,"json":null,"present":false},"a2a_card":{"status":404,"content_type":"text/html","bytes":108841,"sha256":"ce0c37c10b2bb1f5da306f9fa16766177cb14dedb080de657415d82e86c715f8","final_host":"www.fastly.com","final_path":"/.well-known/agent-card.json","ms":393,"path":"/.well-known/agent-card.json","html":true,"json":null,"present":false}},"rings":{"v":1,"candidates":[],"mcp":[],"a2a":null,"webmcp":{"model_context":false,"bridge":false,"mcp_json":false,"link_rel":false,"any":false},"handshakes":0,"get_path":{"status":404,"content_type":"text/html","ms":375,"error":null},"get_host":{"status":530,"content_type":"text/plain","ms":9,"error":null}},"sf":[4,0,0,0,0,0,0,0,0],"derived":{"reachable":true,"home_class":"answers","robots_served":true,"robots_sitemap":true,"content_signal":false,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":[],"blocks_any_ai":false,"files":{"llms_txt":true,"agents_md":false,"agent_skills":false,"mcp_card":false,"api_catalog":false,"a2a_card":false},"any_capability":false,"ai_refused":[],"ai_thin":[],"aipref":0,"aipref_measured":true,"sf":[4,0,0,0,0,0,0,0,0],"rings":{"mcp_declared":false,"mcp_live":false,"mcp_auth":false,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":3,"fh":"www.fastly.com","fp_other_host":[],"fp":"d749449900b33e78","fp_bits":{"gptbot":0,"claudebot":0,"oaisearch":0,"perplexity":0}},"derive_v":5,"frame":"top10k","shard":3},{"v":1,"kind":"crawlcheck-census-record","run":18,"domain":"googleusercontent.com","rank":37,"measured_at":"2026-11-13T18:51:40.579Z","home":{"browser":{"status":404,"content_type":"text/html","bytes":1561,"sha256":"23926e9185d8d43c02807a838ffb373cc1977726094a4e46807c66ada9dd7660","final_host":"googleusercontent.com","final_path":"/","ms":26,"identity":"Mobile browser","html":true},"gptbot":{"status":404,"content_type":"text/html","bytes":1561,"sha256":"23926e9185d8d43c02807a838ffb373cc1977726094a4e46807c66ada9dd7660","final_host":"googleusercontent.com","final_path":"/","ms":8,"identity":"GPTBot","html":true},"claudebot":{"status":404,"content_type":"text/html","bytes":1561,"sha256":"23926e9185d8d43c02807a838ffb373cc1977726094a4e46807c66ada9dd7660","final_host":"googleusercontent.com","final_path":"/","ms":7,"identity":"ClaudeBot","html":true},"oaisearch":{"status":404,"content_type":"text/html","bytes":1561,"sha256":"23926e9185d8d43c02807a838ffb373cc1977726094a4e46807c66ada9dd7660","final_host":"googleusercontent.com","final_path":"/","ms":13,"identity":"OAI-SearchBot","html":true},"perplexity":{"status":404,"content_type":"text/html","bytes":1561,"sha256":"23926e9185d8d43c02807a838ffb373cc1977726094a4e46807c66ada9dd7660","final_host":"googleusercontent.com","final_path":"/","ms":9,"identity":"PerplexityBot","html":true}},"robots":{"status":404,"content_type":"text/html","bytes":1571,"sha256":"6adee83a3eba87d30fdca3997d27097697f2767805f222a0f78c3f04392004d2","final_host":"googleusercontent.com","final_path":"/robots.txt","ms":40,"html":true,"text":null,"agents":null},"sitemap":null,"files":null,"derived":{"reachable":false,"home_class":"other","robots_served":false,"robots_sitemap":false,"content_signal":false,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":[],"blocks_any_ai":false,"files":{"llms_txt":false,"agents_md":false,"agent_skills":false,"mcp_card":false,"api_catalog":false,"a2a_card":false},"any_capability":false,"ai_refused":[],"ai_thin":[],"rings":{"mcp_declared":false,"mcp_live":false,"mcp_auth":false,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":3,"fh":"googleusercontent.com","fp_other_host":[]},"derive_v":5,"frame":"top10k","shard":4},{"v":1,"kind":"crawlcheck-census-record","run":18,"domain":"doubleclick.net","rank":38,"measured_at":"2026-11-13T18:52:33.408Z","home":{"browser":{"status":200,"content_type":"text/html","bytes":124906,"sha256":"068f131b63dc05b4c1eae3b4c6748d756413b994a3aa8b82c1d2cad3ffdf48b3","final_host":"marketingplatform.google.com","final_path":"/about/enterprise","ms":836,"identity":"Mobile browser","html":true,"fp":"a65a9187a0c6597f"},"gptbot":{"status":200,"content_type":"text/html","bytes":124906,"sha256":"7f5398b40d75b21fb890679f0f456c677d08d4cb41a3d24063593586bc64d67d","final_host":"marketingplatform.google.com","final_path":"/about/enterprise","ms":585,"identity":"GPTBot","html":true,"fp":"a65a9187a0c6597f"},"claudebot":{"status":200,"content_type":"text/html","bytes":124906,"sha256":"9822129932f138712a5e3f7c8830eab8a7feb497c5a8309d0fdbcaea48884960","final_host":"marketingplatform.google.com","final_path":"/about/enterprise","ms":588,"identity":"ClaudeBot","html":true,"fp":"a65a9187a0c6597f"},"oaisearch":{"status":200,"content_type":"text/html","bytes":124906,"sha256":"d5a231a1d1a189a36217b84bdd9141cb3e50137ebd6b525e81ef26545a2bd7b5","final_host":"marketingplatform.google.com","final_path":"/about/enterprise","ms":421,"identity":"OAI-SearchBot","html":true,"fp":"a65a9187a0c6597f"},"perplexity":{"status":200,"content_type":"text/html","bytes":124906,"sha256":"6cba6e933f35d8bbdc9385c55debf50fad47528bda45856e725a8d034a010481","final_host":"marketingplatform.google.com","final_path":"/about/enterprise","ms":386,"identity":"PerplexityBot","html":true,"fp":"a65a9187a0c6597f"}},"robots":{"status":200,"content_type":"text/plain","bytes":6681,"sha256":"e8bf1ce79810901a9fef98deac93aaa532b1be5091ccd19f944dc8d86028a5c3","final_host":"doubleclick.net","final_path":"/robots.txt","ms":344,"html":false,"text":"User-agent: *\nUser-agent: Yandex\nDisallow: /search\nAllow: /search/about\nAllow: /search/howsearchworks\nDisallow: /sdch\nDisallow: /groups\nDisallow: /index.html?\nDisallow: /?\nDisallow: /goto?\nAllow: /?hl=\nDisallow: /?hl=*&\nAllow: /?hl=*&gws_rd=ssl$\nDisallow: /?hl=*&*&gws_rd=ssl\nAllow: /?gws_rd=ssl$\nAllow: /?pt1=true$\nDisallow: /imgres\nDisallow: /u/\nDisallow: /setprefs\nDisallow: /m?\nDisallow: /m/\nAllow:    /m/finance\nDisallow: /wml?\nDisallow: /wml/?\nDisallow: /wml/search?\nDisallow: /xhtml?\nDisallow: /xhtml/?\nDisallow: /xhtml/search?\nDisallow: /xml?\nDisallow: /imode?\nDisallow: /imode/?\nDisallow: /imode/search?\nDisallow: /jsky?\nDisallow: /jsky/?\nDisallow: /jsky/search?\nDisallow: /pda?\nDisallow: /pda/?\nDisallow: /pda/search?\nDisallow: /local?\nDisallow: /local_url\nDisallow: /products?\nDisallow: /product_\nDisallow: /products_\nDisallow: /products;\nDisallow: /print\nDisallow: /books/\nDisallow: /bkshp?*dq=\nDisallow: /bkshp?*q=\nDisallow: /books?*dq=\nDisallow: /books?*q=\nDisallow: /books?*qtid=\nDisallow: /books?*output=\nDisallow: /books?*pg=\nDisallow: /books?*jtp=\nDisallow: /books?*jscmd=\nDisallow: /books?*buy=\nDisallow: /books?*zoom=\nAllow: /books/about\nAllow: /books?*zoom=1\nAllow: /books?*zoom=5\nAllow: /books/content?*zoom=1\nAllow: /books/content?*zoom=5\nDisallow: /patents?\nDisallow: /patents/download/\nDisallow: /patents/pdf/\nDisallow: /patents/related/\nDisallow: /scholar\nDisallow: /citations?\nAllow: /citations?user=\nAllow: /citations?view_op=new_profile\nAllow: /citations?view_op=top_venues\nAllow: /scholar_share\nDisallow: /s?\nDisallow: /maps?\nAllow: /maps?daddr=\nAllow: /maps?entry=wc\nAllow: /maps?f=\nAllow: /maps?hl=\nAllow: /maps?q=\nAllow: /maps?saddr=\nAllow: /maps?sid=\nAllow: /maps?*output=classic\nAllow: /maps?*file=\nDisallow: /mapslt?\nDisallow: /maphp?\nDisallow: /maps/\nAllow: /maps/$\nAllow: /maps/@\nAllow: /maps/?daddr=\nAllow: /maps/?entry=wc\nAllow: /maps/?f=\nAllow: /maps/?hl=\nAllow: /maps/?q=\nAllow: /maps/?saddr=\nAllow: /maps/?sid=\nAllow: /maps/api/staticmap?\nAllow: /maps/search/\nAllow: /maps/sitemap.xml\nAllow: /maps/sitemaps/\nAllow: /maps/dir/\nAllow: /maps/d/\nAllow: /maps/reserve\nAllow: /maps/about\nAllow: /maps/contrib/\nAllow: /maps/match\nAllow: /maps/ms?\nAllow: /maps/place/\nAllow: /maps/_/\nAllow: /search?*tbm=map\nAllow: /maps/vt?\nAllow: /maps/preview\nDisallow: /maps/api/js/\nAllow: /maps/api/js\nDisallow: /mld?\nDisallow: /staticmap?\nDisallow: /help/maps/streetview/partners/welcome/\nDisallow: /help/maps/indoormaps/partners/\nDisallow: /lochp?\nDisallow: /ie?\nDisallow: /uds/\nDisallow: /transit?\nDisallow: /trends?\nDisallow: /trends/music?\nDisallow: /trends/hottrends?\nDisallow: /trends/viz?\nDisallow: /trends/embed.js?\nDisallow: /trends/fetchComponent?\nDisallow: /trends/beta\nDisallow: /trends/topics\nDisallow: /trends/explore?\nDisallow: /trends/api\nDisallow: /musica\nDisallow: /musicl\nDisallow: /musics\nDisallow: /urchin_test/\nDisallow: /movies?\nDisallow: /wapsearch?\nDisallow: /reviews/search?\nDisallow: /cbk\nDisallow: /profiles/me\nDisallow: /s2/profiles/me\nAllow: /s2/profiles\nAllow: /s2/oz\nAllow: /s2/photos\nAllow: /s2/search/social\nAllow: /s2/static\nDisallow: /s2\nDisallow: /transconsole/portal/\nDisallow: /aclk\nDisallow: /tbproxy/\nDisallow: /support/forum/search?\nDisallow: /reviews/polls/\nDisallow: /hosted/images/\nDisallow: /accounts/ClientLogin\nDisallow: /accounts/ClientAuth\nDisallow: /accounts/o8\nAllow: /accounts/o8/id\nDisallow: /quality_form?\nDisallow: /labs/popgadget/search\nDisallow: /compressiontest/\nDisallow: /analytics/feeds/\nDisallow: /analytics/partners/comments/\nDisallow: /analytics/portal/\nDisallow: /analytics/uploads/\nAllow: /alerts/manage\nAllow: /alerts/remove\nDisallow: /alerts/\nAllow: /alerts/$\nDisallow: /phone/compare/?\nDisallow: /travel/clk\nDisallow: /travel/lodging/clk\nDisallow: /travel/lodging/imp\nDisallow: /travel/entity\nDisallow: /travel/search\nDisallow: /travel/flights/booking\nDisallow: /travel/flights/s/\nDisallow: /travel/flights/search\nDisallow: /travel/hotels/stories\nDisallow: /travel/hotels/*/stories\nDisallow: /travel/story\nDisallow: /hotelfinder/rpc\nDisallow: /hotels/rpc\nDisallow: /evaluation/\nDisallow: /forms/perks/\nDisallow: /shopping/suppliers/search\nDisallow: /edu/cs4hs/\nDisallow: /trustedstores/s/\nDisallow: /trustedstores/tm2\nDisallow: /trustedstores/verify\nDisallow: /shopping?\nDisallow: /shopping/product/\nDisallow: /shopping/seller\nDisallow: /shopping/ratings/account/metrics\nDisallow: /shopping/ratings/merchant/immersivedetails\nDisallow: /shopping/reviewer\nDisallow: /shopping/search\nDisallow: /shopping/deals\nAllow: /shopping?udm=28$\nAllow: /shopping?udm=shop$\nDisallow: /storefront\nDisallow: /storepicker\nDisallow: /about/careers/applications/candidate-prep\nDisallow: /about/careers/applications/connect-with-a-googler\nDisallow: /about/careers/applications/jobs/results?page=\nDisallow: /about/careers/applications/jobs/results/?page=\nDisallow: /about/careers/applications/jobs/results?*&page=\nDisallow: /about/careers/applications/jobs/results/?*&page=\nDisallow: /landing/signout.html\nDisallow: /gallery/\nDisallow: /landing/now/ontap/\nAllow: /maps/reserve\nAllow: /maps/reserve/partners\nDisallow: /maps/reserve/api/\nDisallow: /maps/reserve/search\nDisallow: /maps/reserve/bookings\nDisallow: /maps/reserve/settings\nDisallow: /maps/reserve/manage\nDisallow: /maps/reserve/payment\nDisallow: /maps/reserve/receipt\nDisallow: /maps/reserve/sellersignup\nDisallow: /maps/reserve/feedback\nDisallow: /maps/reserve/terms\nDisallow: /maps/reserve/m/\nDisallow: /maps/reserve/b/\nDisallow: /maps/reserve/partner-dashboard\nDisallow: /local/cars\nDisallow: /local/dealership/\nDisallow: /local/dining/\nDisallow: /local/place/products/\nDisallow: /local/place/reviews/\nDisallow: /local/place/rap/\nDisallow: /local/tab/\nDisallow: /localservices/\nAllow: /localservices/coupon/\nDisallow: /nonprofits/account/\nDisallow: /uviewer\nDisallow: /landing/cmsnext-root/\n\n# AdsBot\nUser-agent: AdsBot-Google\nDisallow: /maps/api/js/\nAllow: /maps/api/js\nDisallow: /maps/api/place/js/\nDisallow: /maps/api/staticmap\nDisallow: /maps/api/streetview\n\n# New user agent groups must also have a user agent reference in the global (*)\n# group. See \"Order of precedence\" section in\n# https://goo.gle/rep#order-of-precedence-for-user-agents\nUser-agent: Yandex\nDisallow: /about/careers/applications/jobs/results\nDisallow: /about/careers/applications-a/jobs/results\n\n# Crawlers of certain social media sites are allowed to access page markup when\n# google.com/imgres* links are shared. To learn more, please contact\n# images-robots-allowlist@google.com.\nUser-agent: facebookexternalhit\nUser-agent: Twitterbot\nAllow: /imgres\nAllow: /search\nDisallow: /groups\nDisallow: /hosted/images/\nDisallow: /m/\n\nSitemap: https://www.google.com/sitemap.xml\n","agents":{"GPTBot":true,"OAI-SearchBot":true,"ChatGPT-User":true,"ClaudeBot":true,"Claude-SearchBot":true,"PerplexityBot":true,"Google-Extended":true,"Applebot-Extended":true,"CCBot":true,"Bytespider":true,"Meta-ExternalAgent":true,"Amazonbot":true}},"sitemap":{"status":200,"content_type":"text/xml","bytes":2047,"sha256":"0bc80feb7f7900da84ecf0cafdc78fb4dc3a9550413b40722eb73c6c02d0f49c","final_host":"www.google.com","final_path":"/sitemap.xml","ms":278,"url":"https://www.google.com/sitemap.xml","declared":true,"xml":true},"files":{"llms_txt":{"status":404,"content_type":"text/html","bytes":1569,"sha256":"47d71ed470dfe1ca13eb2af8fa019b95871a1bd1381a518ed71359e69b9c3cc7","final_host":"doubleclick.net","final_path":"/llms.txt","ms":81,"path":"/llms.txt","html":true,"json":null,"present":false},"agents_md":{"status":404,"content_type":"text/html","bytes":1570,"sha256":"f08e316602488d86581ce85b3076a13c82fc3f97cdac3deba23801fb66ca908d","final_host":"doubleclick.net","final_path":"/agents.md","ms":281,"path":"/agents.md","html":true,"json":null,"present":false},"agent_skills":{"status":404,"content_type":"text/html","bytes":1596,"sha256":"eee9f4f975e761645c6c385ec645439385500d0062bff7170ed09a8835d6f92b","final_host":"doubleclick.net","final_path":"/.well-known/agent-skills/index.json","ms":81,"path":"/.well-known/agent-skills/index.json","html":true,"json":null,"present":false},"mcp_card":{"status":404,"content_type":"text/html","bytes":1593,"sha256":"e5db4a1d5f7596a7d62e54e192ab48bbf195116fa33a2c7e160fdaf6131945a8","final_host":"doubleclick.net","final_path":"/.well-known/mcp/server-card.json","ms":82,"path":"/.well-known/mcp/server-card.json","html":true,"json":null,"present":false},"api_catalog":{"status":404,"content_type":"text/html","bytes":1584,"sha256":"7de25ab2e3ce288cf29d538f80ed48bf0d5759f87b613e29ed345db4675698aa","final_host":"doubleclick.net","final_path":"/.well-known/api-catalog","ms":81,"path":"/.well-known/api-catalog","html":true,"json":null,"present":false},"a2a_card":{"status":404,"content_type":"text/html","bytes":1588,"sha256":"64d2189fbe1531e7fb6ab2e6040a932d87ae1feb811d4781df85ea7ef6b4d651","final_host":"doubleclick.net","final_path":"/.well-known/agent-card.json","ms":81,"path":"/.well-known/agent-card.json","html":true,"json":null,"present":false}},"rings":{"v":1,"candidates":[],"mcp":[],"a2a":null,"webmcp":{"model_context":false,"bridge":false,"mcp_json":false,"link_rel":false,"any":false},"handshakes":0,"get_path":{"status":404,"content_type":"text/html","ms":81,"error":null},"get_host":{"status":530,"content_type":"text/plain","ms":122,"error":null}},"sf":[1,0,0,1,0,0,3,0,0],"derived":{"reachable":true,"home_class":"answers","robots_served":true,"robots_sitemap":true,"content_signal":false,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":[],"blocks_any_ai":false,"files":{"llms_txt":false,"agents_md":false,"agent_skills":false,"mcp_card":false,"api_catalog":false,"a2a_card":false},"any_capability":false,"ai_refused":[],"ai_thin":[],"aipref":0,"aipref_measured":true,"sf":[1,0,0,1,0,0,3,0,0],"rings":{"mcp_declared":false,"mcp_live":false,"mcp_auth":false,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":3,"fh":"marketingplatform.google.com","fp_other_host":[],"fp":"a65a9187a0c6597f","fp_bits":{"gptbot":0,"claudebot":0,"oaisearch":0,"perplexity":0}},"derive_v":5,"frame":"top10k","shard":5},{"v":1,"kind":"crawlcheck-census-record","run":18,"domain":"cloud.microsoft","rank":39,"measured_at":"2026-11-13T18:53:15.625Z","home":{"browser":{"status":200,"content_type":"text/html","bytes":334646,"sha256":"315185124fb1f89e40149c03420c6b66e80237ed96b462c6aaaabc11a088f2e1","final_host":"m365.cloud.microsoft","final_path":"/","ms":572,"identity":"Desktop browser (device re-check)","html":true,"fp":"ca43656b22f242a1"},"gptbot":{"status":200,"content_type":"text/html","bytes":334646,"sha256":"315185124fb1f89e40149c03420c6b66e80237ed96b462c6aaaabc11a088f2e1","final_host":"m365.cloud.microsoft","final_path":"/","ms":343,"identity":"GPTBot","html":true,"fp":"ca43656b22f242a1"},"claudebot":{"status":200,"content_type":"text/html","bytes":334501,"sha256":"2cf17b95dee31da9a7b26939c506dd2cbbabdc65ee0a7daa8905d72908b7c6f6","final_host":"m365.cloud.microsoft","final_path":"/","ms":347,"identity":"ClaudeBot","html":true,"fp":"ccc3656b22eb42a0"},"oaisearch":{"status":200,"content_type":"text/html","bytes":334646,"sha256":"315185124fb1f89e40149c03420c6b66e80237ed96b462c6aaaabc11a088f2e1","final_host":"m365.cloud.microsoft","final_path":"/","ms":319,"identity":"OAI-SearchBot","html":true,"fp":"ca43656b22f242a1"},"perplexity":{"status":200,"content_type":"text/html","bytes":334646,"sha256":"315185124fb1f89e40149c03420c6b66e80237ed96b462c6aaaabc11a088f2e1","final_host":"m365.cloud.microsoft","final_path":"/","ms":296,"identity":"PerplexityBot","html":true,"fp":"ca43656b22f242a1"},"browser_mobile":{"status":200,"content_type":"text/html","bytes":307167,"sha256":"1a53e35f7eab761411951eb9447323495107a769ef4995bd2792082aed7020d0","final_host":"m365.cloud.microsoft","final_path":"/","ms":525,"identity":"Mobile browser","html":true,"fp":"ca70646b2eab42a1"}},"robots":{"status":200,"content_type":"text/html","bytes":334646,"sha256":"315185124fb1f89e40149c03420c6b66e80237ed96b462c6aaaabc11a088f2e1","final_host":"m365.cloud.microsoft","final_path":"/","ms":638,"html":true,"text":null,"agents":null},"sitemap":{"status":200,"content_type":"text/html","bytes":334501,"sha256":"2cf17b95dee31da9a7b26939c506dd2cbbabdc65ee0a7daa8905d72908b7c6f6","final_host":"m365.cloud.microsoft","final_path":"/","ms":207,"url":"https://cloud.microsoft/sitemap.xml","declared":false,"xml":false},"files":{"llms_txt":{"status":200,"content_type":"text/html","bytes":334646,"sha256":"315185124fb1f89e40149c03420c6b66e80237ed96b462c6aaaabc11a088f2e1","final_host":"m365.cloud.microsoft","final_path":"/","ms":66,"path":"/llms.txt","html":true,"json":null,"present":false},"agents_md":{"status":200,"content_type":"text/html","bytes":334646,"sha256":"315185124fb1f89e40149c03420c6b66e80237ed96b462c6aaaabc11a088f2e1","final_host":"m365.cloud.microsoft","final_path":"/","ms":140,"path":"/agents.md","html":true,"json":null,"present":false},"agent_skills":{"status":200,"content_type":"text/html","bytes":334501,"sha256":"2cf17b95dee31da9a7b26939c506dd2cbbabdc65ee0a7daa8905d72908b7c6f6","final_host":"m365.cloud.microsoft","final_path":"/","ms":63,"path":"/.well-known/agent-skills/index.json","html":true,"json":null,"present":false},"mcp_card":{"status":200,"content_type":"text/html","bytes":334646,"sha256":"315185124fb1f89e40149c03420c6b66e80237ed96b462c6aaaabc11a088f2e1","final_host":"m365.cloud.microsoft","final_path":"/","ms":353,"path":"/.well-known/mcp/server-card.json","html":true,"json":null,"present":false},"api_catalog":{"status":200,"content_type":"text/html","bytes":334501,"sha256":"2cf17b95dee31da9a7b26939c506dd2cbbabdc65ee0a7daa8905d72908b7c6f6","final_host":"m365.cloud.microsoft","final_path":"/","ms":211,"path":"/.well-known/api-catalog","html":true,"json":null,"present":false},"a2a_card":{"status":200,"content_type":"text/html","bytes":334501,"sha256":"2cf17b95dee31da9a7b26939c506dd2cbbabdc65ee0a7daa8905d72908b7c6f6","final_host":"m365.cloud.microsoft","final_path":"/","ms":96,"path":"/.well-known/agent-card.json","html":true,"json":null,"present":false}},"device_variant":true,"rings":{"v":1,"candidates":[],"mcp":[],"a2a":null,"webmcp":{"model_context":false,"bridge":false,"mcp_json":false,"link_rel":false,"any":false},"handshakes":0,"get_path":{"status":200,"content_type":"text/html","ms":103,"error":null},"get_host":{"status":530,"content_type":"text/plain","ms":49,"error":null}},"sf":[1,0,0,0,0,0,2,16,1],"derived":{"reachable":true,"home_class":"answers","robots_served":false,"robots_sitemap":false,"content_signal":false,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":[],"blocks_any_ai":false,"files":{"llms_txt":false,"agents_md":false,"agent_skills":false,"mcp_card":false,"api_catalog":false,"a2a_card":false},"any_capability":false,"ai_refused":[],"ai_thin":[],"sf":[1,0,0,0,0,0,2,16,1],"rings":{"mcp_declared":false,"mcp_live":false,"mcp_auth":false,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":3,"fh":"m365.cloud.microsoft","fp_other_host":[],"fp":"ca43656b22f242a1","fp_bits":{"gptbot":0,"claudebot":7,"oaisearch":0,"perplexity":0}},"derive_v":5,"frame":"top10k","shard":6},{"v":1,"kind":"crawlcheck-census-record","run":18,"domain":"office.net","rank":40,"measured_at":"2026-11-13T18:55:41.951Z","home":{"browser":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"office.net","final_path":"/","ms":5,"identity":"Mobile browser","html":false},"gptbot":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"office.net","final_path":"/","ms":0,"identity":"GPTBot","html":false},"claudebot":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"office.net","final_path":"/","ms":1,"identity":"ClaudeBot","html":false},"oaisearch":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"office.net","final_path":"/","ms":1,"identity":"OAI-SearchBot","html":false},"perplexity":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"office.net","final_path":"/","ms":0,"identity":"PerplexityBot","html":false}},"robots":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"office.net","final_path":"/robots.txt","ms":0,"html":false,"text":null,"agents":null},"sitemap":null,"files":null,"derived":{"reachable":false,"home_class":"no_site","robots_served":false,"robots_sitemap":false,"content_signal":false,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":[],"blocks_any_ai":false,"files":{"llms_txt":false,"agents_md":false,"agent_skills":false,"mcp_card":false,"api_catalog":false,"a2a_card":false},"any_capability":false,"ai_refused":[],"ai_thin":[],"rings":{"mcp_declared":false,"mcp_live":false,"mcp_auth":false,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":3,"fh":"office.net","fp_other_host":[]},"derive_v":5,"frame":"top10k","shard":7},{"v":1,"kind":"crawlcheck-census-record","run":18,"domain":"microsoftonline.com","rank":41,"measured_at":"2026-11-13T18:46:27.088Z","home":{"browser":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"microsoftonline.com","final_path":"/","ms":6,"identity":"Mobile browser","html":false},"gptbot":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"microsoftonline.com","final_path":"/","ms":0,"identity":"GPTBot","html":false},"claudebot":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"microsoftonline.com","final_path":"/","ms":1,"identity":"ClaudeBot","html":false},"oaisearch":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"microsoftonline.com","final_path":"/","ms":0,"identity":"OAI-SearchBot","html":false},"perplexity":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"microsoftonline.com","final_path":"/","ms":0,"identity":"PerplexityBot","html":false}},"robots":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"microsoftonline.com","final_path":"/robots.txt","ms":1,"html":false,"text":null,"agents":null},"sitemap":null,"files":null,"derived":{"reachable":false,"home_class":"no_site","robots_served":false,"robots_sitemap":false,"content_signal":false,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":[],"blocks_any_ai":false,"files":{"llms_txt":false,"agents_md":false,"agent_skills":false,"mcp_card":false,"api_catalog":false,"a2a_card":false},"any_capability":false,"ai_refused":[],"ai_thin":[],"rings":{"mcp_declared":false,"mcp_live":false,"mcp_auth":false,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":3,"fh":"microsoftonline.com","fp_other_host":[]},"derive_v":5,"frame":"top10k","shard":0},{"v":1,"kind":"crawlcheck-census-record","run":18,"domain":"trafficmanager.net","rank":42,"measured_at":"2026-11-13T18:48:18.168Z","home":{"browser":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"trafficmanager.net","final_path":"/","ms":6,"identity":"Mobile browser","html":false},"gptbot":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"trafficmanager.net","final_path":"/","ms":1,"identity":"GPTBot","html":false},"claudebot":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"trafficmanager.net","final_path":"/","ms":1,"identity":"ClaudeBot","html":false},"oaisearch":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"trafficmanager.net","final_path":"/","ms":0,"identity":"OAI-SearchBot","html":false},"perplexity":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"trafficmanager.net","final_path":"/","ms":0,"identity":"PerplexityBot","html":false}},"robots":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"trafficmanager.net","final_path":"/robots.txt","ms":1,"html":false,"text":null,"agents":null},"sitemap":null,"files":null,"derived":{"reachable":false,"home_class":"no_site","robots_served":false,"robots_sitemap":false,"content_signal":false,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":[],"blocks_any_ai":false,"files":{"llms_txt":false,"agents_md":false,"agent_skills":false,"mcp_card":false,"api_catalog":false,"a2a_card":false},"any_capability":false,"ai_refused":[],"ai_thin":[],"rings":{"mcp_declared":false,"mcp_live":false,"mcp_auth":false,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":3,"fh":"trafficmanager.net","fp_other_host":[]},"derive_v":5,"frame":"top10k","shard":1},{"v":1,"kind":"crawlcheck-census-record","run":18,"domain":"sharepoint.com","rank":43,"measured_at":"2026-11-13T18:49:51.543Z","home":{"browser":{"status":200,"content_type":"text/html","bytes":492511,"sha256":"42eda39f2dadb005b02f78a1ce87d204e2849ebaeb3e92373fcd0b70c9fc00b7","final_host":"www.microsoft.com","final_path":"/nl-nl/microsoft-365/sharepoint/collaboration","ms":618,"identity":"Mobile browser","html":true,"fp":"aaff58619f58f311"},"gptbot":{"status":200,"content_type":"text/html","bytes":492511,"sha256":"754a40d2ab90342084aee9480d6dc140d0521c18878a2f73ffe08f79485e9c7c","final_host":"www.microsoft.com","final_path":"/nl-nl/microsoft-365/sharepoint/collaboration","ms":618,"identity":"GPTBot","html":true,"fp":"aaff58619f58d311"},"claudebot":{"status":200,"content_type":"text/html","bytes":492511,"sha256":"1cf8fa144681a0274bdad3aff66a68e2eeb88b577a075660848a3267ce4cfd47","final_host":"www.microsoft.com","final_path":"/nl-nl/microsoft-365/sharepoint/collaboration","ms":502,"identity":"ClaudeBot","html":true,"fp":"aaff58619f58d311"},"oaisearch":{"status":200,"content_type":"text/html","bytes":492511,"sha256":"8ecb1f292503e9714de543222ec9f144c0eec8645d10bface404ae739f54c12f","final_host":"www.microsoft.com","final_path":"/nl-nl/microsoft-365/sharepoint/collaboration","ms":489,"identity":"OAI-SearchBot","html":true,"fp":"aaff58619f58d311"},"perplexity":{"status":200,"content_type":"text/html","bytes":492511,"sha256":"f4cc1d19a7c938152bddc2a019d1f1e3fb64f9992695ee647c3c589c49407f2a","final_host":"www.microsoft.com","final_path":"/nl-nl/microsoft-365/sharepoint/collaboration","ms":474,"identity":"PerplexityBot","html":true,"fp":"a2ff58619f58d311"}},"robots":{"status":520,"content_type":"text/plain","bytes":16,"sha256":"b59e493586b4118c563aaa751e04a7f7bc1c9741d241e0c1d3ba9405c323c570","final_host":"www.microsoft.com","final_path":"/nl-nl/microsoft-365/sharepoint/collaboration","ms":369,"html":false,"text":null,"agents":null},"sitemap":{"status":520,"content_type":"text/plain","bytes":16,"sha256":"b59e493586b4118c563aaa751e04a7f7bc1c9741d241e0c1d3ba9405c323c570","final_host":"www.microsoft.com","final_path":"/nl-nl/microsoft-365/sharepoint/collaboration","ms":234,"url":"https://sharepoint.com/sitemap.xml","declared":false,"xml":false},"files":{"llms_txt":{"status":520,"content_type":"text/plain","bytes":16,"sha256":"b59e493586b4118c563aaa751e04a7f7bc1c9741d241e0c1d3ba9405c323c570","final_host":"www.microsoft.com","final_path":"/nl-nl/microsoft-365/sharepoint/collaboration","ms":392,"path":"/llms.txt","html":false,"json":null,"present":false},"agents_md":{"status":520,"content_type":"text/plain","bytes":16,"sha256":"b59e493586b4118c563aaa751e04a7f7bc1c9741d241e0c1d3ba9405c323c570","final_host":"www.microsoft.com","final_path":"/nl-nl/microsoft-365/sharepoint/collaboration","ms":354,"path":"/agents.md","html":false,"json":null,"present":false},"agent_skills":{"status":400,"content_type":null,"bytes":0,"sha256":"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855","final_host":"sharepoint.com","final_path":"/.well-known/agent-skills/index.json","ms":142,"path":"/.well-known/agent-skills/index.json","html":false,"json":null,"present":false},"mcp_card":{"status":400,"content_type":null,"bytes":0,"sha256":"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855","final_host":"sharepoint.com","final_path":"/.well-known/mcp/server-card.json","ms":112,"path":"/.well-known/mcp/server-card.json","html":false,"json":null,"present":false},"api_catalog":{"status":400,"content_type":null,"bytes":0,"sha256":"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855","final_host":"sharepoint.com","final_path":"/.well-known/api-catalog","ms":434,"path":"/.well-known/api-catalog","html":false,"json":null,"present":false},"a2a_card":{"status":400,"content_type":null,"bytes":0,"sha256":"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855","final_host":"sharepoint.com","final_path":"/.well-known/agent-card.json","ms":123,"path":"/.well-known/agent-card.json","html":false,"json":null,"present":false}},"rings":{"v":1,"candidates":[],"mcp":[],"a2a":null,"webmcp":{"model_context":false,"bridge":false,"mcp_json":false,"link_rel":false,"any":false},"handshakes":0,"get_path":{"status":520,"content_type":"text/plain","ms":180,"error":null},"get_host":{"status":200,"content_type":"text/html","ms":2381,"error":null}},"sf":[4,0,0,0,0,0,4,0,0],"derived":{"reachable":true,"home_class":"answers","robots_served":false,"robots_sitemap":false,"content_signal":false,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":[],"blocks_any_ai":false,"files":{"llms_txt":false,"agents_md":false,"agent_skills":false,"mcp_card":false,"api_catalog":false,"a2a_card":false},"any_capability":false,"ai_refused":[],"ai_thin":[],"sf":[4,0,0,0,0,0,4,0,0],"rings":{"mcp_declared":false,"mcp_live":false,"mcp_auth":false,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":3,"fh":"www.microsoft.com","fp_other_host":[],"fp":"aaff58619f58f311","fp_bits":{"gptbot":1,"claudebot":1,"oaisearch":1,"perplexity":2}},"derive_v":5,"frame":"top10k","shard":2},{"v":1,"kind":"crawlcheck-census-record","run":18,"domain":"netflix.com","rank":44,"measured_at":"2026-11-13T18:50:46.493Z","home":{"browser":{"status":200,"content_type":"text/html","bytes":744485,"sha256":"f58c6ff4b808e3cf8800d0b0e1bc75370d977300a94ff36973bcaef848c4eee9","final_host":"www.netflix.com","final_path":"/","ms":448,"identity":"Mobile browser","html":true,"fp":"c492080d141e988f"},"gptbot":{"status":200,"content_type":"text/html","bytes":734112,"sha256":"d69072260e6f47321a94e5de19bd5ddc0bb7c981f7fdc34b0376ffcaebe417c2","final_host":"www.netflix.com","final_path":"/","ms":473,"identity":"GPTBot","html":true,"fp":"c492080d141e988f"},"claudebot":{"status":200,"content_type":"text/html","bytes":733924,"sha256":"1e71bf60d1bb4bacdb11801459b827c4a55fe139678d0ce8c3ff32a54c2c49c1","final_host":"www.netflix.com","final_path":"/","ms":337,"identity":"ClaudeBot","html":true,"fp":"449a0805105e988f"},"oaisearch":{"status":200,"content_type":"text/html","bytes":734141,"sha256":"ee37503bc039f61b47629f83938af6046ae51dd42a12cd9f3abed0944103070d","final_host":"www.netflix.com","final_path":"/","ms":360,"identity":"OAI-SearchBot","html":true,"fp":"c492080d141e988f"},"perplexity":{"status":200,"content_type":"text/html","bytes":734014,"sha256":"ac983355f05c219f66179e5da21592ad2c6901b0a18fc56fea5760ad2c1afe1f","final_host":"www.netflix.com","final_path":"/","ms":369,"identity":"PerplexityBot","html":true,"fp":"449a0805105e988f"}},"robots":{"status":200,"content_type":"text/plain","bytes":3790,"sha256":"1e14fa0284e9757b52b48dde38b390effae68afdd72447088e72f18f49b1aa09","final_host":"www.netflix.com","final_path":"/robots.txt","ms":86,"html":false,"text":"User-agent: *\nDisallow: /\n\nUser-agent: googlebot\nUser-agent: Googlebot-Video\nUser-agent: Applebot\nUser-agent: bingbot\nUser-agent: Baiduspider\nUser-agent: Baiduspider-mobile\nUser-agent: Baiduspider-video\nUser-agent: Baiduspider-image\nUser-agent: NaverBot\nUser-agent: Y!J-WSC\nUser-agent: YJ-WSC\nUser-agent: Yeti\nUser-agent: Yandex\nUser-agent: YandexBot\nUser-agent: YandexMobileBot\nUser-agent: YandexVideo\nUser-agent: YandexWebmaster\nUser-agent: YandexSitelinks\nUser-agent: SeznamBot\nUser-agent: facebookexternalhit\nUser-agent: GPTBot\nUser-agent: ChatGPT-User\nUser-agent: OAI-SearchBot\nUser-agent: Google-Extended\nUser-agent: ClaudeBot\nUser-agent: Claude-User\nUser-agent: Claude-SearchBot\nUser-agent: PerplexityBot\nAllow: /\n\nDisallow: /accountstatus\nDisallow: /AccountStatus\nDisallow: /aui/inbound\nDisallow: /authenticate\nDisallow: /autologin\nDisallow: /clearcookies\nDisallow: /companies\nDisallow: /editpayment\nDisallow: /emailunsubscribe\nDisallow: /error\nDisallow: /eula\nDisallow: /geooverride\nDisallow: /help\nDisallow: /imagelibrary\nDisallow: /learnmorelayer\nDisallow: /learnmorelayertv\nDisallow: /login\nDisallow: /loginhelp\nDisallow: /loginhelp/lookup\nDisallow: /loginhelpsucess\nDisallow: /LoginHelp\nDisallow: /password\nDisallow: /logout\nDisallow: /Logout\nDisallow: /mcd\nDisallow: /modernizr\nDisallow: /n/\nDisallow: /notamember\nDisallow: /notfound\nDisallow: /notices\nDisallow: /nrdapp\nDisallow: /optout\nDisallow: /overviewblockseeother\nDisallow: /popup/codewhatisthis\nDisallow: /popupdetails\nDisallow: /PopupDetails\nDisallow: /popupprivacypolicy\nDisallow: /privacypolicychanges\nDisallow: /registration\nDisallow: /rememberme\nDisallow: /signout\nDisallow: /signurl\nDisallow: /subscriptioncancel\nDisallow: /tastesurvey\nDisallow: /termsofusechanges\nDisallow: /tvsignup\nDisallow: /upcomingevents\nDisallow: /verifyidentity\nDisallow: /whysecure\n\nDisallow: /arabic\nDisallow: /Arabic\nDisallow: /chinese\nDisallow: /Chinese\nDisallow: /korean\nDisallow: /Korean\n\nDisallow: /airtel\nDisallow: /anan\nDisallow: /bouyguestelecom\nDisallow: /britishairways\nDisallow: /brutus\nDisallow: /comhem\nDisallow: /courts\nDisallow: /csl\nDisallow: /elisa\nDisallow: /entertain\nDisallow: /FireTV\nDisallow: /firetv\nDisallow: /freemonth\nDisallow: /kpn\nDisallow: /lg\nDisallow: /maxis\nDisallow: /Maxis\nDisallow: /meo\nDisallow: /Meo\nDisallow: /orangefrance\nDisallow: /Panasonic\nDisallow: /panasonic\nDisallow: /playstation\nDisallow: /proximus\nDisallow: /qantas\nDisallow: /samsung\nDisallow: /Sony\nDisallow: /sony\nDisallow: /talktalk\nDisallow: /tdc\nDisallow: /telenor\nDisallow: /telfort\nDisallow: /tim\nDisallow: /virginaustralia\nDisallow: /vodafone\nDisallow: /vodafonedemobilelaunch\nDisallow: /xboxone\nDisallow: /xfinity\nDisallow: /xs4all\nDisallow: /ziggo\n\nDisallow: /accountaccess\nDisallow: /AccountAccess\nDisallow: /BillingActivity\nDisallow: /browse\nDisallow: /browse/*\nAllow: /browse/genre/*\nDisallow: /CancelPlan\nDisallow: /ChangePlan\nDisallow: /changeplan\nDisallow: /deviceManagement\nDisallow: /DoNotTest\nDisallow: /EditProfiles\nDisallow: /email\nDisallow: /EmailPreferences\nDisallow: /entrytrap\nDisallow: /HdToggle\nDisallow: /LanguagePreferences\nDisallow: /ManageDevices\nDisallow: /ManageProfiles\nDisallow: /MoviesYouveSeen\nDisallow: /NewWatchInstantlyRSS\nDisallow: /NewWatchInstantlyRSS/*\nDisallow: /payment\nDisallow: /Payment\nDisallow: /phonenumber\nDisallow: /pin\nDisallow: /profiles\nDisallow: /profiles/*\nDisallow: /ProfilesGate\nDisallow: /search\nDisallow: /search/*\nDisallow: /viewingactivity\nDisallow: /WiViewingActivity\nDisallow: /yourAccount\nDisallow: /youraccount\nDisallow: /YourAccount\nDisallow: /YourAccountPayment\n\nUser-agent: AdsBot-Google\nUser-agent: Twitterbot\nUser-agent: Adidxbot\nAllow: /\n\nUser-agent: Yahoo Pipes 1.0\nUser-agent: Facebot\nDisallow: /\nAllow: /tudum\n\nSitemap: https://www.netflix.com/sitemap/index\n","agents":{"GPTBot":true,"OAI-SearchBot":true,"ChatGPT-User":true,"ClaudeBot":true,"Claude-SearchBot":true,"PerplexityBot":true,"Google-Extended":true,"Applebot-Extended":true,"CCBot":false,"Bytespider":false,"Meta-ExternalAgent":false,"Amazonbot":false}},"sitemap":{"status":200,"content_type":"text/xml","bytes":2567966,"sha256":"e874ecfa559e3ae73ea1f89e378b9e1976fb48764aa5555b5567e35baa17336b","final_host":"www.netflix.com","final_path":"/sitemap/index","ms":299,"url":"https://www.netflix.com/sitemap/index","declared":true,"xml":true},"files":{"llms_txt":{"status":200,"content_type":"text/html","bytes":23393,"sha256":"4f13d364eb09d83c929c0103787f9f19fd2077f7957919767e3859bac45fa07b","final_host":"www.netflix.com","final_path":"/notfound","ms":107,"path":"/llms.txt","html":true,"json":null,"present":false},"agents_md":{"status":200,"content_type":"text/html","bytes":23393,"sha256":"610ce97fc61638d86aeb4319f99ffb7ec22243cfd5f9bf0081d11cdea9f5b7e1","final_host":"www.netflix.com","final_path":"/notfound","ms":404,"path":"/agents.md","html":true,"json":null,"present":false},"agent_skills":{"status":200,"content_type":"text/html","bytes":23419,"sha256":"e96d8dcf7c322a8b4db0e4fd8b0cfb1776a07c4c02bc1d05571f0ba48e5aa0ee","final_host":"www.netflix.com","final_path":"/notfound","ms":200,"path":"/.well-known/agent-skills/index.json","html":true,"json":null,"present":false},"mcp_card":{"status":200,"content_type":"text/html","bytes":23416,"sha256":"0fe88ddf8e99ef12aa1268773b0fb793b87b70ab905a03f7338f4300238b0afd","final_host":"www.netflix.com","final_path":"/notfound","ms":93,"path":"/.well-known/mcp/server-card.json","html":true,"json":null,"present":false},"api_catalog":{"status":200,"content_type":"text/html","bytes":23401,"sha256":"930ab8585f460879dec768dd89a2fd3c24494e7c6ae4b13141758a2cd7d6411b","final_host":"www.netflix.com","final_path":"/notfound","ms":109,"path":"/.well-known/api-catalog","html":true,"json":null,"present":false},"a2a_card":{"status":200,"content_type":"text/html","bytes":23405,"sha256":"d34cd33df6f2bdebe39db1a61e15c6a5dff420eab7db8dee0512289d9d10f8a4","final_host":"www.netflix.com","final_path":"/notfound","ms":98,"path":"/.well-known/agent-card.json","html":true,"json":null,"present":false}},"rings":{"v":1,"candidates":[],"mcp":[],"a2a":null,"webmcp":{"model_context":false,"bridge":false,"mcp_json":false,"link_rel":false,"any":false},"handshakes":0,"get_path":{"status":200,"content_type":"text/html","ms":90,"error":null},"get_host":{"status":530,"content_type":"text/plain","ms":56,"error":null}},"sf":[1,0,0,0,0,0,2,0,0],"derived":{"reachable":true,"home_class":"answers","robots_served":true,"robots_sitemap":true,"content_signal":false,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":["CCBot","Bytespider","Meta-ExternalAgent","Amazonbot"],"blocks_any_ai":true,"files":{"llms_txt":false,"agents_md":false,"agent_skills":false,"mcp_card":false,"api_catalog":false,"a2a_card":false},"any_capability":false,"ai_refused":[],"ai_thin":[],"aipref":0,"aipref_measured":true,"sf":[1,0,0,0,0,0,2,0,0],"rings":{"mcp_declared":false,"mcp_live":false,"mcp_auth":false,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":3,"fh":"www.netflix.com","fp_other_host":[],"fp":"c492080d141e988f","fp_bits":{"gptbot":0,"claudebot":5,"oaisearch":0,"perplexity":5}},"derive_v":5,"frame":"top10k","shard":3},{"v":1,"kind":"crawlcheck-census-record","run":18,"domain":"gandi.net","rank":45,"measured_at":"2026-11-13T18:51:40.579Z","home":{"browser":{"status":200,"content_type":"text/html","bytes":189318,"sha256":"4173125bd794cbc777d9c7012f703355908edd0e9aef37e0b41b1eca5318fbfc","final_host":"www.gandi.net","final_path":"/en","ms":854,"identity":"Mobile browser","html":true,"fp":"323a26a3def3f8d3"},"gptbot":{"status":200,"content_type":"text/html","bytes":189318,"sha256":"4173125bd794cbc777d9c7012f703355908edd0e9aef37e0b41b1eca5318fbfc","final_host":"www.gandi.net","final_path":"/en","ms":955,"identity":"GPTBot","html":true,"fp":"323a26a3def3f8d3"},"claudebot":{"status":200,"content_type":"text/html","bytes":189318,"sha256":"4173125bd794cbc777d9c7012f703355908edd0e9aef37e0b41b1eca5318fbfc","final_host":"www.gandi.net","final_path":"/en","ms":955,"identity":"ClaudeBot","html":true,"fp":"323a26a3def3f8d3"},"oaisearch":{"status":200,"content_type":"text/html","bytes":189318,"sha256":"4173125bd794cbc777d9c7012f703355908edd0e9aef37e0b41b1eca5318fbfc","final_host":"www.gandi.net","final_path":"/en","ms":797,"identity":"OAI-SearchBot","html":true,"fp":"323a26a3def3f8d3"},"perplexity":{"status":200,"content_type":"text/html","bytes":189318,"sha256":"4173125bd794cbc777d9c7012f703355908edd0e9aef37e0b41b1eca5318fbfc","final_host":"www.gandi.net","final_path":"/en","ms":778,"identity":"PerplexityBot","html":true,"fp":"323a26a3def3f8d3"}},"robots":{"status":200,"content_type":"text/plain","bytes":9023,"sha256":"3fd451a6143f6a84977d65af0ae1e26a153466423e8d7e08e672f3d1466e1cdc","final_host":"www.gandi.net","final_path":"/robots.txt","ms":815,"html":false,"text":"User-agent: *\r\n\r\n\r\n###### Sitemaps ######\r\nSitemap: https://www.gandi.net/sitemap.xml\r\nSitemap: https://news.gandi.net/sitemaps.xml\r\n\r\n###### Disalow ######\r\nDisallow: /*redirect=*\r\nDisallow: /*navigation=*\r\nDisallow: /*client_id=*\r\nDisallow: /*locale=*\r\nDisallow: /*tld=*\r\nDisallow: /*bulk=*\r\nDisallow: /*bu_id=*\r\nDisallow: /*opened=*\r\nDisallow: /*type=*\r\nDisallow: /*template=*\r\nDisallow: /*size=*\r\nDisallow: /*options=*\r\nDisallow: /*error=*\r\nDisallow: /*grid=*\r\nDisallow: /*currency=*\r\nDisallow: /*?c=*\r\nDisallow: /*&c=*\r\nDisallow: /*?sw=*\r\nDisallow: /*&sw=*\r\nDisallow: /*taxes=*\r\nDisallow: /pdf/*\r\nDisallow: /*/sample/*\r\nDisallow: */prices$\r\nDisallow: */rules$\r\nDisallow: */dates$\r\nDisallow: */news$\r\nAllow: */tld/news$\r\nDisallow: *?country=*\r\nDisallow: *?l=*\r\nDisallow: *?lang=*\r\nDisallow: /gcart/*\r\n\r\n\r\nUser-Agent: facebookexternalhit/1.1\r\nDisallow:\r\n\r\nUser-Agent: Facebot\r\nDisallow:\r\n\r\n###### Suppression robots ######\r\nUser-agent: AhrefsBot\r\nDisallow: /\r\nUser-agent: Alexibot\r\nDisallow: /\r\nUser-agent: Aqua_Products\r\nDisallow: /\r\nUser-agent: asterias\r\nDisallow: /\r\nUser-agent: b2w/0.1\r\nDisallow: /\r\nUser-agent: BackDoorBot/1.0\r\nDisallow: /\r\nUser-agent: BacklinkCrawler\r\nDisallow: /\r\nUser-agent: bdbrandprotect\r\nDisallow: /\r\nUser-agent: Black Hole\r\nDisallow: /\r\nUser-agent: BlowFish/1.0\r\nDisallow: /\r\nUser-agent: boitho.com-dc\r\nDisallow: /\r\nUser-agent: Bookmark search tool\r\nDisallow: /\r\nUser-agent: BotALot\r\nDisallow: /\r\nUser-agent: BotRightHere\r\nDisallow: /\r\nUser-agent: BPImageWalker\r\nDisallow: /\r\nUser-agent: BuiltBotTough\r\nDisallow: /\r\nUser-agent: Bullseye/1.0\r\nDisallow: /\r\nUser-agent: BunnySlippers\r\nDisallow: /\r\nUser-agent: CazoodleBot\r\nDisallow: /\r\nUser-agent: Cegbfeieh\r\nDisallow: /\r\nUser-agent: CheeseBot\r\nDisallow: /\r\nUser-agent: CherryPicker\r\nDisallow: /\r\nUser-agent: CherryPickerElite/1.0\r\nDisallow: /\r\nUser-agent: CherryPickerSE/1.0\r\nDisallow: /\r\nUser-agent: ConveraCrawler\r\nDisallow: /\r\nUser-agent: Copernic\r\nDisallow: /\r\nUser-agent: CopyRightCheck\r\nDisallow: /\r\nUser-agent: cosmos\r\nDisallow: /\r\nUser-agent: Crescent\r\nDisallow: /\r\nUser-agent: Crescent Internet ToolPak HTTP OLE Control v.1.0\r\nDisallow: /\r\nUser-agent: discobot\r\nDisallow: /\r\nUser-agent: DittoSpyder\r\nDisallow: /\r\nUser-agent: dotbot\r\nDisallow: /\r\nUser-agent: Download Ninja\r\nDisallow: /\r\nUser-agent: EmailCollector\r\nDisallow: /\r\nUser-agent: EmailSiphon\r\nDisallow: /\r\nUser-agent: EmailWolf\r\nDisallow: /\r\nUser-agent: EroCrawler\r\nDisallow: /\r\nUser-agent: ExtractorPro\r\nDisallow: /\r\nUser-agent: Ezooms\r\nDisallow: /\r\nUser-agent: FairAd Client\r\nDisallow: /\r\nUser-agent: Fasterfox\r\nDisallow: /\r\nUser-agent: Fetch\r\nDisallow: /\r\nUser-agent: findlinks\r\nDisallow: /\r\nUser-agent: Flaming AttackBot\r\nDisallow: /\r\nUser-agent: Foobot\r\nDisallow: /\r\nUser-agent: Gaisbot\r\nDisallow: /\r\nUser-agent: GetRight/4.2\r\nDisallow: /\r\nUser-agent: Gigabot\r\nDisallow: /\r\nUser-agent: grub\r\nDisallow: /\r\nUser-agent: grub-client\r\nDisallow: /\r\nUser-agent: Harvest/1.5\r\nDisallow: /\r\nUser-agent: heritrix\r\nDisallow: /\r\nUser-agent: hloader\r\nDisallow: /\r\nUser-agent: httplib\r\nDisallow: /\r\nUser-agent: HTTrack\r\nDisallow: / \r\nUser-agent: HTTrack 3.0\r\nDisallow: /\r\nUser-agent: HuaweiSymantecSpider\r\nDisallow: /\r\nUser-agent: humanlinks\r\nDisallow: /\r\nUser-agent: ichiro\r\nDisallow: /\r\nUser-agent: InfoNaviRobot\r\nDisallow: /\r\nUser-agent: IRLbot\r\nDisallow: /\r\nUser-agent: Iron33/1.0.2\r\nDisallow: /\r\nUser-agent: JennyBot\r\nDisallow: /\r\nUser-agent: kalooga\r\nDisallow: /\r\nUser-agent: Kenjin Spider\r\nDisallow: /\r\nUser-agent: Keyword Density/0.9\r\nDisallow: /\r\nUser-agent: larbin\r\nDisallow: /\r\nUser-agent: LexiBot\r\nDisallow: /\r\nUser-agent: LexxeBot\r\nDisallow: /\r\nUser-agent: libWeb/clsHTTP\r\nDisallow: /\r\nUser-agent: libwww\r\nDisallow: /\r\nUser-agent: LinkextractorPro\r\nDisallow: /\r\nUser-agent: linko\r\nDisallow: /\r\nUser-agent: LinkScan/8.1a Unix\r\nDisallow: /\r\nUser-agent: LinkWalker\r\nDisallow: /\r\nUser-agent: LNSpiderguy\r\nDisallow: /\r\nUser-agent: lwp-trivial\r\nDisallow: /\r\nUser-agent: lwp-trivial/1.34\r\nDisallow: /\r\nUser-agent: Mata Hari\r\nDisallow: /\r\nUser-agent: Microsoft URL Control\r\nDisallow: /\r\nUser-agent: Microsoft URL Control - 5.01.4511\r\nDisallow: /\r\nUser-agent: Microsoft URL Control - 6.00.8169\r\nDisallow: /\r\nUser-agent: Microsoft.URL.Control\r\nDisallow: /\r\nUser-agent: MIIxpc\r\nDisallow: /\r\nUser-agent: MIIxpc/4.2\r\nDisallow: /\r\nUser-agent: Mister PiX\r\nDisallow: /\r\nUser-agent: MJ12bot\r\nDisallow: /\r\nUser-agent: MLBot\r\nDisallow: /\r\nUser-agent: moget\r\nDisallow: /\r\nUser-agent: moget/2.1\r\nDisallow: /\r\nUser-agent: Mozilla/4.0 (compatible; BullsEye; Windows 95)\r\nDisallow: /\r\nUser-agent: MSIECrawler\r\nDisallow: /\r\nUser-agent: NaverBot\r\nDisallow: /\r\nUser-agent: NerdyBot\r\nDisallow: /\r\nUser-agent: Net\\ Vampire\r\nDisallow: /\r\nUser-agent: NetAnts\r\nDisallow: /\r\nUser-agent: NetMechanic\r\nDisallow: /\r\nUser-agent: NICErsPRO\r\nDisallow: /\r\nUser-agent: Nutch\r\nDisallow: /\r\nUser-agent: Ocelli\r\nDisallow: /\r\nUser-agent: Offline Explorer\r\nDisallow: /\r\nUser-agent: Openbot\r\nDisallow: /\r\nUser-agent: Openfind\r\nDisallow: /\r\nUser-agent: Openfind data gatherer\r\nDisallow: /\r\nUser-agent: Oracle Ultra Search\r\nDisallow: /\r\nUser-agent: penthesilea\r\nDisallow: /\r\nUser-agent: PerMan\r\nDisallow: /\r\nUser-agent: PhpDig\r\nDisallow: /\r\nUser-agent: ProPowerBot/2.14\r\nDisallow: /\r\nUser-agent: ProWebWalker\r\nDisallow: /\r\nUser-agent: psbot\r\nDisallow: /\r\nUser-agent: Python-urllib\r\nDisallow: /\r\nUser-agent: QueryN Metasearch\r\nDisallow: /\r\nUser-agent: Radiation Retriever 1.1\r\nDisallow: /\r\nUser-agent: RepoMonkey\r\nDisallow: /\r\nUser-agent: RepoMonkey Bait & Tackle/v1.01\r\nDisallow: /\r\nUser-agent: RMA\r\nDisallow: /\r\nUser-agent: rogerbot\r\nDisallow: /\r\nUser-agent: SCOUTjET\r\nDisallow: /\r\nUser-agent: SearchDaimon.com-dc\r\nDisallow: /\r\nUser-agent: searchpreview\r\nDisallow: /\r\nUser-agent: SEOENGBot\r\nDisallow: /\r\nUser-agent: ShopWiki\r\nDisallow: /\r\nUser-agent: sitecheck.internetseer.com\r\nDisallow: /\r\nUser-agent: SiteSnagger\r\nDisallow: /\r\nUser-agent: Snapbot\r\nDisallow: /\r\nUser-agent: Sogou\\ web\\ spider\r\nDisallow: /\r\nUser-agent: Sosospider\r\nDisallow: /\r\nUser-agent: SpankBot\r\nDisallow: /\r\nUser-agent: spanner\r\nDisallow: /\r\nUser-agent: spbot\r\nDisallow: /\r\nUser-agent: SurveyBot\r\nDisallow: /\r\nUser-agent: suzuran\r\nDisallow: /\r\nUser-agent: Szukacz/1.4\r\nDisallow: /\r\nUser-agent: Teleport\r\nDisallow: /\r\nUser-agent: TeleportPro\r\nDisallow: /\r\nUser-agent: Telesoft\r\nDisallow: /\r\nUser-agent: Templeton\r\nDisallow: /\r\nUser-agent: The Intraformant\r\nDisallow: /\r\nUser-agent: TheNomad\r\nDisallow: /\r\nUser-agent: TightTwatBot\r\nDisallow: /\r\nUser-agent: Titan\r\nDisallow: /\r\nUser-agent: toCrawl/UrlDispatcher\r\nDisallow: /\r\nUser-agent: True_Robot\r\nDisallow: /\r\nUser-agent: True_Robot/1.0\r\nDisallow: /\r\nUser-agent: turingos\r\nDisallow: /\r\nUser-agent: TurnitinBot\r\nDisallow: /\r\nUser-agent: TurnitinBot/1.5\r\nDisallow: /\r\nUser-agent: TwengaBot\r\nDisallow: /\r\nUser-agent: Twiceler\r\nDisallow: /\r\nUser-agent: Updownerbot\r\nDisallow: /\r\nUser-agent: URL Control\r\nDisallow: /\r\nUser-agent: URL_Spider_Pro\r\nDisallow: /\r\nUser-agent: URLy Warning\r\nDisallow: /\r\nUser-agent: VCI\r\nDisallow: /\r\nUser-agent: VCI WebViewer VCI WebViewer Win32\r\nDisallow: /\r\nUser-agent: Web Image Collector\r\nDisallow: /\r\nUser-agent: WebAuto\r\nDisallow: /\r\nUser-agent: WebBandit\r\nDisallow: /\r\nUser-agent: WebBandit/3.50\r\nDisallow: /\r\nUser-agent: WebCapture 2.0\r\nDisallow: /\r\nUser-agent: WebCopier\r\nDisallow: /\r\nUser-agent: WebCopier v.2.2\r\nDisallow: /\r\nUser-agent: WebCopier v3.2a\r\nDisallow: /\r\nUser-agent: WebEnhancer\r\nDisallow: /\r\nUser-agent: WebmasterWorldForumBot\r\nDisallow: /\r\nUser-agent: WebSauger\r\nDisallow: /\r\nUser-agent: Website Quester\r\nDisallow: /\r\nUser-agent: Webster Pro\r\nDisallow: /\r\nUser-agent: WebStripper\r\nDisallow: /\r\nUser-agent: WebZIP\r\nDisallow: /\r\nUser-agent: WebZip/4.0\r\nDisallow: /\r\nUser-agent: WebZIP/4.21\r\nDisallow: /\r\nUser-agent: WebZIP/5.0\r\nDisallow: /\r\nUser-agent: wget\r\nDisallow: /\r\nUser-agent: Wget/1.5.3\r\nDisallow: /\r\nUser-agent: Wget/1.6\r\nDisallow: /\r\nUser-agent: woozweb-monitoring\r\nDisallow: /\r\nUser-agent: woriobot\r\nDisallow: /\r\nUser-agent: wprocketbot\r\nAllow: /\r\nUser-agent: WWW-Collector-E\r\nDisallow: /\r\nUser-agent: Xenu\r\nDisallow: /\r\nUser-agent: Xenu's\r\nDisallow: /\r\nUser-agent: Xenu's Link Sleuth 1.1c\r\nDisallow: /\r\nUser-agent: YACYBIT\r\nDisallow: /\r\nUser-agent: yacybot\r\nDisallow: /\r\nUser-agent: Yandex\r\nDisallow: /\r\nUser-agent: YandexBot\r\nDisallow: /\r\nUser-agent: Yeti\r\nDisallow: /\r\nUser-agent: YodaoBot\r\nDisallow: /\r\nUser-agent: Zealbot\r\nDisallow: /\r\nUser-agent: Zeus\r\nDisallow: /\r\nUser-agent: Zeus 32297 Webster Pro V2.9 Win32\r\nDisallow: /\r\nUser-agent: Zeus Link Scout\r\nDisallow: /\r\nUser-agent: ZyBorg\r\nDisallow: /\r\nUser-agent: Mozilla/4.0 (compatible; BullsEye; Windows 95)\r\nDisallow: /\r\nUser-agent: NerdyBot\r\nDisallow: /\r\nUser-agent: WebAuto\r\nDisallow: /\r\nUser-agent: wprocketbot\r\nDisallow: /\r\nUser-agent: DIIbot\r\nDisallow: /\r\nUser-agent: eCatch\r\nDisallow: /\r\nUser-agent: Jetbot\r\nDisallow: /\r\nUser-agent: Openfind data gathere\r\nDisallow: /\r\nUser-agent: QuepasaCreep\r\nDisallow: /\r\nUser-agent: Roverbot\r\nDisallow: /\r\nUser-agent: Foobot\r\nDisallow: /\r\nUser-agent: RepoMonkey Bait & Tackle/v1.01\r\nDisallow: /\r\nUser-agent: ExtractorPro\r\nDisallow: /","agents":{"GPTBot":true,"OAI-SearchBot":true,"ChatGPT-User":true,"ClaudeBot":true,"Claude-SearchBot":true,"PerplexityBot":true,"Google-Extended":true,"Applebot-Extended":true,"CCBot":true,"Bytespider":true,"Meta-ExternalAgent":true,"Amazonbot":true}},"sitemap":{"status":200,"content_type":"text/xml","bytes":529,"sha256":"20e45cfdab4c73574ebe3829d032be6b389f600aac558cddcc91ecbe5f9681f9","final_host":"www.gandi.net","final_path":"/sitemap.xml","ms":133,"url":"https://www.gandi.net/sitemap.xml","declared":true,"xml":true},"files":{"llms_txt":{"status":404,"content_type":"text/html","bytes":118145,"sha256":"faa3c94bfe16278cb8f380b5f0884f67ce86ac93c2cca2d440addef229e2f37c","final_host":"www.gandi.net","final_path":"/llms.txt","ms":471,"path":"/llms.txt","html":true,"json":null,"present":false},"agents_md":{"status":404,"content_type":"text/html","bytes":118146,"sha256":"9ada5eac863d5aaa6c66916336c794713e0a3254699baa145ed2a48c6b6eb1c8","final_host":"www.gandi.net","final_path":"/agents.md","ms":736,"path":"/agents.md","html":true,"json":null,"present":false},"agent_skills":{"status":404,"content_type":"text/html","bytes":118172,"sha256":"fd8227d5b383a8c4415529a00828e83fdcc6a4117a8b8ca42c02f81c2cd66b69","final_host":"www.gandi.net","final_path":"/.well-known/agent-skills/index.json","ms":464,"path":"/.well-known/agent-skills/index.json","html":true,"json":null,"present":false},"mcp_card":{"status":404,"content_type":"text/html","bytes":118169,"sha256":"ec4a1c17b758ede2cb13e28d389066a66be29a8500576be27f118a1d3a0c000b","final_host":"www.gandi.net","final_path":"/.well-known/mcp/server-card.json","ms":507,"path":"/.well-known/mcp/server-card.json","html":true,"json":null,"present":false},"api_catalog":{"status":404,"content_type":"text/html","bytes":118160,"sha256":"4546f178591bd0bd621661ccdede7d4ef470df6ebd97d4378313a07ff956fef7","final_host":"www.gandi.net","final_path":"/.well-known/api-catalog","ms":786,"path":"/.well-known/api-catalog","html":true,"json":null,"present":false},"a2a_card":{"status":404,"content_type":"text/html","bytes":118164,"sha256":"49d2380866df75e240748aff8d14675dc85693ebb8c93bd2bd42f5b6076cd883","final_host":"www.gandi.net","final_path":"/.well-known/agent-card.json","ms":768,"path":"/.well-known/agent-card.json","html":true,"json":null,"present":false}},"rings":{"v":1,"candidates":[],"mcp":[],"a2a":null,"webmcp":{"model_context":false,"bridge":false,"mcp_json":false,"link_rel":false,"any":false},"handshakes":0,"get_path":{"status":200,"content_type":"text/html","ms":502,"error":null},"get_host":{"status":530,"content_type":"text/plain","ms":266,"error":null}},"sf":[1,0,0,0,0,0,0,0,0],"derived":{"reachable":true,"home_class":"answers","robots_served":true,"robots_sitemap":true,"content_signal":false,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":[],"blocks_any_ai":false,"files":{"llms_txt":false,"agents_md":false,"agent_skills":false,"mcp_card":false,"api_catalog":false,"a2a_card":false},"any_capability":false,"ai_refused":[],"ai_thin":[],"aipref":0,"aipref_measured":true,"sf":[1,0,0,0,0,0,0,0,0],"rings":{"mcp_declared":false,"mcp_live":false,"mcp_auth":false,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":3,"fh":"www.gandi.net","fp_other_host":[],"fp":"323a26a3def3f8d3","fp_bits":{"gptbot":0,"claudebot":0,"oaisearch":0,"perplexity":0}},"derive_v":5,"frame":"top10k","shard":4},{"v":1,"kind":"crawlcheck-census-record","run":18,"domain":"cloudfront.net","rank":46,"measured_at":"2026-11-13T18:52:33.408Z","home":{"browser":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"cloudfront.net","final_path":"/","ms":42,"identity":"Mobile browser","html":false},"gptbot":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"cloudfront.net","final_path":"/","ms":16,"identity":"GPTBot","html":false},"claudebot":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"cloudfront.net","final_path":"/","ms":8,"identity":"ClaudeBot","html":false},"oaisearch":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"cloudfront.net","final_path":"/","ms":1,"identity":"OAI-SearchBot","html":false},"perplexity":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"cloudfront.net","final_path":"/","ms":0,"identity":"PerplexityBot","html":false}},"robots":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"cloudfront.net","final_path":"/robots.txt","ms":0,"html":false,"text":null,"agents":null},"sitemap":null,"files":null,"derived":{"reachable":false,"home_class":"no_site","robots_served":false,"robots_sitemap":false,"content_signal":false,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":[],"blocks_any_ai":false,"files":{"llms_txt":false,"agents_md":false,"agent_skills":false,"mcp_card":false,"api_catalog":false,"a2a_card":false},"any_capability":false,"ai_refused":[],"ai_thin":[],"rings":{"mcp_declared":false,"mcp_live":false,"mcp_auth":false,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":3,"fh":"cloudfront.net","fp_other_host":[]},"derive_v":5,"frame":"top10k","shard":5},{"v":1,"kind":"crawlcheck-census-record","run":18,"domain":"youtu.be","rank":47,"measured_at":"2026-11-13T18:53:15.625Z","home":{"browser":{"status":200,"content_type":"text/html","bytes":353950,"sha256":"32a6c90fb8447e6f2fed8740fba0b7c4c63ad0f4eb5c96353ec3881c7b52a196","final_host":"m.youtube.com","final_path":"/","ms":536,"identity":"Mobile browser","html":true},"gptbot":{"status":200,"content_type":"text/html","bytes":874244,"sha256":"86139aa22e90cfa056c6446ac285d8447714ab3534cfb5679a7525f8c0f3fbce","final_host":"www.youtube.com","final_path":"/","ms":467,"identity":"GPTBot","html":true},"claudebot":{"status":200,"content_type":"text/html","bytes":871988,"sha256":"f7b0855aacac27ba9e2845c44da0828c9636a3318aaf4ad84d27ae1a4cc57609","final_host":"www.youtube.com","final_path":"/","ms":466,"identity":"ClaudeBot","html":true},"oaisearch":{"status":200,"content_type":"text/html","bytes":976187,"sha256":"f3f54f35fea8a900c862c854feefb4106653ce46aeb5af8aecb93cb410f30866","final_host":"www.youtube.com","final_path":"/","ms":467,"identity":"OAI-SearchBot","html":true},"perplexity":{"status":200,"content_type":"text/html","bytes":973156,"sha256":"d9a2cbb60681a97e79612018a93aceb107c76173f60f24fe150bec7a268a4935","final_host":"www.youtube.com","final_path":"/","ms":302,"identity":"PerplexityBot","html":true}},"robots":{"status":200,"content_type":"text/plain","bytes":511,"sha256":"c290485c8002c51c24c95765948dbcec9ec88c1f181638d245418a36194fd1a0","final_host":"youtu.be","final_path":"/robots.txt","ms":188,"html":false,"text":"# robots.txt file for youtu.be\n\nUser-agent: *\nDisallow: /api/\nDisallow: /comment\nDisallow: /feeds/videos.xml\nDisallow: /get_video\nDisallow: /get_video_info\nDisallow: /get_midroll_info\nDisallow: /live_chat\nDisallow: /login\nDisallow: /results\nDisallow: /signup\nDisallow: /t/terms\nDisallow: /timedtext_video\nDisallow: /verify_age\nDisallow: /watch_ajax\nDisallow: /watch_fragments_ajax\nDisallow: /watch_popup\nDisallow: /watch_queue_ajax\nDisallow: /youtubei/\n\nAllow: /apple-app-site-association\nAllow: /.well-known/*\n","agents":{"GPTBot":true,"OAI-SearchBot":true,"ChatGPT-User":true,"ClaudeBot":true,"Claude-SearchBot":true,"PerplexityBot":true,"Google-Extended":true,"Applebot-Extended":true,"CCBot":true,"Bytespider":true,"Meta-ExternalAgent":true,"Amazonbot":true}},"sitemap":{"status":200,"content_type":"text/html","bytes":865912,"sha256":"637d0cd8d2069af9734265c2db3c1ce7840b010ce7965d977f4fbfb02a44f372","final_host":"www.youtube.com","final_path":"/watch","ms":232,"url":"https://youtu.be/sitemap.xml","declared":false,"xml":false},"files":{"llms_txt":{"status":200,"content_type":"text/html","bytes":871677,"sha256":"a46d63c3b86eb25ff32e30bd587fb661bcc5fff4d25433bfeaf784577b4eea7e","final_host":"www.youtube.com","final_path":"/watch","ms":256,"path":"/llms.txt","html":true,"json":null,"present":false},"agents_md":{"status":200,"content_type":"text/html","bytes":869579,"sha256":"39cd9dbcd8a1ac9aa24790623952848a81be08e47828c1b1ad360e96da345e47","final_host":"www.youtube.com","final_path":"/watch","ms":174,"path":"/agents.md","html":true,"json":null,"present":false},"agent_skills":{"status":200,"content_type":"text/html","bytes":873183,"sha256":"0d151c4ccebe2b2086d850a9d9eec6a1df9201664c1874dd38a10a420414b790","final_host":"www.youtube.com","final_path":"/watch","ms":193,"path":"/.well-known/agent-skills/index.json","html":true,"json":null,"present":false},"mcp_card":{"status":200,"content_type":"text/html","bytes":874423,"sha256":"90da7def2400117a60782606fe1a084c37dcbfbc06a7543eba17ebb2210e3e65","final_host":"www.youtube.com","final_path":"/watch","ms":185,"path":"/.well-known/mcp/server-card.json","html":true,"json":null,"present":false},"api_catalog":{"status":200,"content_type":"text/html","bytes":865359,"sha256":"f246a8d682ce68b422d3918868ebe4d8e8dcdac5317a1ab378cff1b1764a0f1d","final_host":"www.youtube.com","final_path":"/watch","ms":175,"path":"/.well-known/api-catalog","html":true,"json":null,"present":false},"a2a_card":{"status":200,"content_type":"text/html","bytes":873692,"sha256":"b59afa4cd316e71cc48a25c2b76e17d1837a116d04421bf1a6c9c8bb245bbfc5","final_host":"www.youtube.com","final_path":"/watch","ms":156,"path":"/.well-known/agent-card.json","html":true,"json":null,"present":false}},"rings":{"v":1,"candidates":[],"mcp":[],"a2a":null,"webmcp":{"model_context":false,"bridge":false,"mcp_json":false,"link_rel":false,"any":false},"handshakes":0,"get_path":{"status":200,"content_type":"text/html","ms":234,"error":null},"get_host":{"status":530,"content_type":"text/plain","ms":44,"error":null}},"sf":[1,0,0,0,0,0,2,0,0],"derived":{"reachable":true,"home_class":"answers","robots_served":true,"robots_sitemap":false,"content_signal":false,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":[],"blocks_any_ai":false,"files":{"llms_txt":false,"agents_md":false,"agent_skills":false,"mcp_card":false,"api_catalog":false,"a2a_card":false},"any_capability":false,"ai_refused":[],"ai_thin":[],"aipref":0,"aipref_measured":true,"sf":[1,0,0,0,0,0,2,0,0],"rings":{"mcp_declared":false,"mcp_live":false,"mcp_auth":false,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":3,"fh":"m.youtube.com","fp_other_host":[]},"derive_v":5,"frame":"top10k","shard":6},{"v":1,"kind":"crawlcheck-census-record","run":18,"domain":"x.com","rank":48,"measured_at":"2026-11-13T18:55:41.951Z","home":{"browser":{"status":200,"content_type":"text/html","bytes":46108,"sha256":"b48482c9680cb24dcb1c2c95a21347a37d7a2833f49ac51a833b30bcc0f11ee8","final_host":"x.com","final_path":"/","ms":169,"identity":"Mobile browser","html":true,"fp":"8b48d3564ad225ec"},"gptbot":{"status":402,"content_type":"application/json","bytes":55,"sha256":"35e642f0e1bd426b9d8b0ee0284aefe801873dc13c18819fbeeaf2bf47136664","final_host":"x.com","final_path":"/","ms":2,"identity":"GPTBot","html":false},"claudebot":{"status":403,"content_type":null,"bytes":0,"sha256":"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855","final_host":"x.com","final_path":"/","ms":127,"identity":"ClaudeBot","html":null},"oaisearch":{"status":402,"content_type":"application/json","bytes":55,"sha256":"35e642f0e1bd426b9d8b0ee0284aefe801873dc13c18819fbeeaf2bf47136664","final_host":"x.com","final_path":"/","ms":3,"identity":"OAI-SearchBot","html":false},"perplexity":{"status":402,"content_type":"application/json","bytes":55,"sha256":"35e642f0e1bd426b9d8b0ee0284aefe801873dc13c18819fbeeaf2bf47136664","final_host":"x.com","final_path":"/","ms":2,"identity":"PerplexityBot","html":false}},"robots":{"status":200,"content_type":"text/plain","bytes":3039,"sha256":"b54dd4fefb8786b6441dc95ab04db1dbe4c6b9fcb5e7bb60d8381ab6e40989ff","final_host":"x.com","final_path":"/robots.txt","ms":117,"html":false,"text":"# Google / Bing Search Engine Robots\n# ==================================\n# Shared group so Googlebot and Bingbot always get the same rules.\nUser-agent: Googlebot\nUser-agent: Bingbot\n\nAllow: /*?s=\nAllow: /*?t=\nAllow: /*?ref_src=\nAllow: /hashtag/*?src=\nAllow: /search?q=%23\nAllow: /i/api/\n# Same length as the /*/likes and /*/media rules; RFC 9309 breaks the tie in\n# favor of Allow, so /i/api/ fetches stay crawlable.\nAllow: /i/api/*\nDisallow: /*?lang=en-ss\nAllow: /*?lang=\nDisallow: /search/realtime\nDisallow: /search/users\nDisallow: /search/*/grid\n\nDisallow: /*/analytics\nDisallow: /*/followers\nDisallow: /*/following\nDisallow: /*/verified_followers\n\nDisallow: /account/deactivated\nDisallow: /settings/deactivated\n\n# Only `*` and `$` are metacharacters here (RFC 9309); regex classes match\n# literally. /photo is anchored so /status/*/photo/1 media pages stay open.\nDisallow: /*/likes\nDisallow: /*/likes?\nDisallow: /*/retweets\nDisallow: /*/retweets?\nDisallow: /*/media\nDisallow: /*/media?\nDisallow: /*/photo$\nDisallow: /*/photo?\nAllow: /*?\n\nUser-agent: facebookexternalhit\n\nAllow: /*?lang=\nAllow: /*?s=\nAllow: /*&s=\nAllow: /*?t=\nAllow: /*?ref_src=\nAllow: /hashtag/*?src=\nAllow: /search?*cashtagRestId=\nAllow: /i/api/\nAllow: /i/api/*\nDisallow: /search?q=\nDisallow: /search/realtime\nDisallow: /search/users\nDisallow: /search/*/grid\n\nDisallow: /*?\nDisallow: /*/followers\nDisallow: /*/following\nDisallow: /*/verified_followers\n\nDisallow: /account/deactivated\nDisallow: /settings/deactivated\n\nDisallow: /*/likes\nDisallow: /*/likes?\nDisallow: /*/retweets\nDisallow: /*/retweets?\nDisallow: /*/media\nDisallow: /*/media?\nDisallow: /*/photo$\nDisallow: /*/photo?\n\nUser-Agent: Google-Extended\nDisallow: *\n\nUser-Agent: FacebookBot\nDisallow: *\n\nUser-agent: Discordbot\nDisallow: *\n\n# Meta AI / product crawlers (product tokens; versions like /1.1 are ignored\n# under RFC 9309 token matching).\nUser-agent: meta-webindexer\nDisallow: *\n\nUser-agent: meta-externalagent\nDisallow: *\n\nUser-agent: meta-externalads\nDisallow: *\n\nUser-agent: meta-externalfetcher\nDisallow: *\n\n# Shared AI training preference (IETF draft-kelley-aipref-training-preference-signal),\n# honored by Microsoft, Google, OpenAI, and Anthropic. This is a usage policy\n# token, not a crawler: it does not change crawl access, and `User-agent: *`\n# never matches it.\nUser-agent: __usage_ai-training\nDisallow: /\n\n# Every bot that might possibly read and respect this file\n# ========================================================\nUser-agent: *\nDisallow: /\n\n\n# WHAT-4882 - Keep notification-email links (/i/u) out of search results.\n# Named crawlers stay un-blocked on purpose: they must crawl /i/u to see its\n# X-Robots-Tag noindex (the robots.txt Noindex directive died in 2019).\nDisallow: /i/u\n\n# Wait 1 second between successive requests. See ONBOARD-2698 for details.\nCrawl-delay: 1\n\n# Independent of user agent. Links in the sitemap are full URLs using https://\n# and need to match the protocol of the sitemap.\nSitemap: https://x.com/sitemap.xml\nSitemap: https://x.com/sitemaps/finance0\n","agents":{"GPTBot":false,"OAI-SearchBot":false,"ChatGPT-User":false,"ClaudeBot":false,"Claude-SearchBot":false,"PerplexityBot":false,"Google-Extended":false,"Applebot-Extended":false,"CCBot":false,"Bytespider":false,"Meta-ExternalAgent":false,"Amazonbot":false}},"sitemap":{"status":404,"content_type":"text/html","bytes":46103,"sha256":"86230706a2b0b3bcc60bc22aa76912df6fb9516b61394c091f84a2cba54b136a","final_host":"x.com","final_path":"/sitemap.xml","ms":109,"url":"https://x.com/sitemap.xml","declared":true,"xml":false},"files":{"llms_txt":{"status":404,"content_type":"text/html","bytes":46082,"sha256":"8ff4b2ba1f2d1e4826b3b1e75f83cf692fceb070c9a58dcf362ba78198138125","final_host":"x.com","final_path":"/llms.txt","ms":240,"path":"/llms.txt","html":true,"json":null,"present":false},"agents_md":{"status":404,"content_type":"text/html","bytes":46091,"sha256":"d0b9d0328ba67282130bc6a2c5dad73f76cad1a9f83d6e9dd98d355d0bf05d89","final_host":"x.com","final_path":"/agents.md","ms":167,"path":"/agents.md","html":true,"json":null,"present":false},"agent_skills":{"status":404,"content_type":"text/html","bytes":46271,"sha256":"60aaa1778c58da1bc85a95f8494ab5d8334874902c5d1512db328fcc6929c747","final_host":"x.com","final_path":"/.well-known/agent-skills/index.json","ms":154,"path":"/.well-known/agent-skills/index.json","html":true,"json":null,"present":false},"mcp_card":{"status":404,"content_type":"text/html","bytes":46253,"sha256":"e8a82e2849b8bcd71146a637aebc5ae056132c8570310acd9c5ac1a25983b1b3","final_host":"x.com","final_path":"/.well-known/mcp/server-card.json","ms":155,"path":"/.well-known/mcp/server-card.json","html":true,"json":null,"present":false},"api_catalog":{"status":404,"content_type":"text/html","bytes":46187,"sha256":"532170d822b6a8cf097635a0f9a76383dc317c4e3bc330ca01f14384125f1988","final_host":"x.com","final_path":"/.well-known/api-catalog","ms":110,"path":"/.well-known/api-catalog","html":true,"json":null,"present":false},"a2a_card":{"status":404,"content_type":"text/html","bytes":46211,"sha256":"387dd9fcd43559d15afcf5c176fdec498ecd331c274b77feecf650e6d6360b7a","final_host":"x.com","final_path":"/.well-known/agent-card.json","ms":115,"path":"/.well-known/agent-card.json","html":true,"json":null,"present":false}},"rings":{"v":1,"candidates":[],"mcp":[],"a2a":null,"webmcp":{"model_context":false,"bridge":false,"mcp_json":false,"link_rel":false,"any":false},"handshakes":0,"get_path":{"status":200,"content_type":"text/html","ms":285,"error":null},"get_host":{"status":530,"content_type":"text/plain","ms":104,"error":null}},"sf":[4,0,0,0,0,0,1,0,0],"derived":{"reachable":true,"home_class":"answers","robots_served":true,"robots_sitemap":true,"content_signal":false,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":["GPTBot","OAI-SearchBot","ChatGPT-User","ClaudeBot","Claude-SearchBot","PerplexityBot","Google-Extended","Applebot-Extended","CCBot","Bytespider","Meta-ExternalAgent","Amazonbot"],"blocks_any_ai":true,"files":{"llms_txt":false,"agents_md":false,"agent_skills":false,"mcp_card":false,"api_catalog":false,"a2a_card":false},"any_capability":false,"ai_refused":["gptbot","claudebot","oaisearch","perplexity"],"ai_thin":[],"aipref":2,"aipref_measured":true,"sf":[4,0,0,0,0,0,1,0,0],"rings":{"mcp_declared":false,"mcp_live":false,"mcp_auth":false,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":3,"fh":"x.com","fp_other_host":[],"fp":"8b48d3564ad225ec","fp_bits":{}},"derive_v":5,"frame":"top10k","shard":7},{"v":1,"kind":"crawlcheck-census-record","run":18,"domain":"wordpress.org","rank":49,"measured_at":"2026-11-13T18:46:27.088Z","home":{"browser":{"status":200,"content_type":"text/html","bytes":178381,"sha256":"cbfcf890ec120e61c167cc224de133aec2a5b6f0f7032712aced7c7fb4342982","final_host":"wordpress.org","final_path":"/","ms":1162,"identity":"Mobile browser","html":true,"fp":"39fbeed5967399e0"},"gptbot":{"status":200,"content_type":"text/html","bytes":178381,"sha256":"66aed5c483a622cf64e2c33d5e6c519a73eca691674339a9b5160f975a7f152a","final_host":"wordpress.org","final_path":"/","ms":1200,"identity":"GPTBot","html":true,"fp":"39fbeed5967399e0"},"claudebot":{"status":200,"content_type":"text/html","bytes":178381,"sha256":"cbfcf890ec120e61c167cc224de133aec2a5b6f0f7032712aced7c7fb4342982","final_host":"wordpress.org","final_path":"/","ms":877,"identity":"ClaudeBot","html":true,"fp":"39fbeed5967399e0"},"oaisearch":{"status":200,"content_type":"text/html","bytes":178381,"sha256":"cbfcf890ec120e61c167cc224de133aec2a5b6f0f7032712aced7c7fb4342982","final_host":"wordpress.org","final_path":"/","ms":932,"identity":"OAI-SearchBot","html":true,"fp":"39fbeed5967399e0"},"perplexity":{"status":200,"content_type":"text/html","bytes":178381,"sha256":"cbfcf890ec120e61c167cc224de133aec2a5b6f0f7032712aced7c7fb4342982","final_host":"wordpress.org","final_path":"/","ms":917,"identity":"PerplexityBot","html":true,"fp":"39fbeed5967399e0"}},"robots":{"status":200,"content_type":"text/plain","bytes":976,"sha256":"65185a701db3ab195bcb2f86657300e13153f758af110ac0ffeeea5dc19ab5dc","final_host":"wordpress.org","final_path":"/robots.txt","ms":811,"html":false,"text":"Sitemap: https://wordpress.org/sitemap.xml\nSitemap: https://wordpress.org/news-sitemap.xml\nSitemap: https://wordpress.org/themes/sitemap.xml\nSitemap: https://wordpress.org/plugins/sitemap.xml\nSitemap: https://wordpress.org/news/sitemap.xml\nSitemap: https://wordpress.org/showcase/sitemap.xml\nSitemap: https://wordpress.org/documentation/sitemap.xml\nSitemap: https://wordpress.org/patterns/sitemap.xml\nSitemap: https://wordpress.org/photos/sitemap.xml\nUser-agent: *\nDisallow: /wp-admin/\nAllow: /wp-admin/admin-ajax.php\nAllow: /wp-admin/load-scripts.php\nAllow: /wp-admin/load-styles.php\n\nUser-agent: *\nDisallow: /search\nDisallow: /?s=\n\nUser-agent: GPTBot\nAllow: /\n\nUser-agent: ClaudeBot\nAllow: /\n\nUser-agent: anthropic-ai\nAllow: /\n\nUser-agent: Google-Extended\nAllow: /\n\nUser-agent: Applebot-Extended\nAllow: /\n\nUser-agent: PerplexityBot\nAllow: /\n\nUser-agent: Bytespider\nAllow: /\n\nUser-agent: CCBot\nAllow: /\n\nUser-agent: Copilot\nAllow: /\n\nUser-agent: *\nDisallow: /plugins/search/\n","agents":{"GPTBot":true,"OAI-SearchBot":true,"ChatGPT-User":true,"ClaudeBot":true,"Claude-SearchBot":true,"PerplexityBot":true,"Google-Extended":true,"Applebot-Extended":true,"CCBot":true,"Bytespider":true,"Meta-ExternalAgent":true,"Amazonbot":true}},"sitemap":{"status":200,"content_type":"text/xml","bytes":631,"sha256":"cd23d460235ac050dd80379b4a8b7190bd87b47aef92c07d8f3eb42423d479d8","final_host":"wordpress.org","final_path":"/sitemap.xml","ms":451,"url":"https://wordpress.org/sitemap.xml","declared":true,"xml":true},"files":{"llms_txt":{"status":200,"content_type":"text/plain","bytes":5765,"sha256":"731a4c979c79b5a350375f8b089fc26ed70b20d395de39ad305b716de233b4af","final_host":"wordpress.org","final_path":"/llms.txt","ms":282,"path":"/llms.txt","html":false,"json":null,"present":true},"agents_md":{"status":404,"content_type":"text/html","bytes":124910,"sha256":"9e25e6d78100dccc079302bbccd774f6b295a8a62397845c9270f62fa2c954b3","final_host":"wordpress.org","final_path":"/agents.md","ms":531,"path":"/agents.md","html":true,"json":null,"present":false},"agent_skills":{"status":404,"content_type":"text/html","bytes":124962,"sha256":"4fe82142b1f3cc7021e2db5be8a79a0576309bf512269b654264544bb6a6e192","final_host":"wordpress.org","final_path":"/.well-known/agent-skills/index.json","ms":559,"path":"/.well-known/agent-skills/index.json","html":true,"json":null,"present":false},"mcp_card":{"status":404,"content_type":"text/html","bytes":124956,"sha256":"f6a4502de11b1e40f936bc8bf7fb6f17499b529c791bd59045e947b10e829d60","final_host":"wordpress.org","final_path":"/.well-known/mcp/server-card.json","ms":571,"path":"/.well-known/mcp/server-card.json","html":true,"json":null,"present":false},"api_catalog":{"status":404,"content_type":"text/html","bytes":124938,"sha256":"4f8304e66cbccd6958d4b4c765bda449a568c826f1e257a490b2a79bc961b780","final_host":"wordpress.org","final_path":"/.well-known/api-catalog","ms":626,"path":"/.well-known/api-catalog","html":true,"json":null,"present":false},"a2a_card":{"status":404,"content_type":"text/html","bytes":124946,"sha256":"27537741c63caa842888484b112c4feb4dc3fca48a2547fdf074500289348d3f","final_host":"wordpress.org","final_path":"/.well-known/agent-card.json","ms":628,"path":"/.well-known/agent-card.json","html":true,"json":null,"present":false}},"rings":{"v":1,"candidates":[],"mcp":[],"a2a":null,"webmcp":{"model_context":false,"bridge":false,"mcp_json":false,"link_rel":false,"any":false},"handshakes":0,"get_path":{"status":404,"content_type":"text/html","ms":493,"error":null},"get_host":{"status":200,"content_type":"text/html","ms":1423,"error":null}},"sf":[4,0,0,0,0,0,1,0,0],"derived":{"reachable":true,"home_class":"answers","robots_served":true,"robots_sitemap":true,"content_signal":false,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":[],"blocks_any_ai":false,"files":{"llms_txt":true,"agents_md":false,"agent_skills":false,"mcp_card":false,"api_catalog":false,"a2a_card":false},"any_capability":false,"ai_refused":[],"ai_thin":[],"aipref":0,"aipref_measured":true,"sf":[4,0,0,0,0,0,1,0,0],"rings":{"mcp_declared":false,"mcp_live":false,"mcp_auth":false,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":3,"fh":"wordpress.org","fp_other_host":[],"fp":"39fbeed5967399e0","fp_bits":{"gptbot":0,"claudebot":0,"oaisearch":0,"perplexity":0}},"derive_v":5,"frame":"top10k","shard":0},{"v":1,"kind":"crawlcheck-census-record","run":18,"domain":"digicert.com","rank":50,"measured_at":"2026-11-13T18:48:18.168Z","home":{"browser":{"status":200,"content_type":"text/html","bytes":69492,"sha256":"50cee22a76441a172904da4c810068549120e05c10f5566778cabefb61abb958","final_host":"www.digicert.com","final_path":"/","ms":986,"identity":"Mobile browser","html":true,"fp":"9119de7a93913e97"},"gptbot":{"status":200,"content_type":"text/html","bytes":69491,"sha256":"1a4c80981d44ec81181375e702210e2a7062fa7cc45187fdca2cd351dacdad0c","final_host":"www.digicert.com","final_path":"/","ms":830,"identity":"GPTBot","html":true,"fp":"9119de7a93913e97"},"claudebot":{"status":200,"content_type":"text/html","bytes":69491,"sha256":"e93d6ad5ae15c73405a693d618f795c065b90892fc5efc1df9e9c61530c2cd16","final_host":"www.digicert.com","final_path":"/","ms":826,"identity":"ClaudeBot","html":true,"fp":"9119de7a93913e97"},"oaisearch":{"status":200,"content_type":"text/html","bytes":69492,"sha256":"07d9fd8b77c462f854c025a886339e47e800fd9ce987bc32b8195ac88ee382d3","final_host":"www.digicert.com","final_path":"/","ms":771,"identity":"OAI-SearchBot","html":true,"fp":"9119de7a93913e97"},"perplexity":{"status":200,"content_type":"text/html","bytes":1035,"sha256":"18c643bc8d837fad1deba8ccc399992ab21cb4c282400c0caf8d978b533e2008","final_host":"www.digicert.com","final_path":"/","ms":872,"identity":"PerplexityBot","html":true}},"robots":{"status":200,"content_type":"text/plain","bytes":2605,"sha256":"a54fb2d2729303e9c3fe3b38d8be5112f7794d7423013e38d0234df03ed19bc5","final_host":"www.digicert.com","final_path":"/robots.txt","ms":1436,"html":false,"text":"# Welcome to DigiCert\r\n# Think you have what it takes to join the best team in the world?\r\n# Check out our careers while you're here\r\n# https://www.digicert.com/careers\r\n# Or check out our YouTube channel for our latest\r\n# https://www.youtube.com/user/DigiCertSSL\r\n\r\nUser-agent: *\r\nDisallow: /account/sso/idp\r\nDisallow: */test/\r\nDisallow: */ni/\r\nDisallow: */gated-content-pdfs/\r\nDisallow: /content/dam/digicert/pdfs/ni/\r\nDisallow: /account/saml-guest-request/idp\r\nDisallow: /webinars/*\r\nDisallow: /seals/\r\nDisallow: /link/\r\nDisallow: /secure/*\r\nDisallow: /services/*\r\nDisallow: /in-console-ssm\r\nDisallow: /jp/sid-partner*\r\nDisallow: /ppc/\r\nDisallow: /ru/\r\nDisallow: /ar/\r\nDisallow: /dc/blog/*\r\nDisallow: /dc/news/*\r\nDisallow: /author/*\r\nDisallow: /tag/*\r\nDisallow: /clients/*\r\nDisallow: /voucher/*\r\nDisallow: /bios*\r\nDisallow: /direct/*\r\nDisallow: /*thank\r\nDisallow: /*.xls$\r\nDisallow: /*.docx$\r\nDisallow: /language-masters/*\r\nDisallow: /link*\r\nDisallow: /*feed\r\n\r\nAllow: /*.js\r\nAllow: /*.css\r\n\r\n# AI / answer-engine crawlers\r\n\r\nUser-agent: GPTBot\r\nAllow: /\r\n\r\nUser-agent: OAI-SearchBot\r\nAllow: /\r\n\r\nUser-agent: ChatGPT-User\r\nAllow: /\r\n\r\nUser-agent: PerplexityBot\r\nAllow: /\r\n\r\nUser-agent: Perplexity-User\r\nAllow: /\r\n\r\nUser-agent: ClaudeBot\r\nAllow: /\r\n\r\nUser-agent: Claude-SearchBot\r\nAllow: /\r\n\r\nUser-agent: Claude-User\r\nAllow: /\r\n\r\nUser-agent: Googlebot-Image\r\nAllow: /\r\n\r\nUser-agent: AdsBot-Google\r\nAllow: /\r\n\r\nUser-agent: Google-Extended\r\nAllow: /\r\n\r\nSitemap: https://www.digicert.com/sitemap.xml\r\nSitemap: https://www.digicert.com/cn/sitemap.xml\r\nSitemap: https://www.digicert.com/de/sitemap.xml\r\nSitemap: https://www.digicert.com/es/sitemap.xml\r\nSitemap: https://www.digicert.com/fr/sitemap.xml\r\nSitemap: https://www.digicert.com/it/sitemap.xml\r\nSitemap: https://www.digicert.com/jp/sitemap.xml\r\nSitemap: https://www.digicert.com/kr/sitemap.xml\r\nSitemap: https://www.digicert.com/nl/sitemap.xml\r\nSitemap: https://www.digicert.com/tw/sitemap.xml\r\nSitemap: https://www.digicert.com/pt/sitemap.xml\r\nSitemap: https://www.digicert.com/faq/sitemap.xml\r\nSitemap: https://www.digicert.com/blog/sitemap.xml\r\nSitemap: https://www.digicert.com/eds-sitemap.xml\r\n\r\n# Bot Requests\r\n\r\nUser-agent: XoviBot\r\nDisallow: /\r\n\r\nUser-agent: BLEXBot\r\nDisallow: /\r\n\r\nUser-agent: DotBot\r\nDisallow: /\r\n\r\nUser-agent: MJ12bot\r\nDisallow: /\r\n\r\nUser-agent: Xenu's\r\nDisallow: /\r\n\r\nUser-agent: Xenu's Link Sleuth 1.1c\r\nDisallow: /\r\n\r\nUser-agent: Zeus\r\nDisallow: /\r\n\r\nUser-agent: Zeus 32297 Webster Pro V2.9 Win32\r\nDisallow: /\r\n\r\nUser-agent: Updownerbot\r\nDisallow: /\r\n\r\nUser-agent: Blekkobot\r\nDisallow: /\r\n\r\nUser-agent: 008\r\nDisallow: /","agents":{"GPTBot":true,"OAI-SearchBot":true,"ChatGPT-User":true,"ClaudeBot":true,"Claude-SearchBot":true,"PerplexityBot":true,"Google-Extended":true,"Applebot-Extended":true,"CCBot":true,"Bytespider":true,"Meta-ExternalAgent":true,"Amazonbot":true}},"sitemap":{"status":200,"content_type":"text/html","bytes":1039,"sha256":"5f36781a99afb4e0434ac3f225786fa250b2d54e93b3808299f50381bef7fff0","final_host":"www.digicert.com","final_path":"/sitemap.xml","ms":35,"url":"https://www.digicert.com/sitemap.xml","declared":true,"xml":false},"files":{"llms_txt":{"status":200,"content_type":"text/plain","bytes":8395,"sha256":"02169ec84a351a1d16d7d35ae9186f3edaa67dc827c2a7b55f9cea8f32abd439","final_host":"www.digicert.com","final_path":"/llms.txt","ms":1224,"path":"/llms.txt","html":false,"json":null,"present":true},"agents_md":{"status":200,"content_type":"text/html","bytes":1036,"sha256":"a5d4789276edc5d17d5a7ad94a47f96126b0ad3a116ecedb17206a828356fb99","final_host":"www.digicert.com","final_path":"/agents.md","ms":798,"path":"/agents.md","html":true,"json":null,"present":false},"agent_skills":{"status":200,"content_type":"text/html","bytes":1039,"sha256":"425d26b3fa34e9cbd0b4922a4011329201d5c52383a00ac920c455257a47fbac","final_host":"www.digicert.com","final_path":"/.well-known/agent-skills/index.json","ms":829,"path":"/.well-known/agent-skills/index.json","html":true,"json":null,"present":false},"mcp_card":{"status":200,"content_type":"text/html","bytes":1040,"sha256":"eed69a71a2725ce1bc6d186489d49dcf74365842d54fe04b45bb8552f5e9ba6d","final_host":"www.digicert.com","final_path":"/.well-known/mcp/server-card.json","ms":766,"path":"/.well-known/mcp/server-card.json","html":true,"json":null,"present":false},"api_catalog":{"status":200,"content_type":"text/html","bytes":1038,"sha256":"6e8a106ad404a747583908789ff10ccdbcf3f4bde35fd2b6724db2a5611afc32","final_host":"www.digicert.com","final_path":"/.well-known/api-catalog","ms":733,"path":"/.well-known/api-catalog","html":true,"json":null,"present":false},"a2a_card":{"status":200,"content_type":"text/html","bytes":1038,"sha256":"64939f087c56963eb8c15e175aef98ff38678d2c84c78993713b6f176f84543c","final_host":"www.digicert.com","final_path":"/.well-known/agent-card.json","ms":218,"path":"/.well-known/agent-card.json","html":true,"json":null,"present":false}},"rings":{"v":1,"candidates":[],"mcp":[],"a2a":null,"webmcp":{"model_context":false,"bridge":false,"mcp_json":false,"link_rel":false,"any":false},"handshakes":0,"get_path":{"status":200,"content_type":"text/html","ms":218,"error":null},"get_host":{"status":404,"content_type":null,"ms":955,"error":null}},"sf":[1,0,0,0,0,0,0,0,0],"derived":{"reachable":true,"home_class":"answers","robots_served":true,"robots_sitemap":true,"content_signal":false,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":[],"blocks_any_ai":false,"files":{"llms_txt":true,"agents_md":false,"agent_skills":false,"mcp_card":false,"api_catalog":false,"a2a_card":false},"any_capability":false,"ai_refused":[],"ai_thin":["perplexity"],"aipref":0,"aipref_measured":true,"sf":[1,0,0,0,0,0,0,0,0],"rings":{"mcp_declared":false,"mcp_live":false,"mcp_auth":false,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":3,"fh":"www.digicert.com","fp_other_host":[],"fp":"9119de7a93913e97","fp_bits":{"gptbot":0,"claudebot":0,"oaisearch":0}},"derive_v":5,"frame":"top10k","shard":1},{"v":1,"kind":"crawlcheck-census-record","run":18,"domain":"skype.com","rank":51,"measured_at":"2026-11-13T18:49:51.543Z","home":{"browser":{"status":200,"content_type":"text/html","bytes":61987,"sha256":"142e9658d6b9d0f524c5e085963fdeba54d9594b411cfb648f5f1c541b2b2d37","final_host":"teams.live.com","final_path":"/free","ms":313,"identity":"Mobile browser","html":true,"fp":"e138e2f4827477b1"},"gptbot":{"status":200,"content_type":"text/html","bytes":61966,"sha256":"343dbeb7eae3dfdeee8f8bef5ce243dc9b81f38b478a412608ee185463732505","final_host":"teams.live.com","final_path":"/free","ms":111,"identity":"GPTBot","html":true,"fp":"e138e2f4827477b1"},"claudebot":{"status":200,"content_type":"text/html","bytes":61987,"sha256":"824cb98febe10fac536c4a1b0f51bb96c6cd5cd0b26fa2a735274a2043b94f61","final_host":"teams.live.com","final_path":"/free","ms":295,"identity":"ClaudeBot","html":true,"fp":"e138e2f4827477b1"},"oaisearch":{"status":200,"content_type":"text/html","bytes":61987,"sha256":"d905ca06a0aa92d05cefa9a55a1f82fbf459eee7ebc5991f2a98e51f616a9a1d","final_host":"teams.live.com","final_path":"/free","ms":949,"identity":"OAI-SearchBot","html":true,"fp":"e138e2f4827477b1"},"perplexity":{"status":200,"content_type":"text/html","bytes":61966,"sha256":"36665a8a552be2ff3c19b18950acdab2651ddaefee5c781b0e6f4b825074e35b","final_host":"teams.live.com","final_path":"/free","ms":976,"identity":"PerplexityBot","html":true,"fp":"e138e2f4827477b1"}},"robots":{"status":200,"content_type":"text/html","bytes":61987,"sha256":"3f45a68acb7d68fe8952c5b13209134c6680d2eb92db6404f11f3b4a2cc5a36e","final_host":"teams.live.com","final_path":"/free","ms":203,"html":true,"text":null,"agents":null},"sitemap":{"status":200,"content_type":"text/html","bytes":61987,"sha256":"3670bd7c15b967514b8ff8da1417e8e48db6b53acfd2a146b24b4fb351f8bb3e","final_host":"teams.live.com","final_path":"/free","ms":996,"url":"https://skype.com/sitemap.xml","declared":false,"xml":false},"files":{"llms_txt":{"status":200,"content_type":"text/html","bytes":61987,"sha256":"61963b21d49402f5d4ff3176e4978fe38f829427dab6758c746c5ac96325d7a3","final_host":"teams.live.com","final_path":"/free","ms":560,"path":"/llms.txt","html":true,"json":null,"present":false},"agents_md":{"status":200,"content_type":"text/html","bytes":61966,"sha256":"6c8d8670e5c3ab29dde8a1c8592b5cea8947930b78930fafdc6fd6eec7e5be61","final_host":"teams.live.com","final_path":"/free","ms":529,"path":"/agents.md","html":true,"json":null,"present":false},"agent_skills":{"status":200,"content_type":"text/html","bytes":61987,"sha256":"f77defb9982d0faf7505fa4f03e99ee386c5a82edd545787284e3b012ae95b0d","final_host":"teams.live.com","final_path":"/free","ms":528,"path":"/.well-known/agent-skills/index.json","html":true,"json":null,"present":false},"mcp_card":{"status":200,"content_type":"text/html","bytes":61987,"sha256":"e8d6dada6f247c305596d50117164ac4eeca048ddfeb271e74f1af3c9da3ff3a","final_host":"teams.live.com","final_path":"/free","ms":1004,"path":"/.well-known/mcp/server-card.json","html":true,"json":null,"present":false},"api_catalog":{"status":200,"content_type":"text/html","bytes":61987,"sha256":"74d8ca6fea9a96342d5cbae44b714f6bda16fce2904db8150c70cd5ea41a1899","final_host":"teams.live.com","final_path":"/free","ms":428,"path":"/.well-known/api-catalog","html":true,"json":null,"present":false},"a2a_card":{"status":200,"content_type":"text/html","bytes":61987,"sha256":"4be5e324047c51004a7f586cb64feb1246fa2a3c8a69d65ee358e200f5881eac","final_host":"teams.live.com","final_path":"/free","ms":2230,"path":"/.well-known/agent-card.json","html":true,"json":null,"present":false}},"rings":{"v":1,"candidates":[],"mcp":[],"a2a":null,"webmcp":{"model_context":false,"bridge":false,"mcp_json":false,"link_rel":false,"any":false},"handshakes":0,"get_path":{"status":200,"content_type":"text/html","ms":77,"error":null},"get_host":{"status":530,"content_type":"text/plain","ms":13,"error":null}},"sf":[1,0,0,0,0,0,2,0,0],"derived":{"reachable":true,"home_class":"answers","robots_served":false,"robots_sitemap":false,"content_signal":false,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":[],"blocks_any_ai":false,"files":{"llms_txt":false,"agents_md":false,"agent_skills":false,"mcp_card":false,"api_catalog":false,"a2a_card":false},"any_capability":false,"ai_refused":[],"ai_thin":[],"sf":[1,0,0,0,0,0,2,0,0],"rings":{"mcp_declared":false,"mcp_live":false,"mcp_auth":false,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":3,"fh":"teams.live.com","fp_other_host":[],"fp":"e138e2f4827477b1","fp_bits":{"gptbot":0,"claudebot":0,"oaisearch":0,"perplexity":0}},"derive_v":5,"frame":"top10k","shard":2},{"v":1,"kind":"crawlcheck-census-record","run":18,"domain":"icloud.com","rank":52,"measured_at":"2026-11-13T18:50:46.493Z","home":{"browser":{"status":200,"content_type":"text/html","bytes":31452,"sha256":"c717aece2a7ab8c0b7f3aa9d30bdccac3311b4873c7965bd62fe72041b3ed299","final_host":"www.icloud.com","final_path":"/","ms":97,"identity":"Mobile browser","html":true},"gptbot":{"status":200,"content_type":"text/html","bytes":31452,"sha256":"c717aece2a7ab8c0b7f3aa9d30bdccac3311b4873c7965bd62fe72041b3ed299","final_host":"www.icloud.com","final_path":"/","ms":25,"identity":"GPTBot","html":true},"claudebot":{"status":200,"content_type":"text/html","bytes":31452,"sha256":"c717aece2a7ab8c0b7f3aa9d30bdccac3311b4873c7965bd62fe72041b3ed299","final_host":"www.icloud.com","final_path":"/","ms":26,"identity":"ClaudeBot","html":true},"oaisearch":{"status":200,"content_type":"text/html","bytes":31452,"sha256":"c717aece2a7ab8c0b7f3aa9d30bdccac3311b4873c7965bd62fe72041b3ed299","final_host":"www.icloud.com","final_path":"/","ms":80,"identity":"OAI-SearchBot","html":true},"perplexity":{"status":200,"content_type":"text/html","bytes":31452,"sha256":"c717aece2a7ab8c0b7f3aa9d30bdccac3311b4873c7965bd62fe72041b3ed299","final_host":"www.icloud.com","final_path":"/","ms":25,"identity":"PerplexityBot","html":true}},"robots":{"status":200,"content_type":"text/plain","bytes":1289,"sha256":"7c0da5b622a8703a69e156d3df581d1a2c5d4106a0da2e85c4db8f18c5a9e9c8","final_host":"www.icloud.com","final_path":"/robots.txt","ms":81,"html":false,"text":"\nSitemap: https://www.icloud.com/sitemap.xml\nSitemap: https://www.icloud.com/sitemap-info.xml\n\nUser-agent: *\n\nDisallow: /unsupported_browser\nDisallow: /unsupported_mobile\nDisallow: /enable_cookies\nDisallow: /maintenance\nDisallow: /not_found\nDisallow: /unavailable\nDisallow: /attachment/\nDisallow: /download/\nDisallow: /invites/\nDisallow: /iw/\nDisallow: /bookmarks_extension/\nDisallow: /family_sharing/\nDisallow: /calendar/eventreply/\nDisallow: /fmf_getting_started/\nDisallow: /fmf_location_update/\nDisallow: /fmf_location_update_unsubscribe_undo/\nDisallow: /fmf_unsubscribe/\nDisallow: /fmf_unsubscribe_error/\nDisallow: /fmf_unsubscribe_success/\nDisallow: /fmf_unsubscribe_undo/\nDisallow: /fmf_unsupported_device/\nDisallow: /photos_get_started/\nDisallow: /sharedalbum_unsubscribe/\nDisallow: /sharedalbum_unsubscribe_error/\nDisallow: /sharedalbum_unsubscribe_undo/\nDisallow: /sharedalbum_unsupported_device/\nDisallow: /photostream_unsubscribe/\nDisallow: /iclouddrive/\nDisallow: /notes/\nDisallow: /photos/\nDisallow: /reminders/\nDisallow: /share/\nDisallow: /pages/\nDisallow: /numbers/\nDisallow: /keynote/\nDisallow: /keynote-live/\nDisallow: /sharedalbum/\nDisallow: /ca/ics/\nDisallow: /shared/\n\nUser-agent: Googlebot\nAllow: /\n\nUser-agent: facebookexternalhit\nAllow: /invites/\nAllow: /opengraph/\n","agents":{"GPTBot":true,"OAI-SearchBot":true,"ChatGPT-User":true,"ClaudeBot":true,"Claude-SearchBot":true,"PerplexityBot":true,"Google-Extended":true,"Applebot-Extended":true,"CCBot":true,"Bytespider":true,"Meta-ExternalAgent":true,"Amazonbot":true}},"sitemap":{"status":200,"content_type":"text/xml","bytes":3442,"sha256":"b73212a376f2901d9ff3ec1bb8d59999116437dc44b19691c5ccc0fbcce60430","final_host":"www.icloud.com","final_path":"/sitemap.xml","ms":34,"url":"https://www.icloud.com/sitemap.xml","declared":true,"xml":true},"files":{"llms_txt":{"status":404,"content_type":"text/html","bytes":3250,"sha256":"836485c3395d88777be49942e7f66bdc1a0f1ed982533ff2d0cbeebd52d3f5dc","final_host":"www.icloud.com","final_path":"/llms.txt","ms":339,"path":"/llms.txt","html":true,"json":null,"present":false},"agents_md":{"status":404,"content_type":"text/html","bytes":3250,"sha256":"836485c3395d88777be49942e7f66bdc1a0f1ed982533ff2d0cbeebd52d3f5dc","final_host":"www.icloud.com","final_path":"/agents.md","ms":127,"path":"/agents.md","html":true,"json":null,"present":false},"agent_skills":{"status":404,"content_type":"text/html","bytes":3250,"sha256":"836485c3395d88777be49942e7f66bdc1a0f1ed982533ff2d0cbeebd52d3f5dc","final_host":"www.icloud.com","final_path":"/.well-known/agent-skills/index.json","ms":152,"path":"/.well-known/agent-skills/index.json","html":true,"json":null,"present":false},"mcp_card":{"status":404,"content_type":"text/html","bytes":3250,"sha256":"836485c3395d88777be49942e7f66bdc1a0f1ed982533ff2d0cbeebd52d3f5dc","final_host":"www.icloud.com","final_path":"/.well-known/mcp/server-card.json","ms":115,"path":"/.well-known/mcp/server-card.json","html":true,"json":null,"present":false},"api_catalog":{"status":404,"content_type":"text/html","bytes":3250,"sha256":"836485c3395d88777be49942e7f66bdc1a0f1ed982533ff2d0cbeebd52d3f5dc","final_host":"www.icloud.com","final_path":"/.well-known/api-catalog","ms":98,"path":"/.well-known/api-catalog","html":true,"json":null,"present":false},"a2a_card":{"status":404,"content_type":"text/html","bytes":3250,"sha256":"836485c3395d88777be49942e7f66bdc1a0f1ed982533ff2d0cbeebd52d3f5dc","final_host":"www.icloud.com","final_path":"/.well-known/agent-card.json","ms":103,"path":"/.well-known/agent-card.json","html":true,"json":null,"present":false}},"rings":{"v":1,"candidates":[],"mcp":[],"a2a":null,"webmcp":{"model_context":false,"bridge":false,"mcp_json":false,"link_rel":false,"any":false},"handshakes":0,"get_path":{"status":404,"content_type":"text/html","ms":160,"error":null},"get_host":{"status":530,"content_type":"text/plain","ms":5,"error":null}},"sf":[-1,0,0,0,0,0,0,0,1],"derived":{"reachable":true,"home_class":"answers","robots_served":true,"robots_sitemap":true,"content_signal":false,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":[],"blocks_any_ai":false,"files":{"llms_txt":false,"agents_md":false,"agent_skills":false,"mcp_card":false,"api_catalog":false,"a2a_card":false},"any_capability":false,"ai_refused":[],"ai_thin":[],"aipref":0,"aipref_measured":true,"sf":[-1,0,0,0,0,0,0,0,1],"rings":{"mcp_declared":false,"mcp_live":false,"mcp_auth":false,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":3,"fh":"www.icloud.com","fp_other_host":[]},"derive_v":5,"frame":"top10k","shard":3},{"v":1,"kind":"crawlcheck-census-record","run":18,"domain":"pinterest.com","rank":53,"measured_at":"2026-11-13T18:51:40.579Z","home":{"browser":{"status":200,"content_type":"text/html","bytes":1255832,"sha256":"85b07ad12d72439a015afaa8056ce5258c656574c0a6dde6f0a91542d0ff81ab","final_host":"www.pinterest.com","final_path":"/","ms":151,"identity":"Mobile browser","html":true,"fp":"415139f3e79f27eb"},"gptbot":{"status":403,"content_type":"text/html","bytes":2987,"sha256":"5afc53096d3c1b9f4943017fabdcc2ff37f27c2b726f6cf8413c4b70215ac094","final_host":"www.pinterest.com","final_path":"/","ms":50,"identity":"GPTBot","html":true},"claudebot":{"status":200,"content_type":"text/html","bytes":1354595,"sha256":"3b0fa258d3d35cabb187c5e509b2daf2bec4ba7f1de8e20b9fc229a1fef7bc4d","final_host":"www.pinterest.com","final_path":"/","ms":230,"identity":"ClaudeBot","html":true,"fp":"632525e1f52f3f79"},"oaisearch":{"status":403,"content_type":"text/html","bytes":2987,"sha256":"4e08cc2de344d36fed0777f188e29b41b37948131ce2b72b892f72ce5a090236","final_host":"www.pinterest.com","final_path":"/","ms":54,"identity":"OAI-SearchBot","html":true},"perplexity":{"status":403,"content_type":"text/html","bytes":2987,"sha256":"af1732026ad2cf11a57be97d2a6f9722f38a885a629fcb8226debfd5c99e90a2","final_host":"www.pinterest.com","final_path":"/","ms":43,"identity":"PerplexityBot","html":true}},"robots":{"status":200,"content_type":"text/plain","bytes":32989,"sha256":"70b3dde94cfacc9c148e3f40a596c0bacd867e2fced737ca18655d0570c17ac2","final_host":"www.pinterest.com","final_path":"/robots.txt","ms":82,"html":false,"text":"# Allowlist your bot: https://help.pinterest.com/bot-submission-form\n#\n# Pinterest is hiring!\n#\n# Learn about the SEO work that we're doing at https://medium.com/@Pinterest_Engineering/demystifying-seo-with-experiments-a183b325cf4c\n#\n# Check out some of our available positions at https://www.pinterestcareers.com/\n\nUser-agent: 2checkout\nUser-agent: AASA-Bot\nUser-agent: AccessibleWebBot\nUser-agent: Adagiobot\nUser-agent: adbeat_bot\nUser-agent: ADmantX\nUser-agent: AdometryBot\nUser-agent: Adsense-Snapshot-Google\nUser-agent: AdsTxtCrawler\nUser-agent: Adyen\nUser-agent: AhrefsBot\nUser-agent: AhrefsSiteAudit\nUser-agent: Akismet\nUser-agent: Algolia Crawler\nUser-agent: AmazonProductDiscovery\nUser-agent: Aolbot-News\nUser-agent: APIs-Google\nUser-agent: AppInsights\nUser-agent: Applebot\nUser-agent: AppleNewsBot\nUser-agent: Archive-It\nUser-agent: archive.org_bot\nUser-agent: ArchiveBot\nUser-agent: ArchiveTeam-ArchiveBot\nUser-agent: AspiegelBot\nUser-agent: Audisto Crawler\nUser-agent: Awario\nUser-agent: Baidu-ADS\nUser-agent: Baiduspider\nUser-agent: Barkrowler\nUser-agent: bingbot\nUser-agent: BingPreview\nUser-agent: bitlybot\nUser-agent: bl.uk_ldfc_bot\nUser-agent: BLP_bbot\nUser-agent: bordermanager\nUser-agent: botify\nUser-agent: BrandVerity\nUser-agent: Bravebot\nUser-agent: BrightEdge Crawler\nUser-agent: BrightEdge-Crawler\nUser-agent: BUbiNG\nUser-agent: CaliperBot\nUser-agent: CapitalOneBot\nUser-agent: Catchpoint\nUser-agent: cf-qualys-scanner\nUser-agent: Chrome-Lighthouse\nUser-agent: CJBot\nUser-agent: ClarityBot\nUser-agent: Clickagy Intelligence Bot v2\nUser-agent: Cloudflare-AI-Search\nUser-agent: CloudflareBrowserRenderingCrawler\nUser-agent: coccocbot\nUser-agent: ContextualBot\nUser-agent: CookieReports\nUser-agent: Coveobot\nUser-agent: CriteoBot\nUser-agent: curly-\nUser-agent: cXensebot\nUser-agent: Datadog\nUser-agent: DataForSeoBot\nUser-agent: Daum\nUser-agent: demandbase\nUser-agent: Discordbot\nUser-agent: DotBot\nUser-agent: DoubleVerifyBot\nUser-agent: Doximity-Pipeline\nUser-agent: Dragonbot\nUser-agent: Dratabot\nUser-agent: DuckAssistBot\nUser-agent: DuckDuckBot\nUser-agent: DuckDuckGo-Favicons-Bot\nUser-agent: DVBot\nUser-agent: eContext\nUser-agent: ecurly Webhook\nUser-agent: EngageBDR\nUser-agent: europarchive.org\nUser-agent: evidon\nUser-agent: ExaleadCloudview\nUser-agent: FacebookBot\nUser-agent: facebookcatalog\nUser-agent: facebookexternalhit\nUser-agent: Facebot\nUser-agent: FastDAST\nUser-agent: FeedBurner\nUser-agent: Feedfetcher-Google\nUser-agent: FlipboardProxy\nUser-agent: FlippBot\nUser-agent: Freshping\nUser-agent: GeedoProductSearch\nUser-agent: Google Favicon\nUser-agent: Google News\nUser-agent: Google Page Speed Insights\nUser-agent: Google Wireless Transcoder\nUser-agent: Google-Apps-Script\nUser-agent: Google-Certificates-Bridge\nUser-agent: Google-CWS\nUser-agent: Google-Display-Ads-Bot\nUser-agent: Google-Firebase\nUser-agent: Google-HotelAdsVerifier\nUser-agent: Google-InspectionTool\nUser-agent: Google-NotebookLM\nUser-agent: Google-PageRenderer\nUser-agent: Google-PhysicalWeb\nUser-agent: Google-Pinpoint\nUser-agent: Google-Read-Aloud\nUser-agent: Google-Safety\nUser-agent: Google-Site-Verification\nUser-agent: Google-speakr\nUser-agent: Google-Structured-Data-Testing\nUser-agent: Google-Trust-Services\nUser-agent: google_partner_monitoring\nUser-agent: GoogleAssociationService\nUser-agent: GoogleImageProxy\nUser-agent: GoogleMessages\nUser-agent: GoogleProducer\nUser-agent: GoogleSecurityScanner\nUser-agent: grafana/synthetic-monitoring\nUser-agent: GroovinaAdsbot\nUser-agent: GuestpostsBot\nUser-agent: heritrix\nUser-agent: Hotjar\nUser-agent: HubSpot\nUser-agent: ia_archiver\nUser-agent: IABot\nUser-agent: IAS-Crawler\nUser-agent: ias_crawler\nUser-agent: ias_wombles\nUser-agent: INA dlweb\nUser-agent: infolink\nUser-agent: InteractiveAdvertisingBureau\nUser-agent: ip-label\nUser-agent: J-BRW\nUser-agent: J-DLC\nUser-agent: James BOT\nUser-agent: JobRapido\nUser-agent: Jooblebot\nUser-agent: Jumio\nUser-agent: kakaotalk-scrap\nUser-agent: KargoBot-Artemis\nUser-agent: Leikibot\nUser-agent: LiftoffDroid\nUser-agent: line-poker\nUser-agent: Linguee Bot\nUser-agent: linkchecker.pro\nUser-agent: LOCKSS cache\nUser-agent: LogicMonitor SiteMonitor\nUser-agent: Lyonl\nUser-agent: magpie-crawler\nUser-agent: Mastodon\nUser-agent: MaxPoint Bot\nUser-agent: MBCrawler\nUser-agent: mediaboardbot\nUser-agent: MediaMonitoringBot\nUser-agent: Mediatoolkitbot\nUser-agent: MeltwaterNews\nUser-agent: Metorik\nUser-agent: MgidBot\nUser-agent: MicrosoftPreview\nUser-agent: MixrankBot\nUser-agent: MJ12bot\nUser-agent: moatbot\nUser-agent: MojeekBot\nUser-agent: Monsidobot\nUser-agent: Moreover\nUser-agent: Morningscore Bot\nUser-agent: MSNPTC\nUser-agent: msrbot\nUser-agent: MTRobot\nUser-agent: netarkivindsamling\nUser-agent: NetpeakCheckerBot\nUser-agent: NeumobBot\nUser-agent: Neustar\nUser-agent: Nielsen\nUser-agent: NostoCrawlerBot\nUser-agent: NumeratorBot\nUser-agent: OdklBot\nUser-agent: OKX-dolphin-crawler\nUser-agent: OnCrawl\nUser-agent: OneTrust\nUser-agent: OPBot\nUser-agent: OSZKbot\nUser-agent: outbrain\nUser-agent: Paddle\nUser-agent: PageFreezer\nUser-agent: Pandalytics\nUser-agent: parsely.com\nUser-agent: ParselySharesBot\nUser-agent: PayPal\nUser-agent: peer39_crawler\nUser-agent: Pinterest\nUser-agent: Pinterestbot\nUser-agent: Plukkie\nUser-agent: Prerender\nUser-agent: PriceSpider\nUser-agent: ProjectShield-UrlCheck\nUser-agent: proximic\nUser-agent: QualifiedBot\nUser-agent: Quantcastbot\nUser-agent: Qwantbot\nUser-agent: Qwantify\nUser-agent: Razorpay\nUser-agent: Rigor\nUser-agent: RiverglassScanner\nUser-agent: RKG Url Verifier\nUser-agent: rogerbot-crawler\nUser-agent: RuxitSynthetic\nUser-agent: scanalert\nUser-agent: search.marginalia.nu\nUser-agent: SearchAtlas Bot\nUser-agent: SeekportBot\nUser-agent: SemanticJuice\nUser-agent: SemrushBot\nUser-agent: SemrushBot-BA\nUser-agent: SemrushBot-OCOB\nUser-agent: SemrushBot-SA\nUser-agent: SemrushBot-SI\nUser-agent: SemrushBot-SWA\nUser-agent: sentry\nUser-agent: SentryUptimeBot\nUser-agent: seo-audit-check-bot\nUser-agent: SeobilityBot\nUser-agent: SERankingBacklinksBot\nUser-agent: SeznamBot\nUser-agent: SinceraSyntheticUser\nUser-agent: SISTRIX Optimizer\nUser-agent: SiteAuditBot\nUser-agent: sitebulb\nUser-agent: SiteCheckerBotCrawler\nUser-agent: Siteimprove-SiteCheck\nUser-agent: SiteSearch360\nUser-agent: SkypeUriPreview\nUser-agent: Slack-ImgProxy\nUser-agent: Slackbot\nUser-agent: Slurp\nUser-agent: SmarshBot\nUser-agent: SmartBriefBot\nUser-agent: Snap URL Preview Service\nUser-agent: SnapPreviewBot\nUser-agent: Sogou\nUser-agent: solarwinds\nUser-agent: spbot\nUser-agent: Splunk Synthetics\nUser-agent: SpringBot\nUser-agent: SSL Labs\nUser-agent: ssr.ias-lab.de\nUser-agent: Stape\nUser-agent: SteamChat\nUser-agent: Storebot-Google\nUser-agent: Stripe\nUser-agent: Sucuri\nUser-agent: TagInspector\nUser-agent: TelegramBot\nUser-agent: Teoma\nUser-agent: trendictionbot\nUser-agent: trovitBot\nUser-agent: Tumblr\nUser-agent: twibble.io\nUser-agent: TwinWaveScanner\nUser-agent: um-LN\nUser-agent: VaultPress\nUser-agent: vercel-screenshot\nUser-agent: Viber\nUser-agent: VigLink\nUser-agent: VocusBot\nUser-agent: W3C-checklink\nUser-agent: W3C_Validator\nUser-agent: Wappalyzer\nUser-agent: WhatsApp\nUser-agent: woorankreview\nUser-agent: Y!J-ASR\nUser-agent: Y!J-BRJ/YATS crawler\nUser-agent: Y!J-BRU/VSIDX\nUser-agent: Y!J-BRW\nUser-agent: Y!J-BRY/YATSH crawler\nUser-agent: Y!J-BRZ/YATSHA crawler\nUser-agent: Y!J-MMP/dscv\nUser-agent: Y!J-WSC\nUser-agent: YaDirectFetcher\nUser-agent: Yahoo Ad monitoring\nUser-agent: Yahoo Link Preview\nUser-agent: Yahoo! Slurp\nUser-agent: Yahoo-Japan-ASR\nUser-agent: YahooCacheSystem\nUser-agent: YahooExternalCache\nUser-agent: yahoofeedseeker\nUser-agent: YahooMailProxy\nUser-agent: YaK\nUser-agent: Yandex\nUser-agent: Yeti\nUser-agent: YextBot\nUser-agent: YioopBot\nUser-agent: YJ-SAD\nAllow: /_ngjs/resource/*/get/\nAllow: /business/create/\nAllow: /resource/*/get/\nDisallow: /*/*/*/_tools/*\nDisallow: /*/*/*/more_ideas/\nDisallow: /*/*/_tools/*\nDisallow: /*/*/activity/*\nDisallow: /*/*/group/\nDisallow: /*/*/invite/\nDisallow: /*/*/more_ideas/*\nDisallow: /*/?*amp_client_id*\nDisallow: /*/?z=1\nDisallow: /*/__wishlist__/*\nDisallow: /*/_activities/*\nDisallow: /*/_activity/*\nDisallow: /*/_community/*\nDisallow: /*/_created/*\nDisallow: /*/_followers/*\nDisallow: /*/_following/*\nDisallow: /*/_quick_saves/*\nDisallow: /*/_saved/*\nDisallow: /*/_shop/*\nDisallow: /*/_shopping_list/*\nDisallow: /*/communities/\nDisallow: /*/discovered/*\nDisallow: /*/followers/*\nDisallow: /*/following/*\nDisallow: /*/likes/*\nDisallow: /*/organize/*\nDisallow: /*/pins/\nDisallow: /*/profile-about/\nDisallow: /*/scheduled-pin/*\nDisallow: /*/scheduled-pins/*\nDisallow: /*/story_pins/*\nDisallow: /*/tagged/*\nDisallow: /*/tried/*\nDisallow: /*/video_pins/*\nDisallow: /?show_error=true\nDisallow: /_/_/*\nDisallow: /_/_/pixel/*\nDisallow: /_ngjs/resource/\nDisallow: /about/*\nDisallow: /addboard/\nDisallow: /admin/\nDisallow: /age-appeal/*\nDisallow: /amp/*\nDisallow: /appealed-pin/\nDisallow: /autocomplete/\nDisallow: /board/*\nDisallow: /board_activity/*\nDisallow: /board_activity_comment/*\nDisallow: /bookmarklet/\nDisallow: /branded-pins/\nDisallow: /business/ad_create/\nDisallow: /business/business-access/*\nDisallow: /business/business-hierarchy/*\nDisallow: /business/business-manager/*\nDisallow: /business/catalogs/*\nDisallow: /business/commerce-integrations/*\nDisallow: /business/community-information/\nDisallow: /business/convert/\nDisallow: /business/create-linked-account/\nDisallow: /business/hub/\nDisallow: /business/invite/*\nDisallow: /business/onboarding/\nDisallow: /business/pin_create/\nDisallow: /business/referral/*\nDisallow: /business/site/\nDisallow: /business/switch-account/\nDisallow: /business/top-ads/\nDisallow: /business/update/\nDisallow: /business/verified-merchant/status/\nDisallow: /christmas-spotlight/\nDisallow: /close-account/\nDisallow: /communities/*\nDisallow: /community/*\nDisallow: /community_comment/*\nDisallow: /community_post/*\nDisallow: /connect/*/\nDisallow: /content-claiming/*\nDisallow: /conversation/*\nDisallow: /conversations/*\nDisallow: /convert-business/\nDisallow: /convert-personal/\nDisallow: /copyright-reporting/*\nDisallow: /create_pin/\nDisallow: /creative-studio/\nDisallow: /creative_editor/\nDisallow: /creator-spotlight/*\nDisallow: /creator_hub/_earn\nDisallow: /creator_hub/_partnerships\nDisallow: /csrf_error/\nDisallow: /ct.html\nDisallow: /deactivate-account/\nDisallow: /delete_post/\nDisallow: /discover/*\nDisallow: /edit/*\nDisallow: /editorial/*/*/\nDisallow: /email/*\nDisallow: /email_verification_error/\nDisallow: /engagement/*\nDisallow: /explore/*/?p=*\nDisallow: /facebook/\nDisallow: /fb.html\nDisallow: /follow_board/\nDisallow: /follow_user/\nDisallow: /google_gdn.html\nDisallow: /google_search.html\nDisallow: /hanukkah-spotlight/\nDisallow: /holiday-spotlight/\nDisallow: /holidays-celebrations/\nDisallow: /homefeed/\nDisallow: /homefeed/*\nDisallow: /idea-ads-tool/*\nDisallow: /idea-pin-builder/*\nDisallow: /idea-pin-invite/\nDisallow: /install-shuffles/\nDisallow: /invite/\nDisallow: /invited/\nDisallow: /join/?next=*\nDisallow: /join/signup/?next=*\nDisallow: /kwanzaa-spotlight/\nDisallow: /lens-search/*\nDisallow: /linked/*\nDisallow: /live-session/*\nDisallow: /login/?next=*\nDisallow: /logout/\nDisallow: /m/\nDisallow: /new_invite/\nDisallow: /news_hub/*\nDisallow: /notifications/\nDisallow: /oauth/line/*\nDisallow: /oauth/login/*\nDisallow: /oauth_error/\nDisallow: /offsite/\nDisallow: /parental-consent/confirmation/\nDisallow: /parental-passcode/\nDisallow: /password/reset/\nDisallow: /password/reset/*\nDisallow: /pin-builder/*\nDisallow: /pin-communities/*\nDisallow: /pin-creation-tool/*\nDisallow: /pin-editor/\nDisallow: /pin/*/activities\nDisallow: /pin/*/activity/\nDisallow: /pin/*/analytics\nDisallow: /pin/*/brand-catalog/\nDisallow: /pin/*/brand-catalog/*\nDisallow: /pin/*/comments\nDisallow: /pin/*/email\nDisallow: /pin/*/feedback\nDisallow: /pin/*/idea-pin/related-products/*\nDisallow: /pin/*/likes\nDisallow: /pin/*/promote\nDisallow: /pin/*/reaction\nDisallow: /pin/*/related-products\nDisallow: /pin/*/related-products/*\nDisallow: /pin/*/repin\nDisallow: /pin/*/repins\nDisallow: /pin/*/share\nDisallow: /pin/*/shop-the-look/*\nDisallow: /pin/*/tag\nDisallow: /pin/*/takes\nDisallow: /pin/*/visual-search\nDisallow: /pin/*/visual-shop\nDisallow: /pin/create/*\nDisallow: /pin/find/\nDisallow: /pin/find?url=*\nDisallow: /pin_shop/*\nDisallow: /pinterest-predicts/\nDisallow: /pinterest-wellbeing/*\nDisallow: /pinterestwellbeing/*\nDisallow: /place/*\nDisallow: /prefs/\nDisallow: /product-catalogs/\nDisallow: /product-catalogs/*\nDisallow: /product-feed/\nDisallow: /pw/*\nDisallow: /quiz/*\nDisallow: /recently-viewed/\nDisallow: /refresh_stored_accounts\nDisallow: /refresh_token\nDisallow: /register/\nDisallow: /reports-and-violations/\nDisallow: /request-data/\nDisallow: /resource/\nDisallow: /safe-redirect/\nDisallow: /search/\nDisallow: /search?\nDisallow: /secure/*\nDisallow: /sendHeaderEarly/\nDisallow: /settings/*\nDisallow: /shop_pins/*\nDisallow: /signup/*\nDisallow: /socialmanager/*\nDisallow: /source/*\nDisallow: /sso/*\nDisallow: /story-pin-builder/*\nDisallow: /story-pin-invite/\nDisallow: /story_feed/\nDisallow: /story_pins/*\nDisallow: /suspension-appeal/*\nDisallow: /sw-shell.html\nDisallow: /taggingtool/*\nDisallow: /thanksgiving-spotlight/\nDisallow: /theme/*\nDisallow: /tv/*\nDisallow: /unauth-profile/\nDisallow: /unlink/\nDisallow: /upload-lens-image\nDisallow: /upload-lens-image-history\nDisallow: /upload-shuffle-draft\nDisallow: /v1/urls/count.json\nDisallow: /v3/pidgets/log/\nDisallow: /website/confirm/\nDisallow: /website/confirm/download*\nDisallow: /welcome/\nDisallow: /your-shop/\nDisallow: /your-shop/*\n\nUser-agent: Googlebot\nAllow: /_ngjs/resource/*/get/\nAllow: /business/create/\nAllow: /resource/*/get/\nDisallow: /*/*/*/_tools/*\nDisallow: /*/*/*/more_ideas/\nDisallow: /*/*/_tools/*\nDisallow: /*/*/activity/*\nDisallow: /*/*/group/\nDisallow: /*/*/invite/\nDisallow: /*/*/more_ideas/*\nDisallow: /*/?*amp_client_id*\nDisallow: /*/?z=1\nDisallow: /*/__wishlist__/*\nDisallow: /*/_activities/*\nDisallow: /*/_activity/*\nDisallow: /*/_community/*\nDisallow: /*/_created/*\nDisallow: /*/_followers/*\nDisallow: /*/_following/*\nDisallow: /*/_quick_saves/*\nDisallow: /*/_saved/*\nDisallow: /*/_shop/*\nDisallow: /*/_shopping_list/*\nDisallow: /*/communities/\nDisallow: /*/discovered/*\nDisallow: /*/followers/*\nDisallow: /*/following/*\nDisallow: /*/likes/*\nDisallow: /*/organize/*\nDisallow: /*/pins/\nDisallow: /*/profile-about/\nDisallow: /*/scheduled-pin/*\nDisallow: /*/scheduled-pins/*\nDisallow: /*/story_pins/*\nDisallow: /*/tagged/*\nDisallow: /*/tried/*\nDisallow: /*/video_pins/*\nDisallow: /?show_error=true\nDisallow: /_/_/*\nDisallow: /_/_/pixel/*\nDisallow: /_ngjs/resource/\nDisallow: /about/*\nDisallow: /addboard/\nDisallow: /admin/\nDisallow: /age-appeal/*\nDisallow: /amp/*\nDisallow: /appealed-pin/\nDisallow: /autocomplete/\nDisallow: /board/*\nDisallow: /board_activity/*\nDisallow: /board_activity_comment/*\nDisallow: /bookmarklet/\nDisallow: /branded-pins/\nDisallow: /business/ad_create/\nDisallow: /business/business-access/*\nDisallow: /business/business-hierarchy/*\nDisallow: /business/business-manager/*\nDisallow: /business/catalogs/*\nDisallow: /business/commerce-integrations/*\nDisallow: /business/community-information/\nDisallow: /business/convert/\nDisallow: /business/create-linked-account/\nDisallow: /business/hub/\nDisallow: /business/invite/*\nDisallow: /business/onboarding/\nDisallow: /business/pin_create/\nDisallow: /business/referral/*\nDisallow: /business/site/\nDisallow: /business/switch-account/\nDisallow: /business/top-ads/\nDisallow: /business/update/\nDisallow: /business/verified-merchant/status/\nDisallow: /christmas-spotlight/\nDisallow: /close-account/\nDisallow: /communities/*\nDisallow: /community/*\nDisallow: /community_comment/*\nDisallow: /community_post/*\nDisallow: /connect/*/\nDisallow: /content-claiming/*\nDisallow: /conversation/*\nDisallow: /conversations/*\nDisallow: /convert-business/\nDisallow: /convert-personal/\nDisallow: /copyright-reporting/*\nDisallow: /create_pin/\nDisallow: /creative-studio/\nDisallow: /creative_editor/\nDisallow: /creator-spotlight/*\nDisallow: /creator_hub/_earn\nDisallow: /creator_hub/_partnerships\nDisallow: /csrf_error/\nDisallow: /ct.html\nDisallow: /deactivate-account/\nDisallow: /delete_post/\nDisallow: /discover/*\nDisallow: /edit/*\nDisallow: /editorial/*/*/\nDisallow: /email/*\nDisallow: /email_verification_error/\nDisallow: /engagement/*\nDisallow: /explore/*\nDisallow: /explore/*/?p=*\nDisallow: /facebook/\nDisallow: /fb.html\nDisallow: /follow_board/\nDisallow: /follow_user/\nDisallow: /google_gdn.html\nDisallow: /google_search.html\nDisallow: /hanukkah-spotlight/\nDisallow: /holiday-spotlight/\nDisallow: /holidays-celebrations/\nDisallow: /homefeed/\nDisallow: /homefeed/*\nDisallow: /idea-ads-tool/*\nDisallow: /idea-pin-builder/*\nDisallow: /idea-pin-invite/\nDisallow: /install-shuffles/\nDisallow: /invite/\nDisallow: /invited/\nDisallow: /join/?next=*\nDisallow: /join/signup/?next=*\nDisallow: /kwanzaa-spotlight/\nDisallow: /lens-search/*\nDisallow: /linked/*\nDisallow: /live-session/*\nDisallow: /login/?next=*\nDisallow: /logout/\nDisallow: /m/\nDisallow: /new_invite/\nDisallow: /news_hub/*\nDisallow: /notifications/\nDisallow: /oauth/line/*\nDisallow: /oauth/login/*\nDisallow: /oauth_error/\nDisallow: /offsite/\nDisallow: /parental-consent/confirmation/\nDisallow: /parental-passcode/\nDisallow: /password/reset/\nDisallow: /password/reset/*\nDisallow: /pin-builder/*\nDisallow: /pin-communities/*\nDisallow: /pin-creation-tool/*\nDisallow: /pin-editor/\nDisallow: /pin/*/activities\nDisallow: /pin/*/activity/\nDisallow: /pin/*/analytics\nDisallow: /pin/*/brand-catalog/\nDisallow: /pin/*/brand-catalog/*\nDisallow: /pin/*/comments\nDisallow: /pin/*/email\nDisallow: /pin/*/feedback\nDisallow: /pin/*/idea-pin/related-products/*\nDisallow: /pin/*/likes\nDisallow: /pin/*/promote\nDisallow: /pin/*/reaction\nDisallow: /pin/*/related-products\nDisallow: /pin/*/related-products/*\nDisallow: /pin/*/repin\nDisallow: /pin/*/repins\nDisallow: /pin/*/share\nDisallow: /pin/*/shop-the-look/*\nDisallow: /pin/*/tag\nDisallow: /pin/*/takes\nDisallow: /pin/*/visual-search\nDisallow: /pin/*/visual-shop\nDisallow: /pin/create/*\nDisallow: /pin/find/\nDisallow: /pin/find?url=*\nDisallow: /pin_shop/*\nDisallow: /pinterest-predicts/\nDisallow: /pinterest-wellbeing/*\nDisallow: /pinterestwellbeing/*\nDisallow: /place/*\nDisallow: /prefs/\nDisallow: /product-catalogs/\nDisallow: /product-catalogs/*\nDisallow: /product-feed/\nDisallow: /pw/*\nDisallow: /quiz/*\nDisallow: /recently-viewed/\nDisallow: /refresh_stored_accounts\nDisallow: /refresh_token\nDisallow: /register/\nDisallow: /reports-and-violations/\nDisallow: /request-data/\nDisallow: /resource/\nDisallow: /safe-redirect/\nDisallow: /search/\nDisallow: /search?\nDisallow: /secure/*\nDisallow: /sendHeaderEarly/\nDisallow: /settings/*\nDisallow: /shop_pins/*\nDisallow: /signup/*\nDisallow: /socialmanager/*\nDisallow: /source/*\nDisallow: /sso/*\nDisallow: /story-pin-builder/*\nDisallow: /story-pin-invite/\nDisallow: /story_feed/\nDisallow: /story_pins/*\nDisallow: /suspension-appeal/*\nDisallow: /sw-shell.html\nDisallow: /taggingtool/*\nDisallow: /thanksgiving-spotlight/\nDisallow: /theme/*\nDisallow: /tv/*\nDisallow: /unauth-profile/\nDisallow: /unlink/\nDisallow: /upload-lens-image\nDisallow: /upload-lens-image-history\nDisallow: /upload-shuffle-draft\nDisallow: /v1/urls/count.json\nDisallow: /v3/pidgets/log/\nDisallow: /website/confirm/\nDisallow: /website/confirm/download*\nDisallow: /welcome/\nDisallow: /your-shop/\nDisallow: /your-shop/*\n\nUser-agent: Twitterbot\nAllow: /_ngjs/resource/*/get/\nAllow: /business/create/\nAllow: /pin/*/feedback\nAllow: /pin/*/sent\nAllow: /resource/*/get/\nDisallow: /*/*/*/_tools/*\nDisallow: /*/*/*/more_ideas/\nDisallow: /*/*/_tools/*\nDisallow: /*/*/activity/*\nDisallow: /*/*/group/\nDisallow: /*/*/invite/\nDisallow: /*/*/more_ideas/*\nDisallow: /*/?*amp_client_id*\nDisallow: /*/?z=1\nDisallow: /*/__wishlist__/*\nDisallow: /*/_activities/*\nDisallow: /*/_activity/*\nDisallow: /*/_community/*\nDisallow: /*/_created/*\nDisallow: /*/_followers/*\nDisallow: /*/_following/*\nDisallow: /*/_quick_saves/*\nDisallow: /*/_saved/*\nDisallow: /*/_shop/*\nDisallow: /*/_shopping_list/*\nDisallow: /*/communities/\nDisallow: /*/discovered/*\nDisallow: /*/followers/*\nDisallow: /*/following/*\nDisallow: /*/likes/*\nDisallow: /*/organize/*\nDisallow: /*/pins/\nDisallow: /*/profile-about/\nDisallow: /*/scheduled-pin/*\nDisallow: /*/scheduled-pins/*\nDisallow: /*/story_pins/*\nDisallow: /*/tagged/*\nDisallow: /*/tried/*\nDisallow: /*/video_pins/*\nDisallow: /?show_error=true\nDisallow: /_/_/*\nDisallow: /_/_/pixel/*\nDisallow: /_ngjs/resource/\nDisallow: /about/*\nDisallow: /addboard/\nDisallow: /admin/\nDisallow: /age-appeal/*\nDisallow: /amp/*\nDisallow: /appealed-pin/\nDisallow: /autocomplete/\nDisallow: /board/*\nDisallow: /board_activity/*\nDisallow: /board_activity_comment/*\nDisallow: /bookmarklet/\nDisallow: /branded-pins/\nDisallow: /business/ad_create/\nDisallow: /business/business-access/*\nDisallow: /business/business-hierarchy/*\nDisallow: /business/business-manager/*\nDisallow: /business/catalogs/*\nDisallow: /business/commerce-integrations/*\nDisallow: /business/community-information/\nDisallow: /business/convert/\nDisallow: /business/create-linked-account/\nDisallow: /business/hub/\nDisallow: /business/invite/*\nDisallow: /business/onboarding/\nDisallow: /business/pin_create/\nDisallow: /business/referral/*\nDisallow: /business/site/\nDisallow: /business/switch-account/\nDisallow: /business/top-ads/\nDisallow: /business/update/\nDisallow: /business/verified-merchant/status/\nDisallow: /christmas-spotlight/\nDisallow: /close-account/\nDisallow: /communities/*\nDisallow: /community/*\nDisallow: /community_comment/*\nDisallow: /community_post/*\nDisallow: /connect/*/\nDisallow: /content-claiming/*\nDisallow: /conversation/*\nDisallow: /conversations/*\nDisallow: /convert-business/\nDisallow: /convert-personal/\nDisallow: /copyright-reporting/*\nDisallow: /create_pin/\nDisallow: /creative-studio/\nDisallow: /creative_editor/\nDisallow: /creator-spotlight/*\nDisallow: /creator_hub/_earn\nDisallow: /creator_hub/_partnerships\nDisallow: /csrf_error/\nDisallow: /ct.html\nDisallow: /deactivate-account/\nDisallow: /delete_post/\nDisallow: /discover/*\nDisallow: /edit/*\nDisallow: /editorial/*/*/\nDisallow: /email/*\nDisallow: /email_verification_error/\nDisallow: /engagement/*\nDisallow: /explore/*\nDisallow: /explore/*/?p=*\nDisallow: /facebook/\nDisallow: /fb.html\nDisallow: /follow_board/\nDisallow: /follow_user/\nDisallow: /google_gdn.html\nDisallow: /google_search.html\nDisallow: /hanukkah-spotlight/\nDisallow: /holiday-spotlight/\nDisallow: /holidays-celebrations/\nDisallow: /homefeed/\nDisallow: /homefeed/*\nDisallow: /idea-ads-tool/*\nDisallow: /idea-pin-builder/*\nDisallow: /idea-pin-invite/\nDisallow: /install-shuffles/\nDisallow: /invite/\nDisallow: /invited/\nDisallow: /join/?next=*\nDisallow: /join/signup/?next=*\nDisallow: /kwanzaa-spotlight/\nDisallow: /lens-search/*\nDisallow: /linked/*\nDisallow: /live-session/*\nDisallow: /login/?next=*\nDisallow: /logout/\nDisallow: /m/\nDisallow: /new_invite/\nDisallow: /news_hub/*\nDisallow: /notifications/\nDisallow: /oauth/line/*\nDisallow: /oauth/login/*\nDisallow: /oauth_error/\nDisallow: /offsite/\nDisallow: /parental-consent/confirmation/\nDisallow: /parental-passcode/\nDisallow: /password/reset/\nDisallow: /password/reset/*\nDisallow: /pin-builder/*\nDisallow: /pin-communities/*\nDisallow: /pin-creation-tool/*\nDisallow: /pin-editor/\nDisallow: /pin/*/activities\nDisallow: /pin/*/activity/\nDisallow: /pin/*/analytics\nDisallow: /pin/*/brand-catalog/\nDisallow: /pin/*/brand-catalog/*\nDisallow: /pin/*/comments\nDisallow: /pin/*/email\nDisallow: /pin/*/idea-pin/related-products/*\nDisallow: /pin/*/likes\nDisallow: /pin/*/promote\nDisallow: /pin/*/reaction\nDisallow: /pin/*/related-products\nDisallow: /pin/*/related-products/*\nDisallow: /pin/*/repin\nDisallow: /pin/*/repins\nDisallow: /pin/*/share\nDisallow: /pin/*/shop-the-look/*\nDisallow: /pin/*/tag\nDisallow: /pin/*/takes\nDisallow: /pin/*/visual-search\nDisallow: /pin/*/visual-shop\nDisallow: /pin/create/*\nDisallow: /pin/find/\nDisallow: /pin/find?url=*\nDisallow: /pin_shop/*\nDisallow: /pinterest-predicts/\nDisallow: /pinterest-wellbeing/*\nDisallow: /pinterestwellbeing/*\nDisallow: /place/*\nDisallow: /prefs/\nDisallow: /product-catalogs/\nDisallow: /product-catalogs/*\nDisallow: /product-feed/\nDisallow: /pw/*\nDisallow: /quiz/*\nDisallow: /recently-viewed/\nDisallow: /refresh_stored_accounts\nDisallow: /refresh_token\nDisallow: /register/\nDisallow: /reports-and-violations/\nDisallow: /request-data/\nDisallow: /resource/\nDisallow: /safe-redirect/\nDisallow: /search/\nDisallow: /search?\nDisallow: /secure/*\nDisallow: /sendHeaderEarly/\nDisallow: /settings/*\nDisallow: /shop_pins/*\nDisallow: /signup/*\nDisallow: /socialmanager/*\nDisallow: /source/*\nDisallow: /sso/*\nDisallow: /story-pin-builder/*\nDisallow: /story-pin-invite/\nDisallow: /story_feed/\nDisallow: /story_pins/*\nDisallow: /suspension-appeal/*\nDisallow: /sw-shell.html\nDisallow: /taggingtool/*\nDisallow: /thanksgiving-spotlight/\nDisallow: /theme/*\nDisallow: /tv/*\nDisallow: /unauth-profile/\nDisallow: /unlink/\nDisallow: /upload-lens-image\nDisallow: /upload-lens-image-history\nDisallow: /upload-shuffle-draft\nDisallow: /v1/urls/count.json\nDisallow: /v3/pidgets/log/\nDisallow: /website/confirm/\nDisallow: /website/confirm/download*\nDisallow: /welcome/\nDisallow: /your-shop/\nDisallow: /your-shop/*\n\nUser-agent: Mediapartners-Google\nUser-agent: AmazonAdBot\nAllow: /homefeed/\nAllow: /homefeed/*\nAllow: /pin/\nAllow: /search/*\nDisallow: /\n\nUser-agent: Google-adstxt\nUser-agent: IAB-Tech-Lab\nAllow: /ads.txt\nAllow: /app-ads.txt\nDisallow: /\n\nUser-agent: adidxbot\nUser-agent: AdsBot-Google\nUser-agent: AdsBot-Google-Mobile\nUser-agent: AxonBot\nUser-agent: Google Web Preview\nUser-agent: Google-Ads-Overview\nUser-agent: Google-AdWords-Express\nUser-agent: Google-Adwords-Instant\nUser-agent: LinkedInBot\nUser-agent: meta-externalads\nUser-agent: SnapchatAds\nUser-agent: Taboolabot\nAllow: /\n\nUser-agent: YandexAdditional\nUser-agent: YandexAdditionalBot\nDisallow: /\n\nUser-agent: *\nDisallow: /\n\nSitemap: https://www.pinterest.com/v3_sitemaps/qem_klp_vlm_navboost_sample_high_www.pinterest.com.xml\nSitemap: https://www.pinterest.com/v3_sitemaps/qem_klp_vlm_vase_medium_www.pinterest.com.xml\nSitemap: https://www.pinterest.com/v3_sitemaps/shopping_sitemap_www.pinterest.com.xml\nSitemap: https://www.pinterest.com/v3_sitemaps/seasonal_events_sitemap_20261224_christmas_eve_from_2pm_www.pinterest.com.xml\nSitemap: https://www.pinterest.com/v3_sitemaps/seasonal_events_sitemap_20261231_international_new_year_s_eve_www.pinterest.com.xml\nSitemap: https://www.pinterest.com/v3_sitemaps/qem_klp_vlm_vase_low_www.pinterest.com.xml\nSitemap: https://www.pinterest.com/v3_sitemaps/seasonal_events_sitemap_20261224_christmas_eve_joint_holiday_www.pinterest.com.xml\nSitemap: https://www.pinterest.com/v3_sitemaps/ideas_hub_expansion_v2_sitemap_www.pinterest.com.xml\nSitemap: https://www.pinterest.com/v3_sitemaps/seasonal_events_sitemap_20261129_first_sunday_of_advent_www.pinterest.com.xml\nSitemap: https://www.pinterest.com/v3_sitemaps/today_article_sitemap_www.pinterest.com.xml\nSitemap: https://www.pinterest.com/v3_sitemaps/seasonal_events_sitemap_20261231_new_year_holiday_www.pinterest.com.xml\nSitemap: https://www.pinterest.com/v3_sitemaps/seasonal_events_sitemap_20261126_thanksgiving_www.pinterest.com.xml\nSitemap: https://www.pinterest.com/v3_sitemaps/seasonal_events_sitemap_20261206_cayman_thanksgiving_www.pinterest.com.xml\nSitemap: https://www.pinterest.com/v3_sitemaps/seasonal_events_sitemap_20261127_day_after_thanksgiving_www.pinterest.com.xml\nSitemap: https://www.pinterest.com/v3_sitemaps/qem_klp_vlm_navboost_sample_low_www.pinterest.com.xml\nSitemap: https://www.pinterest.com/v3_sitemaps/ideas_hub_vlm_terms_sitemap_auto_publish_www.pinterest.com.xml\nSitemap: https://www.pinterest.com/v3_sitemaps/ideas_hub_gsc_terms_sitemap_www.pinterest.com.xml\nSitemap: https://www.pinterest.com/v3_sitemaps/ideas_hub_v_expansion_sitemap_www.pinterest.com.xml\nSitemap: https://www.pinterest.com/v3_sitemaps/qem_klp_vlm_navboost_sample_medium_www.pinterest.com.xml\nSitemap: https://www.pinterest.com/v3_sitemaps/seasonal_events_sitemap_20261129_first_advent_sunday_www.pinterest.com.xml\nSitemap: https://www.pinterest.com/v3_sitemaps/seasonal_events_sitemap_20261226_national_day_of_thanksgiving_www.pinterest.com.xml\nSitemap: https://www.pinterest.com/v3_sitemaps/seasonal_events_sitemap_20261031_halloween_www.pinterest.com.xml\nSitemap: https://www.pinterest.com/v3_sitemaps/seasonal_events_sitemap_20261206_christmas_season_begins_www.pinterest.com.xml\nSitemap: https://www.pinterest.com/v3_sitemaps/ideas_hub_v_expansion_sitemap_batch_2_www.pinterest.com.xml\nSitemap: https://www.pinterest.com/v3_sitemaps/seasonal_events_sitemap_20261231_western_new_year_s_eve_www.pinterest.com.xml\nSitemap: https://www.pinterest.com/v3_sitemaps/ideas_hub_answers_terms_sitemap_www.pinterest.com.xml\nSitemap: https://www.pinterest.com/v3_sitemaps/seasonal_events_sitemap_20261225_christmas_www.pinterest.com.xml\nSitemap: https://www.pinterest.com/v3_sitemaps/ideas_hub_world_cup_terms_sitemap_www.pinterest.com.xml\nSitemap: https://www.pinterest.com/v3_sitemaps/seasonal_events_sitemap_20261224_catholic_protestant_christmas_eve_www.pinterest.com.xml\nSitemap: https://www.pinterest.com/v3_sitemaps/qem_klp_vlm_vase_high_www.pinterest.com.xml\nSitemap: https://www.pinterest.com/v3_sitemaps/seasonal_events_sitemap_20261231_new_year_s_eve_from_2pm_www.pinterest.com.xml\nSitemap: https://www.pinterest.com/v3_sitemaps/image_pin_high_quality_tier_0_www.pinterest.com.xml\nSitemap: https://www.pinterest.com/v3_sitemaps/board_link_sitemap_priority_segment_2026q3_www.pinterest.com.xml\nSitemap: https://www.pinterest.com/v3_sitemaps/seasonal_halloween_www.pinterest.com.xml\nSitemap: https://www.pinterest.com/v3_sitemaps/priority_segment_2026q4_www.pinterest.com.xml\nSitemap: https://www.pinterest.com/v3_sitemaps/video_pin_sitemap_www.pinterest.com.xml\nSitemap: https://www.pinterest.com/v3_sitemaps/board_link_sitemap_recent_engagement_tier_0_www.pinterest.com.xml\nSitemap: https://www.pinterest.com/v3_sitemaps/board_link_sitemap_high_cohesion_tier_0_www.pinterest.com.xml\nSitemap: https://www.pinterest.com/v3_sitemaps/image_pin_sitemap_recent_engagement_tier_0_www.pinterest.com.xml\nSitemap: https://www.pinterest.com/v3_sitemaps/board_link_sitemap_www.pinterest.com.xml\nSitemap: https://www.pinterest.com/v3_sitemaps/board_link_sitemap_tier_3_quality_uncrawled_www.pinterest.com.xml\nSitemap: https://www.pinterest.com/v3_sitemaps/board_link_sitemap_priority_segment_2026q4_www.pinterest.com.xml\nSitemap: https://www.pinterest.com/v3_sitemaps/page_quality_model_high_score_www.pinterest.com.xml\nSitemap: https://www.pinterest.com/v3_sitemaps/board_link_sitemap_high_cohesion_tier_1_www.pinterest.com.xml\nSitemap: https://www.pinterest.com/v3_sitemaps/board_link_sitemap_seasonal_halloween_www.pinterest.com.xml\nSitemap: https://www.pinterest.com/v3_sitemaps/image_pin_sitemap_recently_created_www.pinterest.com.xml\nSitemap: https://www.pinterest.com/v3_sitemaps/image_pin_sitemap_high_engagement_tier_1_www.pinterest.com.xml\nSitemap: https://www.pinterest.com/v3_sitemaps/product_pin_sitemap_www.pinterest.com.xml\nSitemap: https://www.pinterest.com/v3_sitemaps/image_pin_high_quality_tier_1_www.pinterest.com.xml\nSitemap: https://www.pinterest.com/v3_sitemaps/priority_segment_2026q3_www.pinterest.com.xml\nSitemap: https://www.pinterest.com/v3_sitemaps/board_link_sitemap_tier_2_quality_uncrawled_www.pinterest.com.xml\nSitemap: https://www.pinterest.com/v3_sitemaps/board_link_sitemap_tier_0_quality_uncrawled_www.pinterest.com.xml\nSitemap: https://www.pinterest.com/v3_sitemaps/board_link_sitemap_tier_1_quality_uncrawled_www.pinterest.com.xml\nSitemap: https://www.pinterest.com/v3_sitemaps/video_product_pin_sitemap_www.pinterest.com.xml\nSitemap: https://www.pinterest.com/v3_sitemaps/board_link_sitemap_recent_engagement_tier_1_www.pinterest.com.xml\nSitemap: https://www.pinterest.com/v3_sitemaps/user_link_sitemap_www.pinterest.com.xml\nSitemap: https://www.pinterest.com/v3_sitemaps/image_pin_sitemap_high_engagement_tier_0_www.pinterest.com.xml\nSitemap: https://www.pinterest.com/v3_sitemaps/gift_board_link_sitemap_www.pinterest.com.xml\nSitemap: https://www.pinterest.com/v3_sitemaps/image_pin_sitemap_recent_engagement_tier_1_www.pinterest.com.xml\nSitemap: https://www.pinterest.com/v3_sitemaps/image_pin_sitemap_www.pinterest.com.xml\nSitemap: https://www.pinterest.com/v3_sitemaps/seasonal_halloween_segment_www.pinterest.com.xml\nSitemap: https://www.pinterest.com/v3_sitemaps/christmas_pin_reinforce_www.pinterest.com.xml\nSitemap: https://www.pinterest.com/v3_sitemaps/new_years_eve_pin_newset_www.pinterest.com.xml\nSitemap: https://www.pinterest.com/v3_sitemaps/christmas_pin_newset_www.pinterest.com.xml\nSitemap: https://www.pinterest.com/v3_sitemaps/new_years_eve_pin_reinforce_www.pinterest.com.xml\nSitemap: https://www.pinterest.com/v3_sitemaps/seasonal_events_sitemap_www.pinterest.com.xml\nSitemap: https://www.pinterest.com/v3_sitemaps/answers_page_sitemap_www.pinterest.com.xml","agents":{"GPTBot":false,"OAI-SearchBot":false,"ChatGPT-User":false,"ClaudeBot":false,"Claude-SearchBot":false,"PerplexityBot":false,"Google-Extended":false,"Applebot-Extended":true,"CCBot":false,"Bytespider":false,"Meta-ExternalAgent":false,"Amazonbot":false}},"sitemap":{"status":404,"content_type":"text/html","bytes":2987,"sha256":"b0087a6e3f0a45f7c200274af52350a680dd6f0fa42b29d605b808f91e351812","final_host":"www.pinterest.com","final_path":"/v3_sitemaps/qem_klp_vlm_navboost_sample_high_www.pinterest.com.xml","ms":39,"url":"https://www.pinterest.com/v3_sitemaps/qem_klp_vlm_navboost_sample_high_www.pinterest.com.xml","declared":true,"xml":false},"files":{"llms_txt":{"status":200,"content_type":"text/html","bytes":1290991,"sha256":"3463528399d48c43594fc818f42d4039e62211482dd80b2f0734e157b5cfb7b9","final_host":"www.pinterest.com","final_path":"/llms.txt","ms":234,"path":"/llms.txt","html":true,"json":null,"present":false},"agents_md":{"status":200,"content_type":"text/html","bytes":1340181,"sha256":"fb5fc6302b66f6d8d4f628dcfa4cb79f720d3590c34b0ef21a9ce6e54bc498bf","final_host":"www.pinterest.com","final_path":"/agents.md","ms":204,"path":"/agents.md","html":true,"json":null,"present":false},"agent_skills":{"status":200,"content_type":"text/html","bytes":1296301,"sha256":"8c15f649895994975a26387d3b8fbd3157273e92ae679c07d78386317ea5e189","final_host":"www.pinterest.com","final_path":"/.well-known/agent-skills/index.json","ms":242,"path":"/.well-known/agent-skills/index.json","html":true,"json":null,"present":false},"mcp_card":{"status":200,"content_type":"text/html","bytes":1296440,"sha256":"6b44d5a950e25cf60cd1943bf0382c4bcd2a4c42c8198ac8fc3aada9e846f548","final_host":"www.pinterest.com","final_path":"/.well-known/mcp/server-card.json","ms":191,"path":"/.well-known/mcp/server-card.json","html":true,"json":null,"present":false},"api_catalog":{"status":200,"content_type":"text/html","bytes":1522332,"sha256":"9bbb39b106b482f0e532b82d9e2918e3bd112cdfe47b05dfb6d63a46a27c049a","final_host":"www.pinterest.com","final_path":"/.well-known/api-catalog","ms":234,"path":"/.well-known/api-catalog","html":true,"json":null,"present":false},"a2a_card":{"status":200,"content_type":"text/html","bytes":1294477,"sha256":"451c208af4e070b3c7fb3dcc6f1e537a9dc72cb5211a9ccb31e7a70420b2ebed","final_host":"www.pinterest.com","final_path":"/.well-known/agent-card.json","ms":191,"path":"/.well-known/agent-card.json","html":true,"json":null,"present":false}},"rings":{"v":1,"candidates":[],"mcp":[],"a2a":null,"webmcp":{"model_context":false,"bridge":false,"mcp_json":false,"link_rel":false,"any":false},"handshakes":0,"get_path":{"status":200,"content_type":"text/html","ms":597,"error":null},"get_host":{"status":530,"content_type":"text/plain","ms":12,"error":null}},"sf":[4,0,0,0,0,0,1,0,1],"derived":{"reachable":true,"home_class":"answers","robots_served":true,"robots_sitemap":true,"content_signal":false,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":["GPTBot","OAI-SearchBot","ChatGPT-User","ClaudeBot","Claude-SearchBot","PerplexityBot","Google-Extended","CCBot","Bytespider","Meta-ExternalAgent","Amazonbot"],"blocks_any_ai":true,"files":{"llms_txt":false,"agents_md":false,"agent_skills":false,"mcp_card":false,"api_catalog":false,"a2a_card":false},"any_capability":false,"ai_refused":["gptbot","oaisearch","perplexity"],"ai_thin":[],"aipref":0,"aipref_measured":true,"sf":[4,0,0,0,0,0,1,0,1],"rings":{"mcp_declared":false,"mcp_live":false,"mcp_auth":false,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":3,"fh":"www.pinterest.com","fp_other_host":[],"fp":"415139f3e79f27eb","fp_bits":{"claudebot":21}},"derive_v":5,"frame":"top10k","shard":4},{"v":1,"kind":"crawlcheck-census-record","run":18,"domain":"tiktok.com","rank":54,"measured_at":"2026-11-13T18:52:33.408Z","home":{"browser":{"status":200,"content_type":"text/html","bytes":173297,"sha256":"097627e23f42525b779010d7cf3829d5fbaa0f84dfcbd0cf9a57b9e96fd74f57","final_host":"www.tiktok.com","final_path":"/","ms":336,"identity":"Mobile browser","html":true},"gptbot":{"status":200,"content_type":"text/html","bytes":414236,"sha256":"53ee87c8a582143c4ae5afbadba9c74d9a55512fe989702e8e457e1561f248b2","final_host":"www.tiktok.com","final_path":"/","ms":495,"identity":"GPTBot","html":true},"claudebot":{"status":200,"content_type":"text/html","bytes":415727,"sha256":"9767beb9f266bd11c111bfbe75d98d5f1579165b6b62b9c25c4e695cbea753ac","final_host":"www.tiktok.com","final_path":"/","ms":1673,"identity":"ClaudeBot","html":true},"oaisearch":{"status":200,"content_type":"text/html","bytes":418281,"sha256":"9018ea5944e7dd244226d88ba68e219139f9aeb29583fe40c2358a556b529ac3","final_host":"www.tiktok.com","final_path":"/","ms":1944,"identity":"OAI-SearchBot","html":true},"perplexity":{"status":200,"content_type":"text/html","bytes":346576,"sha256":"86db0813b73b35d1f04f35acd7e5e2311f210bdb1eb186404dbcc1be6bf3cfd2","final_host":"www.tiktok.com","final_path":"/","ms":426,"identity":"PerplexityBot","html":true}},"robots":{"status":200,"content_type":"text/plain","bytes":1567,"sha256":"835705a2ecb862f49a813b3aa4061e0edfefbb8318316b371f8fa2dba4018802","final_host":"www.tiktok.com","final_path":"/robots.txt","ms":268,"html":false,"text":"User-agent: Baiduspider\nUser-agent: 360Spider\nUser-agent: Sogouspider\nUser-agent: Yisouspider\nUser-agent: PetalBot\nUser-agent: GPTBot\nUser-agent: OAI-SearchBot\nUser-agent: anthropic-ai\nUser-agent: ClaudeBot\nUser-agent: PerplexityBot\nUser-agent: Google-Extended\nUser-agent: Applebot-Extended\nUser-agent: meta-externalagent\nUser-agent: DuckAssistBot\nUser-agent: ChatGPT-User\nUser-agent: Gemini-Deep-Research\nUser-agent: GoogleAgent-Mariner\nUser-agent: Google-NotebookLM\nUser-agent: CCBot\nUser-agent: Claude-User\nUser-agent: Claude-SearchBot\nUser-agent: Perplexity-User\nUser-agent: AI2Bot\nUser-agent: MistralAI-User\nUser-agent: Bytespider\nDisallow: /\n\nUser-agent: *\nAllow: /foryou\nAllow: /discover\nAllow: /about\nAllow: /legal\nAllow: /safety\nAllow: /forgood\nAllow: /community-guidelines\nAllow: /tag\nAllow: /amp\nAllow: /transparency\nAllow: /share\nAllow: /music\nDisallow: /inapp\nDisallow: /auth\nDisallow: /embed/@\nDisallow: /embed/v2\nDisallow: /embed/curated\nDisallow: /link\nDisallow: */directory/\nDisallow: /search/video?\nDisallow: /search/user?q=\nDisallow: /shop/view/product/\nDisallow: /sgtm/g/collect\nDisallow: /api/share/settings\nDisallow: /api/recommend/embed_videos\nDisallow: /discover/trending/detail/\nDisallow: /search?\nDisallow: /search/live?\n\nUser-agent: Bingbot\nDisallow: /discover\nDisallow: /inapp\nDisallow: /auth\nDisallow: /embed/@\nDisallow: /embed/v2\nDisallow: /embed/curated\nDisallow: /link\nDisallow: */directory/\nDisallow: /shop/view/product/\nDisallow: /sgtm/g/collect\nDisallow: /api/share/settings\nDisallow: /api/recommend/embed_videos\nDisallow: /search?\n","agents":{"GPTBot":false,"OAI-SearchBot":false,"ChatGPT-User":false,"ClaudeBot":false,"Claude-SearchBot":false,"PerplexityBot":false,"Google-Extended":false,"Applebot-Extended":false,"CCBot":false,"Bytespider":false,"Meta-ExternalAgent":false,"Amazonbot":true}},"sitemap":{"status":200,"content_type":"application/xml","bytes":521367,"sha256":"4600fe62aabaab0b53b4c57ab1cddea537a052ad6b57445ea86d840252e5f603","final_host":"www.tiktok.com","final_path":"/tos/node/sitemaps/sitemap.xml","ms":3102,"url":"https://tiktok.com/sitemap.xml","declared":false,"xml":true},"files":{"llms_txt":{"status":200,"content_type":"text/html","bytes":33737,"sha256":"78336b13b569abc3d4e59c0d897793c12c48449c8521810279f943e932d9c2d5","final_host":"www.tiktok.com","final_path":"/404","ms":1091,"path":"/llms.txt","html":true,"json":null,"present":false},"agents_md":{"status":200,"content_type":"text/html","bytes":33737,"sha256":"78336b13b569abc3d4e59c0d897793c12c48449c8521810279f943e932d9c2d5","final_host":"www.tiktok.com","final_path":"/404","ms":490,"path":"/agents.md","html":true,"json":null,"present":false},"agent_skills":{"status":200,"content_type":"text/html","bytes":33737,"sha256":"78336b13b569abc3d4e59c0d897793c12c48449c8521810279f943e932d9c2d5","final_host":"www.tiktok.com","final_path":"/404","ms":473,"path":"/.well-known/agent-skills/index.json","html":true,"json":null,"present":false},"mcp_card":{"status":200,"content_type":"text/html","bytes":33737,"sha256":"78336b13b569abc3d4e59c0d897793c12c48449c8521810279f943e932d9c2d5","final_host":"www.tiktok.com","final_path":"/404","ms":525,"path":"/.well-known/mcp/server-card.json","html":true,"json":null,"present":false},"api_catalog":{"status":200,"content_type":"text/html","bytes":33737,"sha256":"78336b13b569abc3d4e59c0d897793c12c48449c8521810279f943e932d9c2d5","final_host":"www.tiktok.com","final_path":"/404","ms":513,"path":"/.well-known/api-catalog","html":true,"json":null,"present":false},"a2a_card":{"status":200,"content_type":"text/html","bytes":33737,"sha256":"78336b13b569abc3d4e59c0d897793c12c48449c8521810279f943e932d9c2d5","final_host":"www.tiktok.com","final_path":"/404","ms":690,"path":"/.well-known/agent-card.json","html":true,"json":null,"present":false}},"rings":{"v":1,"candidates":[],"mcp":[],"a2a":null,"webmcp":{"model_context":false,"bridge":false,"mcp_json":false,"link_rel":false,"any":false},"handshakes":0,"get_path":{"status":200,"content_type":"text/html","ms":547,"error":null},"get_host":{"status":530,"content_type":"text/plain","ms":7,"error":null}},"sf":[4,0,0,0,0,0,1,0,0],"derived":{"reachable":true,"home_class":"answers","robots_served":true,"robots_sitemap":false,"content_signal":false,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":["GPTBot","OAI-SearchBot","ChatGPT-User","ClaudeBot","Claude-SearchBot","PerplexityBot","Google-Extended","Applebot-Extended","CCBot","Bytespider","Meta-ExternalAgent"],"blocks_any_ai":true,"files":{"llms_txt":false,"agents_md":false,"agent_skills":false,"mcp_card":false,"api_catalog":false,"a2a_card":false},"any_capability":false,"ai_refused":[],"ai_thin":[],"aipref":0,"aipref_measured":true,"sf":[4,0,0,0,0,0,1,0,0],"rings":{"mcp_declared":false,"mcp_live":false,"mcp_auth":false,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":3,"fh":"www.tiktok.com","fp_other_host":[]},"derive_v":5,"frame":"top10k","shard":5},{"v":1,"kind":"crawlcheck-census-record","run":18,"domain":"whatsapp.com","rank":55,"measured_at":"2026-11-13T18:53:15.625Z","home":{"browser":{"status":400,"content_type":"text/html","bytes":6278,"sha256":"78ee704dc1eefb25a1ed53e6d56f6ede5d22f71988a038aea50c2619180e4ba0","final_host":"www.whatsapp.com","final_path":"/","ms":389,"identity":"Mobile browser","html":true},"gptbot":{"status":200,"content_type":"text/html","bytes":277943,"sha256":"8d76d33cae82de467190edb4d135915656635e265dcfcb194f84a5e19f33dc41","final_host":"www.whatsapp.com","final_path":"/","ms":585,"identity":"GPTBot","html":true,"fp":"035ecaf7bb074b16"},"claudebot":{"status":200,"content_type":"text/html","bytes":277899,"sha256":"afa028eb1ebb26c6e31288e3ab228a5b7b6f2a75994e50ef01961d6b0f290b9c","final_host":"www.whatsapp.com","final_path":"/","ms":622,"identity":"ClaudeBot","html":true,"fp":"035ecaf7bb074b16"},"oaisearch":{"status":200,"content_type":"text/html","bytes":277951,"sha256":"108dd70584bf8766ac8461936dcb2a267c7a2ec12a1961359e4d9660920bbb57","final_host":"www.whatsapp.com","final_path":"/","ms":618,"identity":"OAI-SearchBot","html":true,"fp":"035ecaf7bb074b16"},"perplexity":{"status":200,"content_type":"text/html","bytes":288079,"sha256":"d4294426c9f7913d024583189ac50eddf0e1370fd536389f6dcbe52b529ce929","final_host":"www.whatsapp.com","final_path":"/","ms":559,"identity":"PerplexityBot","html":true,"fp":"015ecbb7a3070b56"}},"robots":{"status":200,"content_type":"text/plain","bytes":1225,"sha256":"6d6febc5de572ecaca22e0169030eb4e4cd41ecaf12af30d64f2cdd9c000d0cb","final_host":"www.whatsapp.com","final_path":"/robots.txt","ms":106,"html":false,"text":"# Notice: Collection of data on Facebook through automated means is\n# prohibited unless you have express written permission from Facebook\n# and may only be conducted for the limited purpose contained in said\n# permission.\n# See: http://www.facebook.com/apps/site_scraping_tos_terms.php\n\nUser-agent: __usage_ai-training\nDisallow: /\n\nUser-agent: Amazonbot\nDisallow: /\n\nUser-agent: Applebot-Extended\nDisallow: /\n\nUser-agent: Brightbot\nDisallow: /\n\nUser-agent: ClaudeBot\nDisallow: /\n\nUser-agent: Google-Extended\nDisallow: /\n\nUser-agent: GPTBot\nDisallow: /\n\nUser-agent: PerplexityBot\nDisallow: /\n\nUser-agent: PetalBot\nDisallow: /\n\nUser-agent: Scrapy\nDisallow: /\n\nUser-agent: uptimerobot\nDisallow: /\n\nUser-agent: viberbot\nDisallow: /\n\nUser-agent: YaK\nDisallow: /\n\nUser-agent: Yandex\nDisallow: /\n\nUser-agent: Yeti\nDisallow: /\n\nUser-agent: facebookexternalhit\nAllow: *\n\nUser-agent: meta-externalads\nAllow: *\n\nUser-agent: *\nDisallow: /*cursor=\nDisallow: /*fb_comment_id=\nDisallow: /ajax/\nDisallow: /tealium/\nDisallow: /intern/\nDisallow: /internal/\nDisallow: /login/\nDisallow: /oidc/callback/\nDisallow: /*.php\nDisallow: /android/third_party\nDisallow: /settings\n\nSitemap: https://www.whatsapp.com/sitemap/www_whatsapp_com_sitemap.xml.gz","agents":{"GPTBot":false,"OAI-SearchBot":true,"ChatGPT-User":true,"ClaudeBot":false,"Claude-SearchBot":true,"PerplexityBot":false,"Google-Extended":false,"Applebot-Extended":false,"CCBot":true,"Bytespider":true,"Meta-ExternalAgent":true,"Amazonbot":false}},"sitemap":null,"files":null,"derived":{"reachable":false,"home_class":"refused_us","robots_served":false,"robots_sitemap":false,"content_signal":false,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":[],"blocks_any_ai":false,"files":{"llms_txt":false,"agents_md":false,"agent_skills":false,"mcp_card":false,"api_catalog":false,"a2a_card":false},"any_capability":false,"ai_refused":[],"ai_thin":[],"rings":{"mcp_declared":false,"mcp_live":false,"mcp_auth":false,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":3,"fh":"www.whatsapp.com","fp_other_host":[]},"derive_v":5,"frame":"top10k","shard":6},{"v":1,"kind":"crawlcheck-census-record","run":18,"domain":"googledomains.com","rank":56,"measured_at":"2026-11-13T18:55:41.951Z","home":{"browser":{"status":200,"content_type":"text/html","bytes":5354,"sha256":"227cde598113ce7b22e80b2a3800fbd421e5819ecb7d58a2994403b3bd4c4708","final_host":"domains.google","final_path":"/","ms":490,"identity":"Mobile browser","html":true,"fp":"e51811cd1864f4d8"},"gptbot":{"status":200,"content_type":"text/html","bytes":5354,"sha256":"ecf7ec62aa605531c48cbce9aa7f9fab5614f68bb6d3e724d112428774c932d8","final_host":"domains.google","final_path":"/","ms":398,"identity":"GPTBot","html":true,"fp":"e51811cd1864f4d8"},"claudebot":{"status":200,"content_type":"text/html","bytes":5354,"sha256":"05de509d91548864c5e1835cca8bebcbb2dbf5ae6348239d116ac15e66d9f477","final_host":"domains.google","final_path":"/","ms":791,"identity":"ClaudeBot","html":true,"fp":"e51811cd1864f4d8"},"oaisearch":{"status":200,"content_type":"text/html","bytes":5354,"sha256":"4c13aea55a038cb3372ee229795027b7097e98def40430d94099b54e5b862944","final_host":"domains.google","final_path":"/","ms":315,"identity":"OAI-SearchBot","html":true,"fp":"e51811cd1864f4d8"},"perplexity":{"status":200,"content_type":"text/html","bytes":5354,"sha256":"5fb1316c864409052587c776dc052e50f4bc9e5d3c122089b1480fd06203235d","final_host":"domains.google","final_path":"/","ms":314,"identity":"PerplexityBot","html":true,"fp":"e51811cd1864f4d8"}},"robots":{"status":200,"content_type":"text/html","bytes":5354,"sha256":"525f17fb9163b1ee5991b3606d1fc18ef0486cb755810928e5faf148e38974a5","final_host":"domains.google","final_path":"/","ms":269,"html":true,"text":null,"agents":null},"sitemap":{"status":200,"content_type":"text/html","bytes":5354,"sha256":"bda542a82438829c88c5b2227fada5b62808126323b2a9113d2734bfa09cb27f","final_host":"domains.google","final_path":"/","ms":15,"url":"https://googledomains.com/sitemap.xml","declared":false,"xml":false},"files":{"llms_txt":{"status":200,"content_type":"text/html","bytes":5354,"sha256":"3d3e45e1d519b39c9b00d9e952c3d8471027ded969960f3469216bb2d126e16f","final_host":"domains.google","final_path":"/","ms":14,"path":"/llms.txt","html":true,"json":null,"present":false},"agents_md":{"status":200,"content_type":"text/html","bytes":5354,"sha256":"f613cbc6cfb53589f8ef523ba7644184bbb4a8e7edbaefce7c9a1d680778c270","final_host":"domains.google","final_path":"/","ms":12,"path":"/agents.md","html":true,"json":null,"present":false},"agent_skills":{"status":200,"content_type":"text/html","bytes":5354,"sha256":"73db7bff359678f85fe282f7f7f2b7caa36c50ff6bf27196b2dba3e695fc686b","final_host":"domains.google","final_path":"/","ms":13,"path":"/.well-known/agent-skills/index.json","html":true,"json":null,"present":false},"mcp_card":{"status":200,"content_type":"text/html","bytes":5354,"sha256":"3e38ea2bfd82fd631950fec8244172a213f586cae9399add0b904bc6e16a330b","final_host":"domains.google","final_path":"/","ms":13,"path":"/.well-known/mcp/server-card.json","html":true,"json":null,"present":false},"api_catalog":{"status":200,"content_type":"text/html","bytes":5354,"sha256":"c940826ae370d98e2266f2d2caa9576cd3848e22f0218b6c4ecc724551a98627","final_host":"domains.google","final_path":"/","ms":12,"path":"/.well-known/api-catalog","html":true,"json":null,"present":false},"a2a_card":{"status":200,"content_type":"text/html","bytes":5354,"sha256":"76a0895368e75675f9c5ecc5e2dbff4570694c71afcace6a98271d31d5e0914d","final_host":"domains.google","final_path":"/","ms":12,"path":"/.well-known/agent-card.json","html":true,"json":null,"present":false}},"rings":{"v":1,"candidates":[],"mcp":[],"a2a":null,"webmcp":{"model_context":false,"bridge":false,"mcp_json":false,"link_rel":false,"any":false},"handshakes":0,"get_path":{"status":200,"content_type":"text/html","ms":14,"error":null},"get_host":{"status":530,"content_type":"text/plain","ms":9,"error":null}},"sf":[1,0,0,0,0,0,1,0,0],"derived":{"reachable":true,"home_class":"answers","robots_served":false,"robots_sitemap":false,"content_signal":false,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":[],"blocks_any_ai":false,"files":{"llms_txt":false,"agents_md":false,"agent_skills":false,"mcp_card":false,"api_catalog":false,"a2a_card":false},"any_capability":false,"ai_refused":[],"ai_thin":[],"sf":[1,0,0,0,0,0,1,0,0],"rings":{"mcp_declared":false,"mcp_live":false,"mcp_auth":false,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":3,"fh":"domains.google","fp_other_host":[],"fp":"e51811cd1864f4d8","fp_bits":{"gptbot":0,"claudebot":0,"oaisearch":0,"perplexity":0}},"derive_v":5,"frame":"top10k","shard":7},{"v":1,"kind":"crawlcheck-census-record","run":18,"domain":"cloudflare.net","rank":57,"measured_at":"2026-11-13T18:46:27.088Z","home":{"browser":{"status":200,"content_type":"text/html","bytes":64716,"sha256":"8b47ca930bde4842faed8c15d14a8ce012435414f8935234446d115c87a13b68","final_host":"cloudflare.net","final_path":"/home/default.aspx","ms":670,"identity":"Mobile browser","html":true,"fp":"c118113eb6f046ff"},"gptbot":{"status":403,"content_type":"text/html","bytes":5564,"sha256":"1e73db8a320b8c8ecd3b37c1e032202cb799d2a9d0cfa9095d0299884f740b31","final_host":"cloudflare.net","final_path":"/","ms":4,"identity":"GPTBot","html":true},"claudebot":{"status":200,"content_type":"text/html","bytes":64716,"sha256":"b3a547db9877d5e940070ea6110dc13f57ade1f3743881cf6d109f86623f0ff8","final_host":"cloudflare.net","final_path":"/home/default.aspx","ms":572,"identity":"ClaudeBot","html":true,"fp":"c118113eb6f046ff"},"oaisearch":{"status":200,"content_type":"text/html","bytes":64716,"sha256":"10d62528644730f8417d44095b100c17dfbc26a167b08d7f9258b4690e2d7312","final_host":"cloudflare.net","final_path":"/home/default.aspx","ms":533,"identity":"OAI-SearchBot","html":true,"fp":"c118113eb6f046ff"},"perplexity":{"status":200,"content_type":"text/html","bytes":64716,"sha256":"fe4e7bce164248bdfeb4bf098e6ed5b5b5dbcd480dccd880ac61b58545a6087b","final_host":"cloudflare.net","final_path":"/home/default.aspx","ms":533,"identity":"PerplexityBot","html":true,"fp":"c118113eb6f046ff"}},"robots":{"status":200,"content_type":"text/plain","bytes":207,"sha256":"de73d32142f114b8d18bbfd7fa5779ccee5f7a1753873402cfdab59444834ccd","final_host":"cloudflare.net","final_path":"/robots.txt","ms":218,"html":false,"text":"User-agent: *\r\nCrawl-delay: 10\r\nAllow: /\r\n\r\nUser-agent: Eluta.ca\r\nCrawl-delay: 60\r\nAllow: /\r\n\r\nUser-agent: MS Search 6.0 Robot\r\nCrawl-delay: 60\r\nAllow: /\r\n\r\nUser-agent: gsa-crawler\r\nCrawl-delay: 60\r\nAllow: /","agents":{"GPTBot":true,"OAI-SearchBot":true,"ChatGPT-User":true,"ClaudeBot":true,"Claude-SearchBot":true,"PerplexityBot":true,"Google-Extended":true,"Applebot-Extended":true,"CCBot":true,"Bytespider":true,"Meta-ExternalAgent":true,"Amazonbot":true}},"sitemap":{"status":404,"content_type":"text/html","bytes":1286,"sha256":"049e1b638af15c4f3525aed1b4be5d3fd6fe361d999601cbc978dfb34a8c3285","final_host":"cloudflare.net","final_path":"/sitemap.xml","ms":245,"url":"https://cloudflare.net/sitemap.xml","declared":false,"xml":false},"files":{"llms_txt":{"status":404,"content_type":"text/html","bytes":1286,"sha256":"049e1b638af15c4f3525aed1b4be5d3fd6fe361d999601cbc978dfb34a8c3285","final_host":"cloudflare.net","final_path":"/llms.txt","ms":225,"path":"/llms.txt","html":true,"json":null,"present":false},"agents_md":{"status":404,"content_type":"text/html","bytes":1286,"sha256":"049e1b638af15c4f3525aed1b4be5d3fd6fe361d999601cbc978dfb34a8c3285","final_host":"cloudflare.net","final_path":"/agents.md","ms":300,"path":"/agents.md","html":true,"json":null,"present":false},"agent_skills":{"status":200,"content_type":"text/html","bytes":11,"sha256":"ee172faa9c17e8e5f4b63ef56d6d6863a48c223de152be43781305c99e143200","final_host":"cloudflare.net","final_path":"/.well-known/agent-skills/index.json","ms":250,"path":"/.well-known/agent-skills/index.json","html":true,"json":null,"present":false},"mcp_card":{"status":200,"content_type":"text/html","bytes":11,"sha256":"ee172faa9c17e8e5f4b63ef56d6d6863a48c223de152be43781305c99e143200","final_host":"cloudflare.net","final_path":"/.well-known/mcp/server-card.json","ms":253,"path":"/.well-known/mcp/server-card.json","html":true,"json":null,"present":false},"api_catalog":{"status":200,"content_type":"text/html","bytes":11,"sha256":"ee172faa9c17e8e5f4b63ef56d6d6863a48c223de152be43781305c99e143200","final_host":"cloudflare.net","final_path":"/.well-known/api-catalog","ms":495,"path":"/.well-known/api-catalog","html":true,"json":null,"present":false},"a2a_card":{"status":200,"content_type":"text/html","bytes":11,"sha256":"ee172faa9c17e8e5f4b63ef56d6d6863a48c223de152be43781305c99e143200","final_host":"cloudflare.net","final_path":"/.well-known/agent-card.json","ms":268,"path":"/.well-known/agent-card.json","html":true,"json":null,"present":false}},"rings":{"v":1,"candidates":[],"mcp":[],"a2a":null,"webmcp":{"model_context":false,"bridge":false,"mcp_json":false,"link_rel":false,"any":false},"handshakes":0,"get_path":{"status":404,"content_type":"text/html","ms":475,"error":null},"get_host":{"status":530,"content_type":"text/plain","ms":5,"error":null}},"sf":[4,0,0,0,0,0,2,0,0],"derived":{"reachable":true,"home_class":"answers","robots_served":true,"robots_sitemap":false,"content_signal":false,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":[],"blocks_any_ai":false,"files":{"llms_txt":false,"agents_md":false,"agent_skills":false,"mcp_card":false,"api_catalog":false,"a2a_card":false},"any_capability":false,"ai_refused":["gptbot"],"ai_thin":[],"aipref":0,"aipref_measured":true,"sf":[4,0,0,0,0,0,2,0,0],"rings":{"mcp_declared":false,"mcp_live":false,"mcp_auth":false,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":3,"fh":"cloudflare.net","fp_other_host":[],"fp":"c118113eb6f046ff","fp_bits":{"claudebot":0,"oaisearch":0,"perplexity":0}},"derive_v":5,"frame":"top10k","shard":0},{"v":1,"kind":"crawlcheck-census-record","run":18,"domain":"goo.gl","rank":58,"measured_at":"2026-11-13T18:48:18.168Z","home":{"browser":{"status":400,"content_type":"text/html","bytes":9233,"sha256":"467f85f08ed95e4d207f06ae2f39969cbf6c9c25a18a0929a0835ee367404d81","final_host":"goo.gl","final_path":"/","ms":1198,"identity":"Mobile browser","html":true},"gptbot":{"status":400,"content_type":"text/html","bytes":9233,"sha256":"467f85f08ed95e4d207f06ae2f39969cbf6c9c25a18a0929a0835ee367404d81","final_host":"goo.gl","final_path":"/","ms":224,"identity":"GPTBot","html":true},"claudebot":{"status":400,"content_type":"text/html","bytes":9233,"sha256":"467f85f08ed95e4d207f06ae2f39969cbf6c9c25a18a0929a0835ee367404d81","final_host":"goo.gl","final_path":"/","ms":1145,"identity":"ClaudeBot","html":true},"oaisearch":{"status":400,"content_type":"text/html","bytes":9233,"sha256":"467f85f08ed95e4d207f06ae2f39969cbf6c9c25a18a0929a0835ee367404d81","final_host":"goo.gl","final_path":"/","ms":1082,"identity":"OAI-SearchBot","html":true},"perplexity":{"status":400,"content_type":"text/html","bytes":9233,"sha256":"467f85f08ed95e4d207f06ae2f39969cbf6c9c25a18a0929a0835ee367404d81","final_host":"goo.gl","final_path":"/","ms":1072,"identity":"PerplexityBot","html":true}},"robots":{"status":400,"content_type":"text/html","bytes":9258,"sha256":"bae5d66fbe371f66a934935346906d25bb35df6fb4cd5504a359cd324e89ea11","final_host":"goo.gl","final_path":"/robots.txt","ms":1342,"html":true,"text":null,"agents":null},"sitemap":null,"files":null,"derived":{"reachable":false,"home_class":"refused_us","robots_served":false,"robots_sitemap":false,"content_signal":false,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":[],"blocks_any_ai":false,"files":{"llms_txt":false,"agents_md":false,"agent_skills":false,"mcp_card":false,"api_catalog":false,"a2a_card":false},"any_capability":false,"ai_refused":[],"ai_thin":[],"rings":{"mcp_declared":false,"mcp_live":false,"mcp_auth":false,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":3,"fh":"goo.gl","fp_other_host":[]},"derive_v":5,"frame":"top10k","shard":1},{"v":1,"kind":"crawlcheck-census-record","run":18,"domain":"yahoo.com","rank":59,"measured_at":"2026-11-13T18:49:51.543Z","home":{"browser":{"status":200,"content_type":"text/html","bytes":485774,"sha256":"687f61762d46a7b1350d7cce5e6920f32a799c735bbf21ac02f619f28b8055d9","final_host":"www.yahoo.com","final_path":"/","ms":771,"identity":"Mobile browser","html":true,"fp":"62f7ff40512bc92a"},"gptbot":{"status":429,"content_type":"text/html","bytes":23,"sha256":"0d24c98db98d3b4a87b9626a026e7d72c99754ab6bd88211a17bc372350b7d89","final_host":"www.yahoo.com","final_path":"/","ms":356,"identity":"GPTBot","html":true},"claudebot":{"status":404,"content_type":"text/html","bytes":1616,"sha256":"17ea8d8c4f070d0b6fd4c08935fcaae7553667f43acfdd8cb587a090d867e581","final_host":"yahoo.com","final_path":"/","ms":549,"identity":"ClaudeBot","html":true},"oaisearch":{"status":429,"content_type":"text/html","bytes":23,"sha256":"0d24c98db98d3b4a87b9626a026e7d72c99754ab6bd88211a17bc372350b7d89","final_host":"www.yahoo.com","final_path":"/","ms":353,"identity":"OAI-SearchBot","html":true},"perplexity":{"status":404,"content_type":"text/html","bytes":1616,"sha256":"17ea8d8c4f070d0b6fd4c08935fcaae7553667f43acfdd8cb587a090d867e581","final_host":"yahoo.com","final_path":"/","ms":487,"identity":"PerplexityBot","html":true}},"robots":{"status":200,"content_type":"text/plain","bytes":1464,"sha256":"dfcd18f101783bde40c3bdca97bb90dbea1653321fd8b57f81cf5a6a6dc00b4b","final_host":"www.yahoo.com","final_path":"/robots.txt","ms":454,"html":false,"text":"User-agent: *\nDisallow: /info/p.gif\nDisallow: /p/\nDisallow: /r/\nDisallow: /bin/\nDisallow: /caas/\nDisallow: /blank.html\nDisallow: /includes/\nDisallow: /_td_api\nDisallow: /tdv2_fp\nDisallow: /nel_ms\nDisallow: /fp_ms\nDisallow: /sports_fp_ms\nDisallow: /search_ms\nDisallow: /_tdpp_api\nDisallow: /_remote\nDisallow: /_multiremote\nDisallow: /_tdhl_api\nDisallow: /digest\nDisallow: /fpjs\nDisallow: /myjs\nDisallow: /news/m/\n\nUser-agent: ADmantX\nUser-agent: AlphaBot\nUser-agent: anthropic-ai\nUser-agent: AwarioRssBot\nUser-agent: AwarioSmartBot\nUser-agent: BLEXBot\nUser-agent: Buzzbot\nUser-agent: Bytespider\nUser-agent: CCBot\nUser-agent: ChatGPT-User\nUser-agent: claritybot\nUser-agent: Claude-Web\nUser-agent: ClaudeBot\nUser-agent: cohere-ai\nUser-agent: Diffbot\nUser-agent: FacebookBot\nUser-agent: FriendlyCrawler\nUser-agent: Google-Extended\nUser-agent: GPTBot\nUser-agent: huggingface\nUser-agent: ImagesiftBot\nUser-agent: img2dataset\nUser-agent: magpie-crawler\nUser-agent: Meltwater\nUser-agent: Neevabot\nUser-agent: news-please\nUser-agent: NewsNow\nUser-agent: Nutch\nUser-agent: omgili\nUser-agent: omgilibot\nUser-agent: panscient.com\nUser-agent: Perplexity-ai\nUser-agent: PerplexityBot\nUser-agent: PetalBot\nUser-agent: PiplBot\nUser-agent: scoop.it\nUser-agent: Scrapy\nUser-agent: Seekr\nUser-agent: SentiBot\nUser-agent: SeznamBot\nUser-agent: TurnitinBot\nUser-agent: YouBot\nUser-agent: ZumBot\nDisallow: /\n\nUser-agent: Claude-SearchBot\nUser-agent: OAI-SearchBot\nDisallow: */articles/\n","agents":{"GPTBot":false,"OAI-SearchBot":true,"ChatGPT-User":false,"ClaudeBot":false,"Claude-SearchBot":true,"PerplexityBot":false,"Google-Extended":false,"Applebot-Extended":true,"CCBot":false,"Bytespider":false,"Meta-ExternalAgent":true,"Amazonbot":true}},"sitemap":{"status":404,"content_type":"text/html","bytes":4592,"sha256":"262b94353d89692f6b72cddbd5f8cf3edb5ff17fcc29f50de2a2a14e553d0f3a","final_host":"www.yahoo.com","final_path":"/sitemap.xml","ms":227,"url":"https://yahoo.com/sitemap.xml","declared":false,"xml":false},"files":{"llms_txt":{"status":404,"content_type":"text/html","bytes":4589,"sha256":"1cdebc6527851631c8a2ddb81a8e5d5a3eef41be860f6f1fad93362194c13642","final_host":"www.yahoo.com","final_path":"/llms.txt","ms":177,"path":"/llms.txt","html":true,"json":null,"present":false},"agents_md":{"status":404,"content_type":"text/html","bytes":4590,"sha256":"b3e253a199b4da1b67c6a916dbffcb9750eb09b79bf5933e3accb89f1f71895a","final_host":"www.yahoo.com","final_path":"/agents.md","ms":192,"path":"/agents.md","html":true,"json":null,"present":false},"agent_skills":{"status":404,"content_type":"text/html","bytes":4616,"sha256":"eed0c0627b3bc459edd23d377189b34349865c33b3d23aa209ee8a931729b592","final_host":"www.yahoo.com","final_path":"/.well-known/agent-skills/index.json","ms":148,"path":"/.well-known/agent-skills/index.json","html":true,"json":null,"present":false},"mcp_card":{"status":404,"content_type":"text/html","bytes":4613,"sha256":"f508b5ba9075c5348d11596a14e968c33e9daa98d61deecd33d7d51e7b98a984","final_host":"www.yahoo.com","final_path":"/.well-known/mcp/server-card.json","ms":517,"path":"/.well-known/mcp/server-card.json","html":true,"json":null,"present":false},"api_catalog":{"status":404,"content_type":"text/html","bytes":4604,"sha256":"edd5aedd72b0bf01d27c97f2178d52ef7e0b5b35df5c4174db650d543b8e328e","final_host":"www.yahoo.com","final_path":"/.well-known/api-catalog","ms":147,"path":"/.well-known/api-catalog","html":true,"json":null,"present":false},"a2a_card":{"status":404,"content_type":"text/html","bytes":4608,"sha256":"6ef06998f50b4edbff8fb2f08f765c7daaf60ce36917fea9e9ea96828542211f","final_host":"www.yahoo.com","final_path":"/.well-known/agent-card.json","ms":224,"path":"/.well-known/agent-card.json","html":true,"json":null,"present":false}},"rings":{"v":1,"candidates":[],"mcp":[],"a2a":null,"webmcp":{"model_context":false,"bridge":false,"mcp_json":false,"link_rel":false,"any":false},"handshakes":0,"get_path":{"status":404,"content_type":"text/html","ms":152,"error":null},"get_host":{"status":530,"content_type":"text/plain","ms":22,"error":null}},"sf":[4,0,0,0,0,0,0,0,0],"derived":{"reachable":true,"home_class":"answers","robots_served":true,"robots_sitemap":false,"content_signal":false,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":["GPTBot","ChatGPT-User","ClaudeBot","PerplexityBot","Google-Extended","CCBot","Bytespider"],"blocks_any_ai":true,"files":{"llms_txt":false,"agents_md":false,"agent_skills":false,"mcp_card":false,"api_catalog":false,"a2a_card":false},"any_capability":false,"ai_refused":["gptbot","claudebot","oaisearch","perplexity"],"ai_thin":[],"aipref":0,"aipref_measured":true,"sf":[4,0,0,0,0,0,0,0,0],"rings":{"mcp_declared":false,"mcp_live":false,"mcp_auth":false,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":3,"fh":"www.yahoo.com","fp_other_host":[],"fp":"62f7ff40512bc92a","fp_bits":{}},"derive_v":5,"frame":"top10k","shard":2},{"v":1,"kind":"crawlcheck-census-record","run":18,"domain":"googlesyndication.com","rank":60,"measured_at":"2026-11-13T18:50:46.493Z","home":{"browser":{"status":200,"content_type":"text/html","bytes":214941,"sha256":"03d599e0ec8f10ce265d2ac41dd58feb7ffc3d761266c532eef145cbe0d2d5d3","final_host":"www.google.com","final_path":"/","ms":158,"identity":"Mobile browser","html":true},"gptbot":{"status":200,"content_type":"text/html","bytes":31660,"sha256":"4515196ed5dca286b8500b46c597f7f392fbfd4936ff9f5c7c7fe0a09e346d19","final_host":"www.google.com","final_path":"/","ms":83,"identity":"GPTBot","html":true},"claudebot":{"status":200,"content_type":"text/html","bytes":90597,"sha256":"7124145096daf4e700f78f6489083e9e24dfc77c3e80f4903cfb370fd10b5155","final_host":"www.google.com","final_path":"/","ms":85,"identity":"ClaudeBot","html":true},"oaisearch":{"status":200,"content_type":"text/html","bytes":31690,"sha256":"5f076467b2440345fe488d98acb217328fd688641c56d3bda643222a5d075311","final_host":"www.google.com","final_path":"/","ms":85,"identity":"OAI-SearchBot","html":true},"perplexity":{"status":200,"content_type":"text/html","bytes":89544,"sha256":"f46b3cc4c6e221b61391b9854808530bbf98a9597393ec483842177856fee75f","final_host":"www.google.com","final_path":"/","ms":88,"identity":"PerplexityBot","html":true}},"robots":{"status":404,"content_type":"text/html","bytes":1571,"sha256":"6adee83a3eba87d30fdca3997d27097697f2767805f222a0f78c3f04392004d2","final_host":"googlesyndication.com","final_path":"/robots.txt","ms":137,"html":true,"text":null,"agents":null},"sitemap":{"status":404,"content_type":"text/html","bytes":1572,"sha256":"ab724c219c1a70a2f94a80452f31fa802b754912b60e32c40cd39666dd23910a","final_host":"googlesyndication.com","final_path":"/sitemap.xml","ms":113,"url":"https://googlesyndication.com/sitemap.xml","declared":false,"xml":false},"files":{"llms_txt":{"status":404,"content_type":"text/html","bytes":1569,"sha256":"47d71ed470dfe1ca13eb2af8fa019b95871a1bd1381a518ed71359e69b9c3cc7","final_host":"googlesyndication.com","final_path":"/llms.txt","ms":99,"path":"/llms.txt","html":true,"json":null,"present":false},"agents_md":{"status":404,"content_type":"text/html","bytes":1570,"sha256":"f08e316602488d86581ce85b3076a13c82fc3f97cdac3deba23801fb66ca908d","final_host":"googlesyndication.com","final_path":"/agents.md","ms":102,"path":"/agents.md","html":true,"json":null,"present":false},"agent_skills":{"status":404,"content_type":"text/html","bytes":1596,"sha256":"eee9f4f975e761645c6c385ec645439385500d0062bff7170ed09a8835d6f92b","final_host":"googlesyndication.com","final_path":"/.well-known/agent-skills/index.json","ms":120,"path":"/.well-known/agent-skills/index.json","html":true,"json":null,"present":false},"mcp_card":{"status":404,"content_type":"text/html","bytes":1593,"sha256":"e5db4a1d5f7596a7d62e54e192ab48bbf195116fa33a2c7e160fdaf6131945a8","final_host":"googlesyndication.com","final_path":"/.well-known/mcp/server-card.json","ms":114,"path":"/.well-known/mcp/server-card.json","html":true,"json":null,"present":false},"api_catalog":{"status":404,"content_type":"text/html","bytes":1584,"sha256":"7de25ab2e3ce288cf29d538f80ed48bf0d5759f87b613e29ed345db4675698aa","final_host":"googlesyndication.com","final_path":"/.well-known/api-catalog","ms":116,"path":"/.well-known/api-catalog","html":true,"json":null,"present":false},"a2a_card":{"status":404,"content_type":"text/html","bytes":1588,"sha256":"64d2189fbe1531e7fb6ab2e6040a932d87ae1feb811d4781df85ea7ef6b4d651","final_host":"googlesyndication.com","final_path":"/.well-known/agent-card.json","ms":113,"path":"/.well-known/agent-card.json","html":true,"json":null,"present":false}},"rings":{"v":1,"candidates":[],"mcp":[],"a2a":null,"webmcp":{"model_context":false,"bridge":false,"mcp_json":false,"link_rel":false,"any":false},"handshakes":0,"get_path":{"status":404,"content_type":"text/html","ms":109,"error":null},"get_host":{"status":530,"content_type":"text/plain","ms":21,"error":null}},"sf":[1,0,0,0,0,0,0,0,0],"derived":{"reachable":true,"home_class":"answers","robots_served":false,"robots_sitemap":false,"content_signal":false,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":[],"blocks_any_ai":false,"files":{"llms_txt":false,"agents_md":false,"agent_skills":false,"mcp_card":false,"api_catalog":false,"a2a_card":false},"any_capability":false,"ai_refused":[],"ai_thin":["gptbot","claudebot","oaisearch","perplexity"],"sf":[1,0,0,0,0,0,0,0,0],"rings":{"mcp_declared":false,"mcp_live":false,"mcp_auth":false,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":3,"fh":"www.google.com","fp_other_host":[]},"derive_v":5,"frame":"top10k","shard":3},{"v":1,"kind":"crawlcheck-census-record","run":18,"domain":"gvt1.com","rank":61,"measured_at":"2026-11-13T18:51:40.579Z","home":{"browser":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"gvt1.com","final_path":"/","ms":3,"identity":"Mobile browser","html":false},"gptbot":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"gvt1.com","final_path":"/","ms":2,"identity":"GPTBot","html":false},"claudebot":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"gvt1.com","final_path":"/","ms":1,"identity":"ClaudeBot","html":false},"oaisearch":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"gvt1.com","final_path":"/","ms":1,"identity":"OAI-SearchBot","html":false},"perplexity":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"gvt1.com","final_path":"/","ms":1,"identity":"PerplexityBot","html":false}},"robots":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"gvt1.com","final_path":"/robots.txt","ms":1,"html":false,"text":null,"agents":null},"sitemap":null,"files":null,"derived":{"reachable":false,"home_class":"no_site","robots_served":false,"robots_sitemap":false,"content_signal":false,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":[],"blocks_any_ai":false,"files":{"llms_txt":false,"agents_md":false,"agent_skills":false,"mcp_card":false,"api_catalog":false,"a2a_card":false},"any_capability":false,"ai_refused":[],"ai_thin":[],"rings":{"mcp_declared":false,"mcp_live":false,"mcp_auth":false,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":3,"fh":"gvt1.com","fp_other_host":[]},"derive_v":5,"frame":"top10k","shard":4},{"v":1,"kind":"crawlcheck-census-record","run":18,"domain":"roblox.com","rank":62,"measured_at":"2026-11-13T18:52:33.408Z","home":{"browser":{"status":200,"content_type":"text/html","bytes":110949,"sha256":"1ce4fc9f632310ab08ca0d849e9f2c9c1a09ec78c5065e718610639e5922a8f3","final_host":"www.roblox.com","final_path":"/","ms":682,"identity":"Mobile browser","html":true},"gptbot":{"status":200,"content_type":"text/html","bytes":63809,"sha256":"07716f0db71f33c58ee5877a282886ee83351e7aa55247fc978fd97222799b86","final_host":"www.roblox.com","final_path":"/","ms":642,"identity":"GPTBot","html":true},"claudebot":{"status":200,"content_type":"text/html","bytes":63809,"sha256":"28e95d7adc5ae20764ed74521102e119b91929b972c3691ddc3d2caf5aa48a60","final_host":"www.roblox.com","final_path":"/","ms":532,"identity":"ClaudeBot","html":true},"oaisearch":{"status":200,"content_type":"text/html","bytes":63809,"sha256":"78a8beee46d01bfb2c02dd27eeff37b7375f7a3465240eb76f6ad04635354aa6","final_host":"www.roblox.com","final_path":"/","ms":485,"identity":"OAI-SearchBot","html":true},"perplexity":{"status":200,"content_type":"text/html","bytes":63805,"sha256":"4777769efae55b0bcf7f825487a452d43c3bb118648f6c8acc518d211cdfb17e","final_host":"www.roblox.com","final_path":"/","ms":303,"identity":"PerplexityBot","html":true}},"robots":{"status":200,"content_type":"text/plain","bytes":2281,"sha256":"f5c52b75ce949831a002a1572f68cc23e8a22145c0685a41c95f87c6205f3dab","final_host":"www.roblox.com","final_path":"/robots.txt","ms":879,"html":false,"text":"user-agent: *\n\ndisallow: *.ashx\ndisallow: /abusereport/\ndisallow: /*/abusereport/\ndisallow: /admi/\ndisallow: /ads/\ndisallow: /catalog/contents\ndisallow: /catalog/html\ndisallow: /client-status\ndisallow: /data/\ndisallow: /error/\ndisallow: /forum/\ndisallow: /Forum/\ndisallow: /game/report-event\ndisallow: /game/report-stats\ndisallow: /ide/clienttoolbox\ndisallow: /javascript/\ndisallow: /reports/\ndisallow: /thumbnail/resolve-hash\ndisallow: /thumbnail_holder/g\ndisallow: /viewapp/\ndisallow: /error.html\ndisallow: /offline.html\ndisallow: /badges/\ndisallow: /*/badges/\ndisallow: /*.aspx$\ndisallow: /upgrades/checkout/success?giftPurchaseId=*\ndisallow: /report-abuse\ndisallow: /*/report-abuse\n\nSitemap: https://www.roblox.com/sitemap.xml\nSitemap: https://www.roblox.com/sitemap-games.xml\nSitemap: https://www.roblox.com/sitemap-ar.xml\nSitemap: https://www.roblox.com/sitemap-games-ar.xml\nSitemap: https://www.roblox.com/sitemap-de.xml\nSitemap: https://www.roblox.com/sitemap-games-de.xml\nSitemap: https://www.roblox.com/sitemap-es.xml\nSitemap: https://www.roblox.com/sitemap-games-es.xml\nSitemap: https://www.roblox.com/sitemap-fr.xml\nSitemap: https://www.roblox.com/sitemap-games-fr.xml\nSitemap: https://www.roblox.com/sitemap-hi.xml\nSitemap: https://www.roblox.com/sitemap-games-hi.xml\nSitemap: https://www.roblox.com/sitemap-id.xml\nSitemap: https://www.roblox.com/sitemap-games-id.xml\nSitemap: https://www.roblox.com/sitemap-it.xml\nSitemap: https://www.roblox.com/sitemap-games-it.xml\nSitemap: https://www.roblox.com/sitemap-ja.xml\nSitemap: https://www.roblox.com/sitemap-games-ja.xml\nSitemap: https://www.roblox.com/sitemap-ko.xml\nSitemap: https://www.roblox.com/sitemap-games-ko.xml\nSitemap: https://www.roblox.com/sitemap-nl.xml\nSitemap: https://www.roblox.com/sitemap-games-nl.xml\nSitemap: https://www.roblox.com/sitemap-pl.xml\nSitemap: https://www.roblox.com/sitemap-games-pl.xml\nSitemap: https://www.roblox.com/sitemap-pt.xml\nSitemap: https://www.roblox.com/sitemap-games-pt.xml\nSitemap: https://www.roblox.com/sitemap-th.xml\nSitemap: https://www.roblox.com/sitemap-games-th.xml\nSitemap: https://www.roblox.com/sitemap-tr.xml\nSitemap: https://www.roblox.com/sitemap-games-tr.xml\nSitemap: https://www.roblox.com/sitemap-vi.xml\nSitemap: https://www.roblox.com/sitemap-games-vi.xml\n","agents":{"GPTBot":true,"OAI-SearchBot":true,"ChatGPT-User":true,"ClaudeBot":true,"Claude-SearchBot":true,"PerplexityBot":true,"Google-Extended":true,"Applebot-Extended":true,"CCBot":true,"Bytespider":true,"Meta-ExternalAgent":true,"Amazonbot":true}},"sitemap":{"status":200,"content_type":"application/xml","bytes":77436,"sha256":"77db6a61f44a7c10fc36693646cc8e44b34b11998db51c11a8202bc17ee0a2b3","final_host":"www.roblox.com","final_path":"/sitemap.xml","ms":212,"url":"https://www.roblox.com/sitemap.xml","declared":true,"xml":true},"files":{"llms_txt":{"status":404,"content_type":"text/html","bytes":67376,"sha256":"a826c652f83e91908e1c176b8980d7213703de3869beb6984a37c716b2fe2fa5","final_host":"www.roblox.com","final_path":"/llms.txt","ms":329,"path":"/llms.txt","html":true,"json":null,"present":false},"agents_md":{"status":404,"content_type":"text/html","bytes":67376,"sha256":"305bbe383cc8dbe2a8a1994678336a1b775a34a91d0f56b90f0acbe988777c01","final_host":"www.roblox.com","final_path":"/agents.md","ms":351,"path":"/agents.md","html":true,"json":null,"present":false},"agent_skills":{"status":404,"content_type":"application/json","bytes":40,"sha256":"6ed55387e8fd77ee283840993d928721a8c3779847c65e57e058082802c75eff","final_host":"www.roblox.com","final_path":"/.well-known/agent-skills/index.json","ms":298,"path":"/.well-known/agent-skills/index.json","html":false,"json":null,"present":false},"mcp_card":{"status":404,"content_type":"application/json","bytes":40,"sha256":"6ed55387e8fd77ee283840993d928721a8c3779847c65e57e058082802c75eff","final_host":"www.roblox.com","final_path":"/.well-known/mcp/server-card.json","ms":301,"path":"/.well-known/mcp/server-card.json","html":false,"json":null,"present":false},"api_catalog":{"status":404,"content_type":"application/json","bytes":40,"sha256":"6ed55387e8fd77ee283840993d928721a8c3779847c65e57e058082802c75eff","final_host":"www.roblox.com","final_path":"/.well-known/api-catalog","ms":305,"path":"/.well-known/api-catalog","html":false,"json":null,"present":false},"a2a_card":{"status":404,"content_type":"application/json","bytes":40,"sha256":"6ed55387e8fd77ee283840993d928721a8c3779847c65e57e058082802c75eff","final_host":"www.roblox.com","final_path":"/.well-known/agent-card.json","ms":302,"path":"/.well-known/agent-card.json","html":false,"json":null,"present":false}},"rings":{"v":1,"candidates":[],"mcp":[],"a2a":null,"webmcp":{"model_context":false,"bridge":false,"mcp_json":false,"link_rel":false,"any":false},"handshakes":0,"get_path":{"status":404,"content_type":"application/json","ms":264,"error":null},"get_host":{"status":530,"content_type":"text/plain","ms":71,"error":null}},"sf":[4,0,0,0,0,0,2,0,1],"derived":{"reachable":true,"home_class":"answers","robots_served":true,"robots_sitemap":true,"content_signal":false,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":[],"blocks_any_ai":false,"files":{"llms_txt":false,"agents_md":false,"agent_skills":false,"mcp_card":false,"api_catalog":false,"a2a_card":false},"any_capability":false,"ai_refused":[],"ai_thin":["gptbot","claudebot","oaisearch","perplexity"],"aipref":0,"aipref_measured":true,"sf":[4,0,0,0,0,0,2,0,1],"rings":{"mcp_declared":false,"mcp_live":false,"mcp_auth":false,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":3,"fh":"www.roblox.com","fp_other_host":[]},"derive_v":5,"frame":"top10k","shard":5},{"v":1,"kind":"crawlcheck-census-record","run":18,"domain":"endpoints.news","rank":63,"measured_at":"2026-11-13T18:53:15.625Z","home":{"browser":{"status":200,"content_type":"text/html","bytes":118391,"sha256":"2e6e5b86a70f03bc3d4d2616bde03dd35ad3ab87334177ea3f677403e5a62bc6","final_host":"endpoints.news","final_path":"/","ms":40,"identity":"Mobile browser","html":true,"fp":"67163857ef20113d"},"gptbot":{"status":403,"content_type":"text/html","bytes":919,"sha256":"dffac635bc36bd88b09af242f715a6ba32c9dfa5a48355a1d6e016d3225d0d56","final_host":"endpoints.news","final_path":"/","ms":16,"identity":"GPTBot","html":true},"claudebot":{"status":403,"content_type":"text/html","bytes":919,"sha256":"12633a1cbd47c1f10d7789f54563723cd49c76e93027d1a860b82bf1d93ea75c","final_host":"endpoints.news","final_path":"/","ms":6,"identity":"ClaudeBot","html":true},"oaisearch":{"status":403,"content_type":"text/html","bytes":919,"sha256":"1806622484d9d8ae9f3df9b6fc187f41d2d7010ca40df14b16e8bdf7cc5043ce","final_host":"endpoints.news","final_path":"/","ms":7,"identity":"OAI-SearchBot","html":true},"perplexity":{"status":403,"content_type":"text/html","bytes":919,"sha256":"454a1dde2f53518235ae6a4f20e36b4e82765d858fb050f3f6dad7933a8b564c","final_host":"endpoints.news","final_path":"/","ms":16,"identity":"PerplexityBot","html":true}},"robots":{"status":403,"content_type":"text/html","bytes":919,"sha256":"313bd6f2e845efe87c51848c21a92c96b096158089715ae5ca77374c9c4a25c3","final_host":"endpoints.news","final_path":"/robots.txt","ms":63,"html":true,"text":null,"agents":null},"sitemap":{"status":403,"content_type":"text/html","bytes":919,"sha256":"cdac8bb3c80be1cc5100f0cf360210abfaf9d5b03ddc6a269f0304155d94bf2c","final_host":"endpoints.news","final_path":"/sitemap.xml","ms":6,"url":"https://endpoints.news/sitemap.xml","declared":false,"xml":false},"files":{"llms_txt":{"status":403,"content_type":"text/html","bytes":919,"sha256":"6af06b382eed54f1ec58de2e9ff3de553b3b43dcabe36a701e10b29c4a1d4d17","final_host":"endpoints.news","final_path":"/llms.txt","ms":7,"path":"/llms.txt","html":true,"json":null,"present":false},"agents_md":{"status":403,"content_type":"text/html","bytes":919,"sha256":"45d09a560bf37bc4aaddb391fc475226ef16007d42eb230e703dc041021bbfad","final_host":"endpoints.news","final_path":"/agents.md","ms":7,"path":"/agents.md","html":true,"json":null,"present":false},"agent_skills":{"status":403,"content_type":"text/html","bytes":919,"sha256":"f9dbf0daf04da56f5bc0ee03e8290aa091ab00c7206d0bfcea1e6381791d12e8","final_host":"endpoints.news","final_path":"/.well-known/agent-skills/index.json","ms":15,"path":"/.well-known/agent-skills/index.json","html":true,"json":null,"present":false},"mcp_card":{"status":403,"content_type":"text/html","bytes":919,"sha256":"80dfbbcab4044eb328558193e4f23904ff4c5647f140f45e55fcba2b4e456522","final_host":"endpoints.news","final_path":"/.well-known/mcp/server-card.json","ms":7,"path":"/.well-known/mcp/server-card.json","html":true,"json":null,"present":false},"api_catalog":{"status":403,"content_type":"text/html","bytes":919,"sha256":"18bcc5dcab4ab39b62bff86fabd12a10f2030a155c527d390a9971fa5d8c8d95","final_host":"endpoints.news","final_path":"/.well-known/api-catalog","ms":7,"path":"/.well-known/api-catalog","html":true,"json":null,"present":false},"a2a_card":{"status":403,"content_type":"text/html","bytes":919,"sha256":"c2d1719d2bc9258f2fe852bc95ad6f9e179e8dc3424ecb3a6871cffbb65fcbf6","final_host":"endpoints.news","final_path":"/.well-known/agent-card.json","ms":6,"path":"/.well-known/agent-card.json","html":true,"json":null,"present":false}},"rings":{"v":1,"candidates":[],"mcp":[],"a2a":null,"webmcp":{"model_context":false,"bridge":false,"mcp_json":false,"link_rel":false,"any":false},"handshakes":0,"get_path":{"status":403,"content_type":"text/html","ms":8,"error":null},"get_host":{"status":530,"content_type":"text/plain","ms":112,"error":null}},"sf":[1,0,0,0,0,0,2,0,0],"derived":{"reachable":true,"home_class":"answers","robots_served":false,"robots_sitemap":false,"content_signal":false,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":[],"blocks_any_ai":false,"files":{"llms_txt":false,"agents_md":false,"agent_skills":false,"mcp_card":false,"api_catalog":false,"a2a_card":false},"any_capability":false,"ai_refused":["gptbot","claudebot","oaisearch","perplexity"],"ai_thin":[],"sf":[1,0,0,0,0,0,2,0,0],"rings":{"mcp_declared":false,"mcp_live":false,"mcp_auth":false,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":3,"fh":"endpoints.news","fp_other_host":[],"fp":"67163857ef20113d","fp_bits":{}},"derive_v":5,"frame":"top10k","shard":6},{"v":1,"kind":"crawlcheck-census-record","run":18,"domain":"windows.net","rank":64,"measured_at":"2026-11-13T18:55:41.951Z","home":{"browser":{"status":526,"content_type":"text/plain","bytes":16,"sha256":"4c543fe8a4d739c6f622b04303ac9e9d9fafbf00da950fc916ad826da2c0cdac","final_host":"windows.net","final_path":"/","ms":434,"identity":"Mobile browser","html":false},"gptbot":{"status":526,"content_type":"text/plain","bytes":16,"sha256":"4c543fe8a4d739c6f622b04303ac9e9d9fafbf00da950fc916ad826da2c0cdac","final_host":"windows.net","final_path":"/","ms":425,"identity":"GPTBot","html":false},"claudebot":{"status":526,"content_type":"text/plain","bytes":16,"sha256":"4c543fe8a4d739c6f622b04303ac9e9d9fafbf00da950fc916ad826da2c0cdac","final_host":"windows.net","final_path":"/","ms":652,"identity":"ClaudeBot","html":false},"oaisearch":{"status":526,"content_type":"text/plain","bytes":16,"sha256":"4c543fe8a4d739c6f622b04303ac9e9d9fafbf00da950fc916ad826da2c0cdac","final_host":"windows.net","final_path":"/","ms":433,"identity":"OAI-SearchBot","html":false},"perplexity":{"status":526,"content_type":"text/plain","bytes":16,"sha256":"4c543fe8a4d739c6f622b04303ac9e9d9fafbf00da950fc916ad826da2c0cdac","final_host":"windows.net","final_path":"/","ms":498,"identity":"PerplexityBot","html":false}},"robots":{"status":526,"content_type":"text/plain","bytes":16,"sha256":"4c543fe8a4d739c6f622b04303ac9e9d9fafbf00da950fc916ad826da2c0cdac","final_host":"windows.net","final_path":"/robots.txt","ms":487,"html":false,"text":null,"agents":null},"sitemap":null,"files":null,"derived":{"reachable":false,"home_class":"no_site","robots_served":false,"robots_sitemap":false,"content_signal":false,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":[],"blocks_any_ai":false,"files":{"llms_txt":false,"agents_md":false,"agent_skills":false,"mcp_card":false,"api_catalog":false,"a2a_card":false},"any_capability":false,"ai_refused":[],"ai_thin":[],"rings":{"mcp_declared":false,"mcp_live":false,"mcp_auth":false,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":3,"fh":"windows.net","fp_other_host":[]},"derive_v":5,"frame":"top10k","shard":7},{"v":1,"kind":"crawlcheck-census-record","run":18,"domain":"adobe.com","rank":65,"measured_at":"2026-11-13T18:46:27.088Z","home":{"browser":{"status":200,"content_type":"text/html","bytes":51346,"sha256":"5c0adc58abd0b7f928acd6ba58a183345ae4224cb1c33542fe9a6ad18fa1b543","final_host":"www.adobe.com","final_path":"/","ms":231,"identity":"Mobile browser","html":true,"fp":"cf1259fe84823eca"},"gptbot":{"status":403,"content_type":"text/html","bytes":367,"sha256":"70ff6020fe9ad9c940ee6b3d6b71b9087613b966b573ec5b672531cbcb9b8b89","final_host":"www.adobe.com","final_path":"/","ms":82,"identity":"GPTBot","html":true},"claudebot":{"status":403,"content_type":"text/html","bytes":367,"sha256":"514943c3e46a31846f8f52b9264ddacf468ccaa35e0dd705493b01778eb31641","final_host":"www.adobe.com","final_path":"/","ms":86,"identity":"ClaudeBot","html":true},"oaisearch":{"status":403,"content_type":"text/html","bytes":367,"sha256":"d386fa94c6ba815a4649e9bcd8c66950b63049419509ac5c2e2c2ae7f8482d68","final_host":"www.adobe.com","final_path":"/","ms":86,"identity":"OAI-SearchBot","html":true},"perplexity":{"status":403,"content_type":"text/html","bytes":367,"sha256":"c31b42a9069ed219ec4ca19339d73218851d8542ebe3db2c541b77d3022928af","final_host":"www.adobe.com","final_path":"/","ms":110,"identity":"PerplexityBot","html":true}},"robots":{"status":200,"content_type":"text/plain","bytes":10199,"sha256":"41081faa0fd323774e225ab51e8a73262055b24ecc7732557229fb93938f9d84","final_host":"www.adobe.com","final_path":"/robots.txt","ms":346,"html":false,"text":"# The use of robots or other automated means to access the Adobe site\n# without the express permission of Adobe is strictly prohibited.\n# Notwithstanding the foregoing, Adobe may permit automated access to\n# access certain Adobe pages but solely for the limited purpose of\n# including content in publicly available search engines. Any other\n# use of robots or failure to obey the robots exclusion standards set\n# forth at http://www.robotstxt.org/ is strictly prohibited.\n\n# Details about Googlebot available at: http://www.google.com/bot.html\n# The Google search engine can see everything\nUser-agent: gsa-crawler-www \nAllow: /assets/globalnav/v1.1.4/css/globalnav.css\nAllow: /assets/globalnav/v1.1.4/*/globalnav.js\nAllow: /au/special/symposium\nAllow: /sea/special/symposium\nAllow: /*special/products/marketing-cloud/*.css$\nAllow: /*special/products/marketing-cloud/*.png$\nAllow: /*special/products/marketing-cloud/*.js$\nAllow: /www.adobe.com/special/ri/prod/assets/*.css$\nAllow: /www.adobe.com/special/ri/prod/assets/*.jpg$\nAllow: /www.adobe.com/special/ri/prod/assets/*.js$\nAllow: /www.adobe.com/special/ri/prod/assets/*.png$\nAllow: /www.adobe.com/special/retail-shopping-insights/\nAllow: /uk/special/landing/student/digitaledgeaward/\nAllow: /uk/special/landing/student/digitaledgeaward/\nAllow: /uk/special/landing/student/lovethejourney\nAllow: /fr/special/landing/student/lovethejourney\nDisallow: /assets/globalnav/\nDisallow: /*creativesuite/bridge\nDisallow: /cfusion/gpr/\nDisallow: /cfusion/search/\nDisallow: /cfusion/partnerportal/\nDisallow: /globalsearch\nDisallow: /cgi-local/\nDisallow: /de/products/index-notabs.html\nDisallow: /de/products/index-tabs.html\nDisallow: /devnet-archive/\nDisallow: /devnet/archive\nDisallow: /*devnet/tagsearch\nDisallow: /emea/eemeachannel/\nDisallow: /events/ciocouncil/\nDisallow: /events/executivecouncil/\nDisallow: /google_indexing/\nDisallow: /*.html_\nDisallow: /licensing/distribution/strategies/\nDisallow: /*limited/\nDisallow: /livedocs/\nDisallow: /*Ljava.lang.Object*\nDisallow: /products/acrobat/readstep2.html;JMICROSITE2SESSION*\nDisallow: /*products/flashplayer/distribution3.html\nDisallow: /*products/flashplayer/distribution4.html\nDisallow: /*special/\nDisallow: /*.sql$\nDisallow: /*supportservice/\nDisallow: /*.translate.html\nDisallow: /*products/adobeconnect/enterprise/trial.html\nDisallow: /*products/adobeconnect/enterprise/trial-check.html\nDisallow: /*products/adobeconnect/enterprise/trial-thankyou.html\nDisallow: /*products/adobeconnect/enterprise/trial-error.html\nDisallow: /*products/photoshop/selector\nDisallow: /*unsubscribe.html\nDisallow: /content/dam/acom/en/marketing-cloud/offers/pdf\nDisallow: /content/dam/www/us/en/adi/2020/pdfs/*\nDisallow: /content/dotcom/*.html$\nDisallow: /content/dam/www/us/en/offer/digital-trends-2020/digital-trends-2020-full-report.pdf\nDisallow: /plans-fragments/\nDisallow: /plans.modals*\n\n# The Omniture search engine can see everything\nUser-agent: Atomz/1.0\nDisallow: /devnet-archive/\nDisallow: /devnet/archive\nDisallow: /events/ciocouncil/\nDisallow: /events/executivecouncil/\nDisallow: /licensing/distribution/strategies/\nDisallow: /*limited/\nDisallow: /livedocs/\nDisallow: /products/flashplayer/distribution3.html\nDisallow: /products/flashplayer/distribution4.html\nDisallow: /*special/\nDisallow: /*.sql$\nDisallow: /*supportservice/\n\nUser-agent: *\nAllow: /assets/globalnav/v1.1.4/css/globalnav.css\nAllow: /assets/globalnav/v1.1.4/*globalnav.js\nAllow: /au/special/symposium\nAllow: /sea/special/symposium\nAllow: /*special/products/marketing-cloud/*.css$\nAllow: /*special/products/marketing-cloud/*.js$\nAllow: /*special/products/marketing-cloud/*.png$\nAllow: /www.adobe.com/special/ri/prod/assets/*.css$\nAllow: /www.adobe.com/special/ri/prod/assets/*.jpg$\nAllow: /www.adobe.com/special/ri/prod/assets/*.js$\nAllow: /www.adobe.com/special/ri/prod/assets/*.png$\nAllow: /www.adobe.com/special/retail-shopping-insights/\nAllow: /uk/special/landing/student/digitaledgeaward/\nAllow: /uk/special/landing/student/lovethejourney\nAllow: /fr/special/landing/student/lovethejourney\nAllow: /community/gallery$\nAllow: /community/gallery/assets$\nAllow: /community/gallery/challenges$\nDisallow: /assets/globalnav/\nDisallow: /*creativesuite/bridge\nDisallow: /cfusion/gpr/\nDisallow: /cfusion/search/\nDisallow: /cfusion/partnerportal/\nDisallow: /globalsearch\nDisallow: /cgi-local/\nDisallow: /de/products/index-notabs.html\nDisallow: /de/products/index-tabs.html\nDisallow: /devnet-archive/\nDisallow: /devnet/archive\nDisallow: /*devnet/tagsearch\nDisallow: /emea/eemeachannel/\nDisallow: /events/ciocouncil/\nDisallow: /events/executivecouncil/\nDisallow: /google_indexing/\nDisallow: /*.html_\nDisallow: /licensing/distribution/strategies/\nDisallow: /*limited/\nDisallow: /livedocs/\nDisallow: /*Ljava.lang.Object*\nDisallow: /products/acrobat/readstep2.html;JMICROSITE2SESSION*\nDisallow: /*products/flashplayer/distribution3.html\nDisallow: /*products/flashplayer/distribution4.html\nDisallow: /*special/\nDisallow: /*.sql$\nDisallow: /*supportservice/\nDisallow: /*.translate.html\nDisallow: /*products/adobeconnect/enterprise/trial.html\nDisallow: /*products/adobeconnect/enterprise/trial-check.html\nDisallow: /*products/adobeconnect/enterprise/trial-thankyou.html\nDisallow: /*products/adobeconnect/enterprise/trial-error.html\nDisallow: /*products/photoshop/selector\nDisallow: /*unsubscribe.html\nDisallow: /content/dam/acom/en/marketing-cloud/offers/pdf\nDisallow: /content/dotcom/*.html$\nDisallow: /plans-fragments/\nDisallow: /content/dam/www/us/en/offer/digital-trends-2020-in-media-and-entertainment-html/digital-trends-2020-in-media-and-entertainment.pdf\nDisallow: /content/dam/www/us/en/offer/digital-trends-2020-in-media-and-entertainment-html/digital-trends-2020-in-media-and-entertainment-DE.pdf\nDisallow: /content/dam/www/us/en/offer/digital-trends-2020-in-media-and-entertainment-html/digital-trends-2020-in-media-and-entertainment-EN-US.pdf\nDisallow: /content/dam/www/us/en/offer/digital-trends-2020-in-media-and-entertainment-html/digital-trends-2020-in-media-and-entertainment-EN.pdf\nDisallow: /content/dam/www/us/en/offer/digital-trends-2020-in-media-and-entertainment-html/digital-trends-2020-in-media-and-entertainment-ES.pdf\nDisallow: /content/dam/www/us/en/offer/digital-trends-2020-in-media-and-entertainment-html/digital-trends-2020-in-media-and-entertainment-FR.pdf\nDisallow: /content/dam/www/us/en/offer/digital-trends-2020-in-media-and-entertainment-html/digital-trends-2020-in-media-and-entertainment-IT.pdf\nDisallow: /content/dam/www/us/en/offer/digital-trends-2020-in-media-and-entertainment-html/digital-trends-2020-in-media-and-entertainment-JA.pdf\nDisallow: /content/dam/www/us/en/offer/digital-trends-2020-in-media-and-entertainment-html/digital-trends-2020-in-media-and-entertainment-KR.pdf\nDisallow: /content/dam/www/us/en/offer/digital-trends-2020-in-media-and-entertainment-html/digital-trends-2020-in-media-and-entertainment-zh-Hans.pdf\nDisallow: /content/dam/www/us/en/offer/digital-trends-2020-in-media-and-entertainment-html/digital-trends-2020-in-media-and-entertainment-zh-Hant.pdf\nDisallow: /content/dam/www/us/en/offer/digital-trends-2020-in-media-and-entertainment-html/digital_trends_21_core_report_TC_FA.pdf\nDisallow: /content/dam/dx/us/en/resources/reports/digital-trends-2021-in-high-tech/2021_Digital_Trends_High_Tech_in_Focus.pdf\nDisallow: /content/dam/dx/us/en/resources/digital-trends-2021-in-telecommunications-media-and-entertainment/Digital_Trends_2021_Media_Entertainment_and_Telecoms_Report.pdf\nDisallow: /content/dam/dx/us/en/resources/digital-trends-2021-in-retail/2021_Digital_Trends_Retail_in_Focus.pdf\nDisallow: /content/dam/dx/us/en/resources/digital-trends-2021-in-manufacturing/2021_Digital_Trends_Manufacturing_in_Focus.pdf\nDisallow: /content/dam/www/us/en/offer/technology-trends-2021/eConsultancy_Digital_Trends_2021_IT_Report.pdf\nDisallow: /content/dam/www/us/en/offer/pdf/digital-trends-2020-in-financial-services.pdf\nDisallow: /content/dam/www/us/en/offer/pdf/digital-trends-2020-in-retail.pdf\nDisallow: /content/dam/www/us/en/offer/pdf/digital-trends-2020-in-media-and-entertainment.pdf\nDisallow: /content/dam/www/us/en/offer/pdf/digital-trends-2020-in-travel-and-hospitality.pdf\nDisallow: /content/dam/www/us/en/offer/pdf/digital-trends-2020-in-telecommunications.pdf\nDisallow: /content/dam/www/us/en/offer/pdf/digital-trends-2020-in-advertising.pdf\nDisallow: /content/dam/www/us/en/offer/pdf/digital-trends-2020-in-high-tech.pdf\nDisallow: /content/dam/www/us/en/offer/pdf/digital-trends-2020-in-IT.pdf\nDisallow: /content/dam/www/us/en/adi/2020/pdfs/*\nDisallow: /content/dam/www/us/en/offer/digital-trends-2021-core/digital-trends-2021-full-report-*.pdf\nDisallow: /*langmaster/\nDisallow: /*express/*.mp4$\nDisallow: /en-AE/learn/\nDisallow: /en-BE/learn/\nDisallow: /en-CY/learn/\nDisallow: /en-GR/learn/\nDisallow: /en-HK/learn/\nDisallow: /en-ID/learn/\nDisallow: /en-IL/learn/\nDisallow: /en-LU/learn/\nDisallow: /en-MT/learn/\nDisallow: /en-MY/learn/\nDisallow: /en-PH/learn/\nDisallow: /en-SA/learn/\nDisallow: /en-SG/learn/\nDisallow: /en-TH/learn/\nDisallow: /en-VN/learn/\nDisallow: /en-x-MenA/learn/\nDisallow: /en-ZA/learn/\nDisallow: /*?searchId=\nDisallow: /*?tasks=\nDisallow: /*?campaignid=\nDisallow: /*?skwcid=\nDisallow: /products/photoshop/photoshop-brushes.h*\nDisallow: /community/gallery\n\nUser-agent: Googlebot\nAllow: /?promoid=RTQCN3LX\nDisallow: /*promoid=\nDisallow: /*trackingid=\nDisallow: /*?searchId=\nDisallow: /*?tasks=\nDisallow: /*?campaignid=\nDisallow: /*?skwcid=\nDisallow: /products/photoshop/photoshop-brushes.h*\n\nUser-agent: AdsBot-Google\nAllow: /\n\nUser-agent: AdsBot-Google-Mobile\nAllow: /\n\nUser-agent: SearchmetricsBot\nDisallow: /\n\n# XML sitemaps\nSitemap: https://www.adobe.com/cc-product.index.xml\nSitemap: https://www.adobe.com/creativecloud/sitemap-index.xml\nSitemap: https://www.adobe.com/acrobat/sitemap-index.xml\nSitemap: https://www.adobe.com/home-sitemap.xml\nSitemap: https://www.adobe.com/plans-catalog-sitemap.xml\nSitemap: https://www.adobe.com/learn/sitemap_index.xml\n\n# CC Express \nSitemap: https://www.adobe.com/express/sitemap-index.xml\n\n#Naver\nSitemap: https://www.adobe.com/naver.index.xml\n\n#Education\nSitemap: https://www.adobe.com/edu/sitemap-index.xml\n","agents":{"GPTBot":true,"OAI-SearchBot":true,"ChatGPT-User":true,"ClaudeBot":true,"Claude-SearchBot":true,"PerplexityBot":true,"Google-Extended":true,"Applebot-Extended":true,"CCBot":true,"Bytespider":true,"Meta-ExternalAgent":true,"Amazonbot":true}},"sitemap":{"status":200,"content_type":"application/xml","bytes":8649,"sha256":"8f5710016d9dce1daf3d0d0cbe81f62c1154d28961484d0fb8c518a22a5d638d","final_host":"www.adobe.com","final_path":"/cc-product.index.xml","ms":62,"url":"https://www.adobe.com/cc-product.index.xml","declared":true,"xml":true},"files":{"llms_txt":{"status":200,"content_type":"text/plain","bytes":10499,"sha256":"811353cb603ff604ecd257b7e24643f1c7e157837015221a951d9d2db0c494d8","final_host":"www.adobe.com","final_path":"/llms.txt","ms":130,"path":"/llms.txt","html":false,"json":null,"present":true},"agents_md":{"status":404,"content_type":"text/html","bytes":1323,"sha256":"2c9dd78b24598e2cff988fcf6e809931ee5450e5d0a5a57d7e97a91092e018b5","final_host":"www.adobe.com","final_path":"/agents.md","ms":195,"path":"/agents.md","html":true,"json":null,"present":false},"agent_skills":{"status":404,"content_type":"text/html","bytes":1323,"sha256":"2c9dd78b24598e2cff988fcf6e809931ee5450e5d0a5a57d7e97a91092e018b5","final_host":"www.adobe.com","final_path":"/.well-known/agent-skills/index.json","ms":140,"path":"/.well-known/agent-skills/index.json","html":true,"json":null,"present":false},"mcp_card":{"status":404,"content_type":"text/html","bytes":1323,"sha256":"2c9dd78b24598e2cff988fcf6e809931ee5450e5d0a5a57d7e97a91092e018b5","final_host":"www.adobe.com","final_path":"/.well-known/mcp/server-card.json","ms":135,"path":"/.well-known/mcp/server-card.json","html":true,"json":null,"present":false},"api_catalog":{"status":404,"content_type":"text/html","bytes":1323,"sha256":"2c9dd78b24598e2cff988fcf6e809931ee5450e5d0a5a57d7e97a91092e018b5","final_host":"www.adobe.com","final_path":"/.well-known/api-catalog","ms":251,"path":"/.well-known/api-catalog","html":true,"json":null,"present":false},"a2a_card":{"status":404,"content_type":"text/html","bytes":1323,"sha256":"2c9dd78b24598e2cff988fcf6e809931ee5450e5d0a5a57d7e97a91092e018b5","final_host":"www.adobe.com","final_path":"/.well-known/agent-card.json","ms":131,"path":"/.well-known/agent-card.json","html":true,"json":null,"present":false}},"rings":{"v":1,"candidates":[],"mcp":[],"a2a":null,"webmcp":{"model_context":false,"bridge":false,"mcp_json":false,"link_rel":false,"any":false},"handshakes":0,"get_path":{"status":404,"content_type":"text/html","ms":134,"error":null},"get_host":{"status":530,"content_type":"text/plain","ms":5,"error":null}},"sf":[4,0,0,0,0,0,0,0,0],"derived":{"reachable":true,"home_class":"answers","robots_served":true,"robots_sitemap":true,"content_signal":false,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":[],"blocks_any_ai":false,"files":{"llms_txt":true,"agents_md":false,"agent_skills":false,"mcp_card":false,"api_catalog":false,"a2a_card":false},"any_capability":false,"ai_refused":["gptbot","claudebot","oaisearch","perplexity"],"ai_thin":[],"aipref":0,"aipref_measured":true,"sf":[4,0,0,0,0,0,0,0,0],"rings":{"mcp_declared":false,"mcp_live":false,"mcp_auth":false,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":3,"fh":"www.adobe.com","fp_other_host":[],"fp":"cf1259fe84823eca","fp_bits":{}},"derive_v":5,"frame":"top10k","shard":0},{"v":1,"kind":"crawlcheck-census-record","run":18,"domain":"pv-cdn.net","rank":66,"measured_at":"2026-11-13T18:48:18.168Z","home":{"browser":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"pv-cdn.net","final_path":"/","ms":6,"identity":"Mobile browser","html":false},"gptbot":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"pv-cdn.net","final_path":"/","ms":1,"identity":"GPTBot","html":false},"claudebot":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"pv-cdn.net","final_path":"/","ms":0,"identity":"ClaudeBot","html":false},"oaisearch":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"pv-cdn.net","final_path":"/","ms":1,"identity":"OAI-SearchBot","html":false},"perplexity":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"pv-cdn.net","final_path":"/","ms":0,"identity":"PerplexityBot","html":false}},"robots":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"pv-cdn.net","final_path":"/robots.txt","ms":1,"html":false,"text":null,"agents":null},"sitemap":null,"files":null,"derived":{"reachable":false,"home_class":"no_site","robots_served":false,"robots_sitemap":false,"content_signal":false,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":[],"blocks_any_ai":false,"files":{"llms_txt":false,"agents_md":false,"agent_skills":false,"mcp_card":false,"api_catalog":false,"a2a_card":false},"any_capability":false,"ai_refused":[],"ai_thin":[],"rings":{"mcp_declared":false,"mcp_live":false,"mcp_auth":false,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":3,"fh":"pv-cdn.net","fp_other_host":[]},"derive_v":5,"frame":"top10k","shard":1},{"v":1,"kind":"crawlcheck-census-record","run":18,"domain":"windowsupdate.com","rank":67,"measured_at":"2026-11-13T18:49:51.543Z","home":{"browser":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"windowsupdate.com","final_path":"/","ms":19,"identity":"Mobile browser","html":false},"gptbot":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"windowsupdate.com","final_path":"/","ms":1,"identity":"GPTBot","html":false},"claudebot":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"windowsupdate.com","final_path":"/","ms":1,"identity":"ClaudeBot","html":false},"oaisearch":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"windowsupdate.com","final_path":"/","ms":1,"identity":"OAI-SearchBot","html":false},"perplexity":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"windowsupdate.com","final_path":"/","ms":1,"identity":"PerplexityBot","html":false}},"robots":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"windowsupdate.com","final_path":"/robots.txt","ms":1,"html":false,"text":null,"agents":null},"sitemap":null,"files":null,"derived":{"reachable":false,"home_class":"no_site","robots_served":false,"robots_sitemap":false,"content_signal":false,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":[],"blocks_any_ai":false,"files":{"llms_txt":false,"agents_md":false,"agent_skills":false,"mcp_card":false,"api_catalog":false,"a2a_card":false},"any_capability":false,"ai_refused":[],"ai_thin":[],"rings":{"mcp_declared":false,"mcp_live":false,"mcp_auth":false,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":3,"fh":"windowsupdate.com","fp_other_host":[]},"derive_v":5,"frame":"top10k","shard":2},{"v":1,"kind":"crawlcheck-census-record","run":18,"domain":"spotify.com","rank":68,"measured_at":"2026-11-13T18:50:46.493Z","home":{"browser":{"status":200,"content_type":"text/html","bytes":315593,"sha256":"6b8fed4ac2e89238c166566b30b901e8bd9e299ee44028e31a1db0d4dec02a5e","final_host":"open.spotify.com","final_path":"/","ms":611,"identity":"Mobile browser","html":true,"fp":"1b050ae5da3d3c21"},"gptbot":{"status":200,"content_type":"text/html","bytes":172382,"sha256":"88df3e03da8b6de012da61b8cbdeec8f9dfc66557ce4afa965403df28bcdc254","final_host":"open.spotify.com","final_path":"/","ms":558,"identity":"GPTBot","html":true},"claudebot":{"status":200,"content_type":"text/html","bytes":172382,"sha256":"574c539110472a3960b78bf0c4b8c4f2f41b3b6f93ef13b1abe2a84236a17b0e","final_host":"open.spotify.com","final_path":"/","ms":416,"identity":"ClaudeBot","html":true},"oaisearch":{"status":200,"content_type":"text/html","bytes":172382,"sha256":"69eb88ea82ee0063940c0737b0e35b13aeb9fe52447342f30bd5b87f73a4393d","final_host":"open.spotify.com","final_path":"/","ms":453,"identity":"OAI-SearchBot","html":true},"perplexity":{"status":200,"content_type":"text/html","bytes":172382,"sha256":"745a6bad1b723b936d8fa25fee6c3aebec86705ce506c30506ce1dec96375d1e","final_host":"open.spotify.com","final_path":"/","ms":222,"identity":"PerplexityBot","html":true}},"robots":{"status":200,"content_type":"text/plain","bytes":635,"sha256":"b68c89548f813d5c465c13051353b9452dba066f14146cafabb46dafd239b738","final_host":"www.spotify.com","final_path":"/robots.txt","ms":484,"html":false,"text":"User-agent: *\nDisallow: /*/about-us/contact/contact-spotify-password/\nDisallow: /*/about-us/contact/contact-spotify-account/\nDisallow: /*/get-spotify/*\nDisallow: /*/xhr/*\nDisallow: /*/external/*\nDisallow: /*/legal/*?ets=\nDisallow: /*/legal/advertiser-terms-and-conditions/\nDisallow: /*/legal/gdpr-article-15-information/\nDisallow: /*/legal/spotify-controller-data-processing-terms/\nDisallow: /*/legal/podcast-api-terms/\nDisallow: /*/legal/distribution-api-terms/\nDisallow: /*/account/cls/*\nDisallow: /*/starbuckspartners\nDisallow: /starbuckspartners\nDisallow: /ppt/*?\nDisallow: /partner/*?\nSitemap: https://www.spotify.com/sitemap.xml\n","agents":{"GPTBot":true,"OAI-SearchBot":true,"ChatGPT-User":true,"ClaudeBot":true,"Claude-SearchBot":true,"PerplexityBot":true,"Google-Extended":true,"Applebot-Extended":true,"CCBot":true,"Bytespider":true,"Meta-ExternalAgent":true,"Amazonbot":true}},"sitemap":{"status":200,"content_type":"application/xml","bytes":1114,"sha256":"2ae8c8fbf612a2977db25fba637d68380b5855470af21c83312aed14e1339258","final_host":"www.spotify.com","final_path":"/sitemap.xml","ms":110,"url":"https://www.spotify.com/sitemap.xml","declared":true,"xml":true},"files":{"llms_txt":{"status":200,"content_type":"text/html","bytes":50829,"sha256":"b1ab81c229a5a7370c7fa02e90c4811d69f594afbc406dbf9f890508e5f435eb","final_host":"www.spotify.com","final_path":"/us/llms.txt","ms":312,"path":"/llms.txt","html":true,"json":null,"present":false},"agents_md":{"status":200,"content_type":"text/html","bytes":50829,"sha256":"b1ab81c229a5a7370c7fa02e90c4811d69f594afbc406dbf9f890508e5f435eb","final_host":"www.spotify.com","final_path":"/us/agents.md","ms":266,"path":"/agents.md","html":true,"json":null,"present":false},"agent_skills":{"status":404,"content_type":"text/html","bytes":3310,"sha256":"554b0c3a6ad268de5b3c8b3deb4205936ce1f3dd80303ec31af3b906d4fa9cf5","final_host":"www.spotify.com","final_path":"/.well-known/agent-skills/index.json","ms":131,"path":"/.well-known/agent-skills/index.json","html":true,"json":null,"present":false},"mcp_card":{"status":404,"content_type":"text/html","bytes":3310,"sha256":"554b0c3a6ad268de5b3c8b3deb4205936ce1f3dd80303ec31af3b906d4fa9cf5","final_host":"www.spotify.com","final_path":"/.well-known/mcp/server-card.json","ms":128,"path":"/.well-known/mcp/server-card.json","html":true,"json":null,"present":false},"api_catalog":{"status":404,"content_type":"text/html","bytes":3310,"sha256":"554b0c3a6ad268de5b3c8b3deb4205936ce1f3dd80303ec31af3b906d4fa9cf5","final_host":"www.spotify.com","final_path":"/.well-known/api-catalog","ms":203,"path":"/.well-known/api-catalog","html":true,"json":null,"present":false},"a2a_card":{"status":404,"content_type":"text/html","bytes":3310,"sha256":"554b0c3a6ad268de5b3c8b3deb4205936ce1f3dd80303ec31af3b906d4fa9cf5","final_host":"www.spotify.com","final_path":"/.well-known/agent-card.json","ms":263,"path":"/.well-known/agent-card.json","html":true,"json":null,"present":false}},"rings":{"v":1,"candidates":[],"mcp":[],"a2a":null,"webmcp":{"model_context":false,"bridge":false,"mcp_json":false,"link_rel":false,"any":false},"handshakes":0,"get_path":{"status":200,"content_type":"text/html","ms":268,"error":null},"get_host":{"status":530,"content_type":"text/plain","ms":131,"error":null}},"sf":[4,0,0,0,0,0,1,0,0],"derived":{"reachable":true,"home_class":"answers","robots_served":true,"robots_sitemap":true,"content_signal":false,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":[],"blocks_any_ai":false,"files":{"llms_txt":false,"agents_md":false,"agent_skills":false,"mcp_card":false,"api_catalog":false,"a2a_card":false},"any_capability":false,"ai_refused":[],"ai_thin":["gptbot","claudebot","oaisearch","perplexity"],"aipref":0,"aipref_measured":true,"sf":[4,0,0,0,0,0,1,0,0],"rings":{"mcp_declared":false,"mcp_live":false,"mcp_auth":false,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":3,"fh":"open.spotify.com","fp_other_host":[],"fp":"1b050ae5da3d3c21","fp_bits":{}},"derive_v":5,"frame":"top10k","shard":3},{"v":1,"kind":"crawlcheck-census-record","run":18,"domain":"wa.me","rank":69,"measured_at":"2026-11-13T18:51:40.579Z","home":{"browser":{"status":400,"content_type":"text/html","bytes":6278,"sha256":"3234b6bd5ee5749585c0483d899e9551b107f52b12cc7e9ecaf115bc31d2c811","final_host":"api.whatsapp.com","final_path":"/","ms":233,"identity":"Mobile browser","html":true},"gptbot":{"status":200,"content_type":"text/html","bytes":187383,"sha256":"4fc6d86b452c5a25a1c3fa431884101139f2ab9687e9975b340f49dbf3beddbd","final_host":"api.whatsapp.com","final_path":"/","ms":336,"identity":"GPTBot","html":true,"fp":"75c68d3fb9077b36"},"claudebot":{"status":200,"content_type":"text/html","bytes":187334,"sha256":"27d49531fd4d103c0a10c5919493d6049be0cf4015c01ac1a92cecf30b3f9b3e","final_host":"api.whatsapp.com","final_path":"/","ms":329,"identity":"ClaudeBot","html":true,"fp":"75c68d3fb9077b36"},"oaisearch":{"status":200,"content_type":"text/html","bytes":187380,"sha256":"583dc732277771550b8191979897b77de07d7ae71cf1acbaa0e10c0f56aa890f","final_host":"api.whatsapp.com","final_path":"/","ms":312,"identity":"OAI-SearchBot","html":true,"fp":"75c68d3fb9077b36"},"perplexity":{"status":200,"content_type":"text/html","bytes":197638,"sha256":"82c6e4fe0398dfc3010ba8f26dc183ae119d70b971bc04c9e76a14df3237dea1","final_host":"api.whatsapp.com","final_path":"/","ms":460,"identity":"PerplexityBot","html":true,"fp":"6516893f3b077b3e"}},"robots":{"status":200,"content_type":"text/plain","bytes":1525,"sha256":"6ceebd7d44206c973321a46e46b94102a8edcc37eea87394b2dd94ffe713df8d","final_host":"wa.me","final_path":"/robots.txt","ms":104,"html":false,"text":"# Notice: Collection of data on WhatsApp through automated means is prohibited\n# See: https://www.whatsapp.com/legal/terms-of-service\nuser-agent: *\ndisallow: /usephone/\ndisallow: /message/\ndisallow: /qr/\ndisallow: /pay/\ndisallow: /stickerpack/\ndisallow: /biztools/\ndisallow: /settings/\ndisallow: /disappearing_messages/\ndisallow: /download/\ndisallow: /archive_settings/\ndisallow: /tos/\ndisallow: /support/\ndisallow: /bizsearch/\ndisallow: /directory/\ndisallow: /guia/\ndisallow: /profile/\ndisallow: /status/\ndisallow: /premium/\ndisallow: /community/\ndisallow: /advertise/\ndisallow: /biztab/\ndisallow: /biz-edit-profile/\ndisallow: /biz-edit-description/\ndisallow: /biz-edit-catalog/\ndisallow: /biz-location/\ndisallow: /biz-hours/\ndisallow: /biz-linked-accounts/\ndisallow: /privacy/\ndisallow: /biz-website/\ndisallow: /proxy/\ndisallow: /biz-price-tier/\ndisallow: /chat-transfer/\ndisallow: /calling/\ndisallow: /ais/\ndisallow: /marketing-message/\ndisallow: /invite/\ndisallow: /contacts/\ndisallow: /edit-profile-picture/\ndisallow: /wabai/\ndisallow: /notifications/\ndisallow: /favorites/\ndisallow: /meta_verified/\ndisallow: /biz-agents-onboarding/\ndisallow: /subscription_hub/\ndisallow: /share-whatsapp-web/\ndisallow: /biz-ai-learning-summary/\ndisallow: /waffle/\ndisallow: /family/\ndisallow: /daily-ads-summary/\ndisallow: /tools/\ndisallow: /join_nova_waitlist/\ndisallow: /man/\ndisallow: /hatch/\ndisallow: /subscriptions/\ndisallow: /biz-broadcast-home/\ndisallow: /business-broadcast/\nsitemap: https://wa.me/smp/wa_smb_custom_url.xml.gz","agents":{"GPTBot":true,"OAI-SearchBot":true,"ChatGPT-User":true,"ClaudeBot":true,"Claude-SearchBot":true,"PerplexityBot":true,"Google-Extended":true,"Applebot-Extended":true,"CCBot":true,"Bytespider":true,"Meta-ExternalAgent":true,"Amazonbot":true}},"sitemap":null,"files":null,"derived":{"reachable":false,"home_class":"refused_us","robots_served":false,"robots_sitemap":false,"content_signal":false,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":[],"blocks_any_ai":false,"files":{"llms_txt":false,"agents_md":false,"agent_skills":false,"mcp_card":false,"api_catalog":false,"a2a_card":false},"any_capability":false,"ai_refused":[],"ai_thin":[],"rings":{"mcp_declared":false,"mcp_live":false,"mcp_auth":false,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":3,"fh":"api.whatsapp.com","fp_other_host":[]},"derive_v":5,"frame":"top10k","shard":4},{"v":1,"kind":"crawlcheck-census-record","run":18,"domain":"myfritz.net","rank":70,"measured_at":"2026-11-13T18:52:33.408Z","home":{"browser":{"status":200,"content_type":"text/html","bytes":6885,"sha256":"0ae8e018e72bcac840700b49e691dd87595c3c6fe0bf94aa6ca9e47030b2fe66","final_host":"www.myfritz.net","final_path":"/devices","ms":3263,"identity":"Mobile browser","html":true},"gptbot":{"status":200,"content_type":"text/html","bytes":6885,"sha256":"0ae8e018e72bcac840700b49e691dd87595c3c6fe0bf94aa6ca9e47030b2fe66","final_host":"www.myfritz.net","final_path":"/devices","ms":3183,"identity":"GPTBot","html":true},"claudebot":{"status":200,"content_type":"text/html","bytes":6885,"sha256":"0ae8e018e72bcac840700b49e691dd87595c3c6fe0bf94aa6ca9e47030b2fe66","final_host":"www.myfritz.net","final_path":"/devices","ms":2957,"identity":"ClaudeBot","html":true},"oaisearch":{"status":200,"content_type":"text/html","bytes":6885,"sha256":"0ae8e018e72bcac840700b49e691dd87595c3c6fe0bf94aa6ca9e47030b2fe66","final_host":"www.myfritz.net","final_path":"/devices","ms":2593,"identity":"OAI-SearchBot","html":true},"perplexity":{"status":200,"content_type":"text/html","bytes":6885,"sha256":"0ae8e018e72bcac840700b49e691dd87595c3c6fe0bf94aa6ca9e47030b2fe66","final_host":"www.myfritz.net","final_path":"/devices","ms":2440,"identity":"PerplexityBot","html":true}},"robots":{"status":200,"content_type":"text/plain","bytes":396,"sha256":"b3bcd97fea71d357a13505629dbefa700279b72c572479a6c9ec8def9814fc16","final_host":"myfritz.net","final_path":"/robots.txt","ms":1337,"html":false,"text":"User-agent: Yandex \nDisallow: /\n\nUser-agent: meta-externalagent\nDisallow: /\n\nUser-agent: meta-externalads\nDisallow: /\n\nUser-agent: meta-webindexer\nDisallow: /\n\nUser-agent: *\nDisallow: /\nAllow: /$\nAllow: /devices$\nAllow: /devices/$\nAllow: /devices/index.html$\nAllow: /sitemap.xml\nAllow: /legal-notice\nAllow: /terms\nAllow: /about\nHost: www.myfritz.net\n\nSitemap: https://www.myfritz.net/sitemap.xml\n","agents":{"GPTBot":true,"OAI-SearchBot":true,"ChatGPT-User":true,"ClaudeBot":true,"Claude-SearchBot":true,"PerplexityBot":true,"Google-Extended":true,"Applebot-Extended":true,"CCBot":true,"Bytespider":true,"Meta-ExternalAgent":false,"Amazonbot":true}},"sitemap":{"status":200,"content_type":"text/xml","bytes":2903,"sha256":"b83c75a3e8aa820ed2f4297239776d397858323c41e1edc4fcb339d16e973040","final_host":"www.myfritz.net","final_path":"/sitemap.xml","ms":308,"url":"https://www.myfritz.net/sitemap.xml","declared":true,"xml":true},"files":{"llms_txt":{"status":200,"content_type":"text/html","bytes":6885,"sha256":"0ae8e018e72bcac840700b49e691dd87595c3c6fe0bf94aa6ca9e47030b2fe66","final_host":"www.myfritz.net","final_path":"/devices","ms":594,"path":"/llms.txt","html":true,"json":null,"present":false},"agents_md":{"status":200,"content_type":"text/html","bytes":6885,"sha256":"0ae8e018e72bcac840700b49e691dd87595c3c6fe0bf94aa6ca9e47030b2fe66","final_host":"www.myfritz.net","final_path":"/devices","ms":598,"path":"/agents.md","html":true,"json":null,"present":false},"agent_skills":{"status":200,"content_type":"text/html","bytes":6885,"sha256":"0ae8e018e72bcac840700b49e691dd87595c3c6fe0bf94aa6ca9e47030b2fe66","final_host":"www.myfritz.net","final_path":"/devices","ms":833,"path":"/.well-known/agent-skills/index.json","html":true,"json":null,"present":false},"mcp_card":{"status":200,"content_type":"text/html","bytes":6885,"sha256":"0ae8e018e72bcac840700b49e691dd87595c3c6fe0bf94aa6ca9e47030b2fe66","final_host":"www.myfritz.net","final_path":"/devices","ms":662,"path":"/.well-known/mcp/server-card.json","html":true,"json":null,"present":false},"api_catalog":{"status":200,"content_type":"text/html","bytes":6885,"sha256":"0ae8e018e72bcac840700b49e691dd87595c3c6fe0bf94aa6ca9e47030b2fe66","final_host":"www.myfritz.net","final_path":"/devices","ms":600,"path":"/.well-known/api-catalog","html":true,"json":null,"present":false},"a2a_card":{"status":200,"content_type":"text/html","bytes":6885,"sha256":"0ae8e018e72bcac840700b49e691dd87595c3c6fe0bf94aa6ca9e47030b2fe66","final_host":"www.myfritz.net","final_path":"/devices","ms":613,"path":"/.well-known/agent-card.json","html":true,"json":null,"present":false}},"rings":{"v":1,"candidates":[],"mcp":[],"a2a":null,"webmcp":{"model_context":false,"bridge":false,"mcp_json":false,"link_rel":false,"any":false},"handshakes":0,"get_path":{"status":200,"content_type":"text/html","ms":671,"error":null},"get_host":{"status":530,"content_type":"text/plain","ms":626,"error":null}},"sf":[1,0,0,0,0,0,0,0,0],"derived":{"reachable":true,"home_class":"answers","robots_served":true,"robots_sitemap":true,"content_signal":false,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":["Meta-ExternalAgent"],"blocks_any_ai":true,"files":{"llms_txt":false,"agents_md":false,"agent_skills":false,"mcp_card":false,"api_catalog":false,"a2a_card":false},"any_capability":false,"ai_refused":[],"ai_thin":[],"aipref":0,"aipref_measured":true,"sf":[1,0,0,0,0,0,0,0,0],"rings":{"mcp_declared":false,"mcp_live":false,"mcp_auth":false,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":3,"fh":"www.myfritz.net","fp_other_host":[]},"derive_v":5,"frame":"top10k","shard":5},{"v":1,"kind":"crawlcheck-census-record","run":18,"domain":"msn.com","rank":71,"measured_at":"2026-11-13T18:53:15.625Z","home":{"browser":{"status":200,"content_type":"text/html","bytes":316911,"sha256":"4d42e3a3b576071a25d2ece23ac00f90b62bddac222c88394d8e9e4f319e3dd0","final_host":"www.msn.com","final_path":"/","ms":662,"identity":"Mobile browser","html":true},"gptbot":{"status":200,"content_type":"text/html","bytes":49016,"sha256":"532da7b0382995a633cb2576ddbcdc1b2cdd533a6a0eec66ec1333314fd0cf76","final_host":"www.msn.com","final_path":"/","ms":186,"identity":"GPTBot","html":true},"claudebot":{"status":200,"content_type":"text/html","bytes":48918,"sha256":"a5f6ce67e10af36de819a8b85928c01debad7746ac6b193d6df46a4c34a18dbc","final_host":"www.msn.com","final_path":"/","ms":51,"identity":"ClaudeBot","html":true},"oaisearch":{"status":200,"content_type":"text/html","bytes":44802,"sha256":"1dfead53703ef5216f849a942afa52d990e08e15efa9d63e4e9a706709a7a57d","final_host":"www.msn.com","final_path":"/","ms":52,"identity":"OAI-SearchBot","html":true},"perplexity":{"status":200,"content_type":"text/html","bytes":49023,"sha256":"eb5a33a79a24a07006da0c95bdf0360be2468b14819ffbc5e4fd6b6ec1247e6e","final_host":"www.msn.com","final_path":"/","ms":56,"identity":"PerplexityBot","html":true}},"robots":{"status":200,"content_type":"text/plain","bytes":5245,"sha256":"aefbffa0a0a083e30637972046b42afc1569f7813c5687739a5ecdee0a6be4dd","final_host":"www.msn.com","final_path":"/robots.txt","ms":192,"html":false,"text":"User-agent: *\r\nDisallow: /*/health/search/filter\r\nDisallow: /spartan\r\nDisallow: /pt-ao\r\nDisallow: /*preview=\r\nDisallow: /*/autos/marketplace/product/\r\nDisallow: /*/cars/marketplace/product/\r\nDisallow: /*?item=\r\nDisallow: /*&item=\r\nDisallow: /codeCoverage\r\nDisallow: /vpBaselines\r\nDisallow: /msnBaselines\r\nDisallow: /*/studio\r\nDisallow: /*/indexnowdelete/\r\nAllow: /*/shopping/*/ar-\r\nAllow: /*/shopping/*/ss-\r\nAllow: /*/shopping/*/vi-\r\nDisallow: /*/shopping/\r\n\r\nUser-agent: AdsBot-Google\r\nAllow: /\r\nDisallow: /*/health/search/filter\r\n\r\nUser-agent: Verity\r\nCrawl-delay: 3\r\nDisallow: /*/health/search/filter\r\nDisallow: /spartan\r\nDisallow: /pt-ao\r\nDisallow: /*preview=\r\nDisallow: /*/autos/marketplace/product/\r\nDisallow: /*/cars/marketplace/product/\r\nDisallow: /*?item=\r\nDisallow: /*&item=\r\nDisallow: /codeCoverage\r\nDisallow: /vpBaselines\r\nDisallow: /msnBaselines\r\nDisallow: /*/studio\r\nDisallow: /*/indexnowdelete/\r\nAllow: /*/shopping/*/ar-\r\nAllow: /*/shopping/*/ss-\r\nAllow: /*/shopping/*/vi-\r\nDisallow: /*/shopping/\r\n\r\nUser-agent: ias_crawler\r\nCrawl-delay: 0.1\r\nDisallow: /*/health/search/filter\r\nDisallow: /spartan\r\nDisallow: /pt-ao\r\nDisallow: /*preview=\r\nDisallow: /*/autos/marketplace/product/\r\nDisallow: /*/cars/marketplace/product/\r\nDisallow: /*?item=\r\nDisallow: /*&item=\r\nDisallow: /codeCoverage\r\nDisallow: /vpBaselines\r\nDisallow: /msnBaselines\r\nDisallow: /*/studio\r\nDisallow: /*/indexnowdelete/\r\nAllow: /*/shopping/*/ar-\r\nAllow: /*/shopping/*/ss-\r\nAllow: /*/shopping/*/vi-\r\nDisallow: /*/shopping/\r\n\r\nUser-agent: Mediapartners-Google\r\nCrawl-delay: 0.5\r\nDisallow: /*/health/search/filter\r\nDisallow: /spartan\r\nDisallow: /pt-ao\r\nDisallow: /*preview=\r\nDisallow: /*/autos/marketplace/product/\r\nDisallow: /*/cars/marketplace/product/\r\nDisallow: /*?item=\r\nDisallow: /*&item=\r\nDisallow: /codeCoverage\r\nDisallow: /vpBaselines\r\nDisallow: /msnBaselines\r\nDisallow: /*/studio\r\nDisallow: /*/indexnowdelete/\r\nAllow: /*/shopping/*/ar-\r\nAllow: /*/shopping/*/ss-\r\nAllow: /*/shopping/*/vi-\r\nDisallow: /*/shopping/\r\n\r\nUser-agent: CriteoBot\r\nCrawl-delay: 0.5\r\nDisallow: /*/health/search/filter\r\nDisallow: /spartan\r\nDisallow: /pt-ao\r\nDisallow: /*preview=\r\nDisallow: /*/autos/marketplace/product/\r\nDisallow: /*/cars/marketplace/product/\r\nDisallow: /*?item=\r\nDisallow: /*&item=\r\nDisallow: /codeCoverage\r\nDisallow: /vpBaselines\r\nDisallow: /msnBaselines\r\nDisallow: /*/studio\r\nDisallow: /*/indexnowdelete/\r\nAllow: /*/shopping/*/ar-\r\nAllow: /*/shopping/*/ss-\r\nAllow: /*/shopping/*/vi-\r\nDisallow: /*/shopping/\r\n\r\nUser-agent: yandex.com/bot\r\nCrawl-delay: 1\r\nDisallow: /*/weather/\r\nDisallow: /*/clima/\r\nDisallow: /*/cuaca/\r\nDisallow: /*/eltiempo/\r\nDisallow: /*/el-tiempo/\r\nDisallow: /*/havadurumu/\r\nDisallow: /*/idojaras/\r\nDisallow: /*/meteo/\r\nDisallow: /*/meteorologia/\r\nDisallow: /*/pocasi/\r\nDisallow: /*/pogoda/\r\nDisallow: /*/saa/\r\nDisallow: /*/vader/\r\nDisallow: /*/vejr/\r\nDisallow: /*/weer/\r\nDisallow: /*/wetter/\r\nAllow: /*/shopping/*/ar-\r\nAllow: /*/shopping/*/ss-\r\nAllow: /*/shopping/*/vi-\r\nDisallow: /*/shopping/\r\n\r\nUser-agent: admantx-ussy04/3.2\r\nAllow: /\r\n\r\nUser-agent: TTD-Content\r\nAllow: /\r\n\r\nUser-agent: AmazonAdBot\r\nAllow: /\r\n\r\nUser-agent: AI2Bot\r\nUser-agent: Ai2Bot-Dolma\r\nUser-agent: Amazonbot\r\nUser-agent: anthropic-ai\r\nUser-agent: Applebot\r\nUser-agent: Applebot-Extended\r\nUser-agent: Bedrockbot\r\nUser-agent: Bytespider\r\nUser-agent: CCBot\r\nUser-agent: ChatGLM\r\nUser-agent: ChatGPT-User\r\nUser-agent: ClaudeBot\r\nUser-agent: Claude-User\r\nUser-agent: Claude-SearchBot \r\nUser-agent: Claude-Web\r\nUser-agent: cohere-ai\r\nUser-agent: cohere-training-data-crawler\r\nUser-agent: Cotoyogi\r\nUser-agent: DeepSeekBot\r\nUser-agent: Diffbot\r\nUser-agent: DuckAssistBot\r\nUser-agent: FacebookBot\r\nUser-agent: FriendlyCrawler\r\nUser-agent: Gensparkbot \r\nUser-agent: Google-CloudVertexBot\r\nUser-agent: Google-Extended\r\nUser-agent: GoogleOther\r\nUser-agent: GoogleOther-Image\r\nUser-agent: GoogleOther-Video\r\nUser-agent: GPTBot\r\nUser-agent: Grok\r\nUser-agent: iaskspider/2.0\r\nUser-agent: ICC-Crawler\r\nUser-agent: ImagesiftBot\r\nUser-agent: Img2dataset\r\nUser-agent: ISSCyberRiskCrawler\r\nUser-agent: Kangaroo Bot\r\nUser-agent: KunatoCrawler\r\nUser-agent: magpie-crawler\r\nUser-agent: Meltwater\r\nUser-agent: Meta-ExternalAgent\r\nUser-agent: Meta-ExternalFetcher\r\nUser-agent: MistralAI\r\nUser-agent: MistralAI-User\r\nUser-agent: Momentumbot\r\nUser-agent: OAI-SearchBot\r\nUser-agent: omgili\r\nUser-agent: Omgilibot\r\nUser-agent: Other\r\nUser-agent: PanguBot\r\nUser-agent: Perplexity-ai\r\nUser-agent: PerplexityBot\r\nUser-agent: Perplexity-User\r\nUser-agent: QualifiedBot\r\nUser-agent: Scrapy\r\nUser-agent: Seekr\r\nUser-agent: Sidetrade indexer bot\r\nUser-agent: TikTokSpider\r\nUser-agent: Timpibot\r\nUser-agent: Twitterbot\r\nUser-agent: VelenPublicWebCrawler\r\nUser-agent: WARDBot\r\nUser-agent: Webzio-Extended\r\nUser-agent: WRTNBot\r\nUser-agent: YouBot\r\nDisallow: /\r\n\r\nSitemap: https://www.msn.com/sitemaps/casualgames/sitemap-index.xml\r\nSitemap: https://www.msn.com/sitemaps/consumption/index.xml\r\nSitemap: https://www.msn.com/sitemaps/health/health-sitemap-index.xml\r\nSitemap: https://www.msn.com/staticsb/statics/latest/0/casualgames/sitemaps/sitemap-index.xml\r\nSitemap: https://www.msn.com/staticsb/statics/latest/0/finance/sitemaps/sitemap-index.xml\r\nSitemap: https://www.msn.com/sitemaps/weather/sitemap-index.xml","agents":{"GPTBot":false,"OAI-SearchBot":false,"ChatGPT-User":false,"ClaudeBot":false,"Claude-SearchBot":false,"PerplexityBot":false,"Google-Extended":false,"Applebot-Extended":false,"CCBot":false,"Bytespider":false,"Meta-ExternalAgent":false,"Amazonbot":false}},"sitemap":{"status":200,"content_type":"text/xml","bytes":5402,"sha256":"6357cf8b4653e26a07eca3573c30413dbf7e64d2e58f3b9ee45505f3f96607c8","final_host":"www.msn.com","final_path":"/sitemaps/casualgames/sitemap-index.xml","ms":24,"url":"https://www.msn.com/sitemaps/casualgames/sitemap-index.xml","declared":true,"xml":true},"files":{"llms_txt":{"status":404,"content_type":"text/html","bytes":19143,"sha256":"10fcf682b17971154f37ba43feed59f3f054c12d46e8d9dab193696175e960ff","final_host":"www.msn.com","final_path":"/llms.txt","ms":63,"path":"/llms.txt","html":true,"json":null,"present":false},"agents_md":{"status":404,"content_type":"text/html","bytes":19143,"sha256":"10fcf682b17971154f37ba43feed59f3f054c12d46e8d9dab193696175e960ff","final_host":"www.msn.com","final_path":"/agents.md","ms":62,"path":"/agents.md","html":true,"json":null,"present":false},"agent_skills":{"status":404,"content_type":"text/html","bytes":19143,"sha256":"10fcf682b17971154f37ba43feed59f3f054c12d46e8d9dab193696175e960ff","final_host":"www.msn.com","final_path":"/.well-known/agent-skills/index.json","ms":59,"path":"/.well-known/agent-skills/index.json","html":true,"json":null,"present":false},"mcp_card":{"status":404,"content_type":"text/html","bytes":19143,"sha256":"10fcf682b17971154f37ba43feed59f3f054c12d46e8d9dab193696175e960ff","final_host":"www.msn.com","final_path":"/.well-known/mcp/server-card.json","ms":60,"path":"/.well-known/mcp/server-card.json","html":true,"json":null,"present":false},"api_catalog":{"status":404,"content_type":"text/html","bytes":19143,"sha256":"10fcf682b17971154f37ba43feed59f3f054c12d46e8d9dab193696175e960ff","final_host":"www.msn.com","final_path":"/.well-known/api-catalog","ms":63,"path":"/.well-known/api-catalog","html":true,"json":null,"present":false},"a2a_card":{"status":404,"content_type":"text/html","bytes":19143,"sha256":"10fcf682b17971154f37ba43feed59f3f054c12d46e8d9dab193696175e960ff","final_host":"www.msn.com","final_path":"/.well-known/agent-card.json","ms":68,"path":"/.well-known/agent-card.json","html":true,"json":null,"present":false}},"rings":{"v":1,"candidates":[],"mcp":[],"a2a":null,"webmcp":{"model_context":false,"bridge":false,"mcp_json":false,"link_rel":false,"any":false},"handshakes":0,"get_path":{"status":404,"content_type":"text/html","ms":57,"error":null},"get_host":{"status":404,"content_type":"text/html","ms":257,"error":null}},"sf":[4,0,0,0,0,0,0,0,1],"derived":{"reachable":true,"home_class":"answers","robots_served":true,"robots_sitemap":true,"content_signal":false,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":["GPTBot","OAI-SearchBot","ChatGPT-User","ClaudeBot","Claude-SearchBot","PerplexityBot","Google-Extended","Applebot-Extended","CCBot","Bytespider","Meta-ExternalAgent","Amazonbot"],"blocks_any_ai":true,"files":{"llms_txt":false,"agents_md":false,"agent_skills":false,"mcp_card":false,"api_catalog":false,"a2a_card":false},"any_capability":false,"ai_refused":[],"ai_thin":["gptbot","claudebot","oaisearch","perplexity"],"aipref":0,"aipref_measured":true,"sf":[4,0,0,0,0,0,0,0,1],"rings":{"mcp_declared":false,"mcp_live":false,"mcp_auth":false,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":3,"fh":"www.msn.com","fp_other_host":[]},"derive_v":5,"frame":"top10k","shard":6},{"v":1,"kind":"crawlcheck-census-record","run":18,"domain":"zoom.us","rank":72,"measured_at":"2026-11-13T18:55:41.951Z","home":{"browser":{"status":200,"content_type":"text/html","bytes":321055,"sha256":"38ed90965e8f5be1ac2f3ec88d1a057ecd95a147540d7588290409b447e41c19","final_host":"www.zoom.com","final_path":"/","ms":116,"identity":"Mobile browser","html":true,"fp":"b1bead5010e26a2d"},"gptbot":{"status":200,"content_type":"text/html","bytes":321055,"sha256":"0658500a350d71d0acc213560db614bdd7e644c1e6341b36d48c70091d11d608","final_host":"www.zoom.com","final_path":"/","ms":105,"identity":"GPTBot","html":true,"fp":"b1bead5010e26a2d"},"claudebot":{"status":200,"content_type":"text/html","bytes":321055,"sha256":"3b28ca7c33c28c9052685ed8e115e9466f6faeee82b4a000e9b52eee42e41aa3","final_host":"www.zoom.com","final_path":"/","ms":101,"identity":"ClaudeBot","html":true,"fp":"b1bead5010e26a2d"},"oaisearch":{"status":200,"content_type":"text/html","bytes":321055,"sha256":"6d1a47ad9068a8ae0eed3eeb2f8b0a00532ad5fa98c2207ed18913d6f637efc4","final_host":"www.zoom.com","final_path":"/","ms":25,"identity":"OAI-SearchBot","html":true,"fp":"b1bead5010e26a2d"},"perplexity":{"status":200,"content_type":"text/html","bytes":321055,"sha256":"82aa883852bbfe097ccadae3ea00a73ce7328e27ef005b1f709bf97a07d01621","final_host":"www.zoom.com","final_path":"/","ms":27,"identity":"PerplexityBot","html":true,"fp":"b1bead5010e26a2d"}},"robots":{"status":200,"content_type":"text/plain","bytes":2005,"sha256":"b3ed9f53f38c480b046b4d5a648aa7a879c6720957301506dc282b32fb2f5593","final_host":"zoom.us","final_path":"/robots.txt","ms":180,"html":false,"text":"User-Agent: *\nDisallow: /download/*/Zoom_launcher.exe\nDisallow: /download/*/zoomusLauncher.zip\nDisallow: /docs/image/new/brand/outdated/\nDisallow: /docs/doc/2-Page-All-Products.pdf\nDisallow: /docs/doc/zoom-apps-security-privacy-faq.pdf\nDisallow: /docs/doc/zoom-apps-data-management.pdf\nDisallow: /wc/*\nDisallow: /web/*\nDisallow: /share*\nDisallow: /support*\nDisallow: /j/*\nDisallow: /z/*\nDisallow: /s/*\nDisallow: /u/*\nDisallow: /w/*\nDisallow: /e/*\nDisallow: /account*\nDisallow: /start*\nDisallow: /postattendee*\nDisallow: /rec/*\nDisallow: /register*\nDisallow: /webinar/register/*\nDisallow: /webinar/registerevent/*\nDisallow: /meeting/register/*\nDisallow: /*oauth_signin\nDisallow: /my/*\nDisallow: /download2*\nDisallow: /*?\n\nUser-agent: Googlebot\nAllow: /join\n\nUser-agent: LinkedInBot\nAllow: /webinar/register/*\nAllow: /webinar/registerevent/*\nAllow: /meeting/register/*\n\nUser-agent: facebookexternalhit\nAllow: /webinar/register/*\nAllow: /webinar/registerevent/*\nAllow: /meeting/register/*\n\nUser-agent: Twitterbot\nAllow: /webinar/register/*\nAllow: /webinar/registerevent/*\nAllow: /meeting/register/*\n\nUser-agent: PinterestBot\nAllow: /webinar/register/*\nAllow: /webinar/registerevent/*\nAllow: /meeting/register/*\n\nUser-agent: InstagramBot\nAllow: /webinar/register/*\nAllow: /webinar/registerevent/*\nAllow: /meeting/register/*\n\nUser-agent: Slackbot\nAllow: /webinar/register/*\nAllow: /webinar/registerevent/*\nAllow: /meeting/register/*\n\nUser-agent: WhatsApp\nAllow: /webinar/register/*\nAllow: /webinar/registerevent/*\nAllow: /meeting/register/*\n\nUser-agent: redditbot\nAllow: /webinar/register/*\nAllow: /webinar/registerevent/*\nAllow: /meeting/register/*\n\nUser-agent: Tumblr\nAllow: /webinar/register/*\nAllow: /webinar/registerevent/*\nAllow: /meeting/register/*\n\nUser-agent: TikTokBot\nAllow: /webinar/register/*\nAllow: /webinar/registerevent/*\nAllow: /meeting/register/*\n\nUser-agent: Snapchat\nAllow: /webinar/register/*\nAllow: /webinar/registerevent/*\nAllow: /meeting/register/*\n\nSitemap: https://zoom.us/sitemap.xml","agents":{"GPTBot":true,"OAI-SearchBot":true,"ChatGPT-User":true,"ClaudeBot":true,"Claude-SearchBot":true,"PerplexityBot":true,"Google-Extended":true,"Applebot-Extended":true,"CCBot":true,"Bytespider":true,"Meta-ExternalAgent":true,"Amazonbot":true}},"sitemap":{"status":200,"content_type":"text/xml","bytes":432921,"sha256":"fc46be69d0c7b7529bb15d13a177daf4e337d6ec8f0b3e0875e0706c138489b8","final_host":"zoom.us","final_path":"/sitemap.xml","ms":288,"url":"https://zoom.us/sitemap.xml","declared":true,"xml":true},"files":{"llms_txt":{"status":404,"content_type":"text/html","bytes":526103,"sha256":"602d381117bc7ab2dc666879302f1f06824d33c6e92d96fbf79c0cc09dd2458f","final_host":"zoom.us","final_path":"/llms.txt","ms":232,"path":"/llms.txt","html":true,"json":null,"present":false},"agents_md":{"status":404,"content_type":"text/html","bytes":526103,"sha256":"298544ff6f83550dd334460fd0ed2c16e821e4f065ecbee412edcbd1af3bcab8","final_host":"zoom.us","final_path":"/agents.md","ms":690,"path":"/agents.md","html":true,"json":null,"present":false},"agent_skills":{"status":200,"content_type":"application/json","bytes":28503,"sha256":"5f91ddfd957cda871aa39b74ea88144863f150eb7deb6a81129ef8c68315349a","final_host":"zoom.us","final_path":"/.well-known/agent-skills/index.json","ms":288,"path":"/.well-known/agent-skills/index.json","html":false,"json":true,"present":true},"mcp_card":{"status":200,"content_type":"application/json","bytes":10008,"sha256":"558897a06ccbaeec16bd6b82dbb87c046b56786987d5c9d95ff709b41df5fe20","final_host":"zoom.us","final_path":"/.well-known/mcp/server-card.json","ms":439,"path":"/.well-known/mcp/server-card.json","html":false,"json":true,"present":true},"api_catalog":{"status":200,"content_type":"application/linkset+json","bytes":34738,"sha256":"8fcefa04c48ae2f36da3b608b8726456e8a032255dff83e36fb3fb4129bad22f","final_host":"zoom.us","final_path":"/.well-known/api-catalog","ms":293,"path":"/.well-known/api-catalog","html":false,"json":true,"present":true},"a2a_card":{"status":404,"content_type":"text/html","bytes":526103,"sha256":"8434834efa34234d9bfbdeae46b5170dfa8fe36b0a6bc57cbcff9b828fb5ce15","final_host":"zoom.us","final_path":"/.well-known/agent-card.json","ms":251,"path":"/.well-known/agent-card.json","html":true,"json":null,"present":false}},"rings":{"v":1,"candidates":[{"url":"https://mcp.zoom.us/mcp/zoom/streamable","source":"mcp_card"},{"url":"https://mcp.zoom.us/mcp/productivity/streamable","source":"mcp_card"},{"url":"https://mcp.zoom.us/mcp/canvas/streamable","source":"mcp_card"},{"url":"https://mcp.zoom.us/mcp/tasks/streamable","source":"mcp_card"},{"url":"https://mcp.zoom.us/mcp/workflow/streamable","source":"mcp_card"},{"url":"https://mcp.zoom.us/mcp/whiteboard/streamable","source":"mcp_card"}],"mcp":[{"url":"https://mcp.zoom.us/mcp/zoom/streamable","source":"mcp_card","outcome":"auth_required","status":401,"ms":179,"initialized":false,"protocol_version":null,"server":null,"tools":null,"tools_sha256":null,"auth_header":true},{"url":"https://mcp.zoom.us/mcp/productivity/streamable","source":"mcp_card","outcome":"auth_required","status":401,"ms":182,"initialized":false,"protocol_version":null,"server":null,"tools":null,"tools_sha256":null,"auth_header":true},{"url":"https://mcp.zoom.us/mcp/canvas/streamable","source":"mcp_card","outcome":"auth_required","status":401,"ms":716,"initialized":false,"protocol_version":null,"server":null,"tools":null,"tools_sha256":null,"auth_header":true}],"a2a":null,"webmcp":{"model_context":false,"bridge":false,"mcp_json":false,"link_rel":false,"any":false},"handshakes":3,"get_path":{"status":404,"content_type":"text/html","ms":232,"error":null},"get_host":{"status":404,"content_type":"text/html","ms":427,"error":null}},"sf":[1,0,0,0,0,0,3,0,0],"derived":{"reachable":true,"home_class":"answers","robots_served":true,"robots_sitemap":true,"content_signal":false,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":[],"blocks_any_ai":false,"files":{"llms_txt":false,"agents_md":false,"agent_skills":true,"mcp_card":true,"api_catalog":true,"a2a_card":false},"any_capability":true,"ai_refused":[],"ai_thin":[],"aipref":0,"aipref_measured":true,"sf":[1,0,0,0,0,0,3,0,0],"rings":{"mcp_declared":true,"mcp_live":false,"mcp_auth":true,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":3,"fh":"www.zoom.com","fp_other_host":[],"fp":"b1bead5010e26a2d","fp_bits":{"gptbot":0,"claudebot":0,"oaisearch":0,"perplexity":0}},"derive_v":5,"frame":"top10k","shard":7},{"v":1,"kind":"crawlcheck-census-record","run":18,"domain":"ntp.org","rank":73,"measured_at":"2026-11-13T18:46:27.088Z","home":{"browser":{"status":200,"content_type":"text/html","bytes":9114,"sha256":"9812869b85080fba2b26749348b535e391dfdc861753788aefc52b86262e3c41","final_host":"www.ntp.org","final_path":"/","ms":2121,"identity":"Mobile browser","html":true,"fp":"4d02f33496f546f0"},"gptbot":{"status":200,"content_type":"text/html","bytes":9114,"sha256":"9812869b85080fba2b26749348b535e391dfdc861753788aefc52b86262e3c41","final_host":"www.ntp.org","final_path":"/","ms":2096,"identity":"GPTBot","html":true,"fp":"4d02f33496f546f0"},"claudebot":{"status":200,"content_type":"text/html","bytes":9114,"sha256":"9812869b85080fba2b26749348b535e391dfdc861753788aefc52b86262e3c41","final_host":"www.ntp.org","final_path":"/","ms":2179,"identity":"ClaudeBot","html":true,"fp":"4d02f33496f546f0"},"oaisearch":{"status":200,"content_type":"text/html","bytes":9114,"sha256":"9812869b85080fba2b26749348b535e391dfdc861753788aefc52b86262e3c41","final_host":"www.ntp.org","final_path":"/","ms":2196,"identity":"OAI-SearchBot","html":true,"fp":"4d02f33496f546f0"},"perplexity":{"status":200,"content_type":"text/html","bytes":9114,"sha256":"9812869b85080fba2b26749348b535e391dfdc861753788aefc52b86262e3c41","final_host":"www.ntp.org","final_path":"/","ms":2182,"identity":"PerplexityBot","html":true,"fp":"4d02f33496f546f0"}},"robots":{"status":200,"content_type":"text/plain","bytes":16,"sha256":"5a5a4434f108ef034c256b9d3e169e516fb806aa6f6353e53eff9655732fbb79","final_host":"www.ntp.org","final_path":"/robots.txt","ms":2067,"html":false,"text":"\nUser-agent: *\n\n","agents":{"GPTBot":true,"OAI-SearchBot":true,"ChatGPT-User":true,"ClaudeBot":true,"Claude-SearchBot":true,"PerplexityBot":true,"Google-Extended":true,"Applebot-Extended":true,"CCBot":true,"Bytespider":true,"Meta-ExternalAgent":true,"Amazonbot":true}},"sitemap":{"status":200,"content_type":"application/xml","bytes":420,"sha256":"2e06093ff2fdc0a20aab23a1b62a673884891c2b7db081ac3846c375598cf4a9","final_host":"www.ntp.org","final_path":"/sitemap.xml","ms":535,"url":"https://ntp.org/sitemap.xml","declared":false,"xml":true},"files":{"llms_txt":{"status":404,"content_type":"text/html","bytes":6601,"sha256":"ce78ad119af14e88e0bd97553a54a70a036626840ba00e22b9aebc2022165394","final_host":"www.ntp.org","final_path":"/llms.txt","ms":329,"path":"/llms.txt","html":true,"json":null,"present":false},"agents_md":{"status":404,"content_type":"text/html","bytes":6601,"sha256":"ce78ad119af14e88e0bd97553a54a70a036626840ba00e22b9aebc2022165394","final_host":"www.ntp.org","final_path":"/agents.md","ms":443,"path":"/agents.md","html":true,"json":null,"present":false},"agent_skills":{"status":404,"content_type":"text/html","bytes":6601,"sha256":"ce78ad119af14e88e0bd97553a54a70a036626840ba00e22b9aebc2022165394","final_host":"www.ntp.org","final_path":"/.well-known/agent-skills/index.json","ms":858,"path":"/.well-known/agent-skills/index.json","html":true,"json":null,"present":false},"mcp_card":{"status":404,"content_type":"text/html","bytes":6601,"sha256":"ce78ad119af14e88e0bd97553a54a70a036626840ba00e22b9aebc2022165394","final_host":"www.ntp.org","final_path":"/.well-known/mcp/server-card.json","ms":451,"path":"/.well-known/mcp/server-card.json","html":true,"json":null,"present":false},"api_catalog":{"status":404,"content_type":"text/html","bytes":6601,"sha256":"ce78ad119af14e88e0bd97553a54a70a036626840ba00e22b9aebc2022165394","final_host":"www.ntp.org","final_path":"/.well-known/api-catalog","ms":440,"path":"/.well-known/api-catalog","html":true,"json":null,"present":false},"a2a_card":{"status":404,"content_type":"text/html","bytes":6601,"sha256":"ce78ad119af14e88e0bd97553a54a70a036626840ba00e22b9aebc2022165394","final_host":"www.ntp.org","final_path":"/.well-known/agent-card.json","ms":949,"path":"/.well-known/agent-card.json","html":true,"json":null,"present":false}},"rings":{"v":1,"candidates":[],"mcp":[],"a2a":null,"webmcp":{"model_context":false,"bridge":false,"mcp_json":false,"link_rel":false,"any":false},"handshakes":0,"get_path":{"status":404,"content_type":"text/html","ms":526,"error":null},"get_host":{"status":530,"content_type":"text/plain","ms":86,"error":null}},"sf":[-1,0,0,0,0,0,3,0,0],"derived":{"reachable":true,"home_class":"answers","robots_served":true,"robots_sitemap":false,"content_signal":false,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":[],"blocks_any_ai":false,"files":{"llms_txt":false,"agents_md":false,"agent_skills":false,"mcp_card":false,"api_catalog":false,"a2a_card":false},"any_capability":false,"ai_refused":[],"ai_thin":[],"aipref":0,"aipref_measured":true,"sf":[-1,0,0,0,0,0,3,0,0],"rings":{"mcp_declared":false,"mcp_live":false,"mcp_auth":false,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":3,"fh":"www.ntp.org","fp_other_host":[],"fp":"4d02f33496f546f0","fp_bits":{"gptbot":0,"claudebot":0,"oaisearch":0,"perplexity":0}},"derive_v":5,"frame":"top10k","shard":0},{"v":1,"kind":"crawlcheck-census-record","run":18,"domain":"qq.com","rank":74,"measured_at":"2026-11-13T18:48:18.168Z","home":{"browser":{"status":200,"content_type":"text/html","bytes":131352,"sha256":"0f4c677c3396c013b974e7bd1ef8ce3c38d05bf25269e41e2529aca7d2ac3eb1","final_host":"www.qq.com","final_path":"/","ms":1393,"identity":"Mobile browser","html":true,"fp":"839be47633272ae2"},"gptbot":{"status":200,"content_type":"text/html","bytes":131344,"sha256":"aed3dec1c716013351916206605fb52ac6f258673c3fa97f4dd06c6fc39a69fb","final_host":"www.qq.com","final_path":"/","ms":1364,"identity":"GPTBot","html":true,"fp":"839be47633272ae2"},"claudebot":{"status":200,"content_type":"text/html","bytes":131344,"sha256":"2d9d440b7b8bf724e1993026583d9ed0b385620fb7977144103ced7df016b425","final_host":"www.qq.com","final_path":"/","ms":763,"identity":"ClaudeBot","html":true,"fp":"839be47633272ae2"},"oaisearch":{"status":200,"content_type":"text/html","bytes":131353,"sha256":"07bd4ae2fe7e8652ce728176fc2ac640153b0d90855463c39967a19418881367","final_host":"www.qq.com","final_path":"/","ms":1570,"identity":"OAI-SearchBot","html":true,"fp":"839be47633272ae2"},"perplexity":{"status":200,"content_type":"text/html","bytes":131353,"sha256":"073ba386e94c27563259af6bf6c7b1695015f8ab19f6569740a70dd99c8982dd","final_host":"www.qq.com","final_path":"/","ms":772,"identity":"PerplexityBot","html":true,"fp":"839be47633272ae2"}},"robots":{"status":200,"content_type":"text/plain","bytes":71,"sha256":"1f5d4ddc6513b31998acadbb56fe02e8b9eb244a3eb64d36f446f146e0bc3d5f","final_host":"www.qq.com","final_path":"/robots.txt","ms":932,"html":false,"text":"User-agent: *\nDisallow: \nSitemap: https://news.qq.com/sitemap/index.xml","agents":{"GPTBot":true,"OAI-SearchBot":true,"ChatGPT-User":true,"ClaudeBot":true,"Claude-SearchBot":true,"PerplexityBot":true,"Google-Extended":true,"Applebot-Extended":true,"CCBot":true,"Bytespider":true,"Meta-ExternalAgent":true,"Amazonbot":true}},"sitemap":{"status":200,"content_type":"application/xml","bytes":105125,"sha256":"215b602fc2a27bba72420f8831fc1a6a15b9f09684fe8ca784fb2666fc287f6e","final_host":"news.qq.com","final_path":"/sitemap/index.xml","ms":613,"url":"https://news.qq.com/sitemap/index.xml","declared":true,"xml":true},"files":{"llms_txt":{"status":200,"content_type":"text/html","bytes":4183,"sha256":"dd3ca8f2a7ee2f282645a07d06786a1fdb6f0caa1e090063805dc51f525c2464","final_host":"h5.news.qq.com","final_path":"/static/babyhome.htm","ms":904,"path":"/llms.txt","html":true,"json":null,"present":false},"agents_md":{"status":200,"content_type":"text/html","bytes":4183,"sha256":"dd3ca8f2a7ee2f282645a07d06786a1fdb6f0caa1e090063805dc51f525c2464","final_host":"h5.news.qq.com","final_path":"/static/babyhome.htm","ms":901,"path":"/agents.md","html":true,"json":null,"present":false},"agent_skills":{"status":200,"content_type":"text/html","bytes":4183,"sha256":"dd3ca8f2a7ee2f282645a07d06786a1fdb6f0caa1e090063805dc51f525c2464","final_host":"h5.news.qq.com","final_path":"/static/babyhome.htm","ms":890,"path":"/.well-known/agent-skills/index.json","html":true,"json":null,"present":false},"mcp_card":{"status":200,"content_type":"text/html","bytes":4183,"sha256":"dd3ca8f2a7ee2f282645a07d06786a1fdb6f0caa1e090063805dc51f525c2464","final_host":"h5.news.qq.com","final_path":"/static/babyhome.htm","ms":892,"path":"/.well-known/mcp/server-card.json","html":true,"json":null,"present":false},"api_catalog":{"status":200,"content_type":"text/html","bytes":4183,"sha256":"dd3ca8f2a7ee2f282645a07d06786a1fdb6f0caa1e090063805dc51f525c2464","final_host":"h5.news.qq.com","final_path":"/static/babyhome.htm","ms":458,"path":"/.well-known/api-catalog","html":true,"json":null,"present":false},"a2a_card":{"status":200,"content_type":"text/html","bytes":4183,"sha256":"dd3ca8f2a7ee2f282645a07d06786a1fdb6f0caa1e090063805dc51f525c2464","final_host":"h5.news.qq.com","final_path":"/static/babyhome.htm","ms":170,"path":"/.well-known/agent-card.json","html":true,"json":null,"present":false}},"rings":{"v":1,"candidates":[],"mcp":[],"a2a":null,"webmcp":{"model_context":false,"bridge":false,"mcp_json":false,"link_rel":false,"any":false},"handshakes":0,"get_path":{"status":200,"content_type":"text/html","ms":395,"error":null},"get_host":{"status":530,"content_type":"text/plain","ms":306,"error":null}},"sf":[4,0,0,0,0,0,1,0,0],"derived":{"reachable":true,"home_class":"answers","robots_served":true,"robots_sitemap":true,"content_signal":false,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":[],"blocks_any_ai":false,"files":{"llms_txt":false,"agents_md":false,"agent_skills":false,"mcp_card":false,"api_catalog":false,"a2a_card":false},"any_capability":false,"ai_refused":[],"ai_thin":[],"aipref":0,"aipref_measured":true,"sf":[4,0,0,0,0,0,1,0,0],"rings":{"mcp_declared":false,"mcp_live":false,"mcp_auth":false,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":3,"fh":"www.qq.com","fp_other_host":[],"fp":"839be47633272ae2","fp_bits":{"gptbot":0,"claudebot":0,"oaisearch":0,"perplexity":0}},"derive_v":5,"frame":"top10k","shard":1},{"v":1,"kind":"crawlcheck-census-record","run":18,"domain":"root-servers.net","rank":75,"measured_at":"2026-11-13T18:49:51.543Z","home":{"browser":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"root-servers.net","final_path":"/","ms":26,"identity":"Mobile browser","html":false},"gptbot":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"root-servers.net","final_path":"/","ms":1,"identity":"GPTBot","html":false},"claudebot":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"root-servers.net","final_path":"/","ms":1,"identity":"ClaudeBot","html":false},"oaisearch":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"root-servers.net","final_path":"/","ms":1,"identity":"OAI-SearchBot","html":false},"perplexity":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"root-servers.net","final_path":"/","ms":1,"identity":"PerplexityBot","html":false}},"robots":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"root-servers.net","final_path":"/robots.txt","ms":2,"html":false,"text":null,"agents":null},"sitemap":null,"files":null,"derived":{"reachable":false,"home_class":"no_site","robots_served":false,"robots_sitemap":false,"content_signal":false,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":[],"blocks_any_ai":false,"files":{"llms_txt":false,"agents_md":false,"agent_skills":false,"mcp_card":false,"api_catalog":false,"a2a_card":false},"any_capability":false,"ai_refused":[],"ai_thin":[],"rings":{"mcp_declared":false,"mcp_live":false,"mcp_auth":false,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":3,"fh":"root-servers.net","fp_other_host":[]},"derive_v":5,"frame":"top10k","shard":2},{"v":1,"kind":"crawlcheck-census-record","run":18,"domain":"nginx.org","rank":76,"measured_at":"2026-11-13T18:50:46.493Z","home":{"browser":{"status":200,"content_type":"text/html","bytes":6503,"sha256":"c89c77b64bc4de01e4ecb86b904770574e58744f287d9332c5637351b475b6d3","final_host":"nginx.org","final_path":"/","ms":491,"identity":"Mobile browser","html":true,"fp":"d5a07d05d1501da2"},"gptbot":{"status":200,"content_type":"text/html","bytes":6503,"sha256":"c89c77b64bc4de01e4ecb86b904770574e58744f287d9332c5637351b475b6d3","final_host":"nginx.org","final_path":"/","ms":449,"identity":"GPTBot","html":true,"fp":"d5a07d05d1501da2"},"claudebot":{"status":200,"content_type":"text/html","bytes":6503,"sha256":"c89c77b64bc4de01e4ecb86b904770574e58744f287d9332c5637351b475b6d3","final_host":"nginx.org","final_path":"/","ms":379,"identity":"ClaudeBot","html":true,"fp":"d5a07d05d1501da2"},"oaisearch":{"status":200,"content_type":"text/html","bytes":6503,"sha256":"c89c77b64bc4de01e4ecb86b904770574e58744f287d9332c5637351b475b6d3","final_host":"nginx.org","final_path":"/","ms":284,"identity":"OAI-SearchBot","html":true,"fp":"d5a07d05d1501da2"},"perplexity":{"status":200,"content_type":"text/html","bytes":6503,"sha256":"c89c77b64bc4de01e4ecb86b904770574e58744f287d9332c5637351b475b6d3","final_host":"nginx.org","final_path":"/","ms":59,"identity":"PerplexityBot","html":true,"fp":"d5a07d05d1501da2"}},"robots":{"status":200,"content_type":"text/plain","bytes":72,"sha256":"8f5b42f5d23ca623fa42f917a5c90b8677ef8d7c9b5ffc0324bbea8e7067f26e","final_host":"nginx.org","final_path":"/robots.txt","ms":414,"html":false,"text":"User-Agent: *\nDisallow: /libxslt/\nSitemap: http://nginx.org/sitemap.xml\n","agents":{"GPTBot":true,"OAI-SearchBot":true,"ChatGPT-User":true,"ClaudeBot":true,"Claude-SearchBot":true,"PerplexityBot":true,"Google-Extended":true,"Applebot-Extended":true,"CCBot":true,"Bytespider":true,"Meta-ExternalAgent":true,"Amazonbot":true}},"sitemap":{"status":200,"content_type":"text/xml","bytes":41548,"sha256":"f74d21966e65e01a5acc8aadf1d2811dabe401dcb5cd497c9e2a81f0f4037414","final_host":"nginx.org","final_path":"/sitemap.xml","ms":187,"url":"http://nginx.org/sitemap.xml","declared":true,"xml":true},"files":{"llms_txt":{"status":404,"content_type":"text/html","bytes":2078,"sha256":"3bc24a23c571aa75630ad6e6c6b44a981f381a68981e8cfba5eee97eb64bf587","final_host":"nginx.org","final_path":"/llms.txt","ms":65,"path":"/llms.txt","html":true,"json":null,"present":false},"agents_md":{"status":404,"content_type":"text/html","bytes":2078,"sha256":"3bc24a23c571aa75630ad6e6c6b44a981f381a68981e8cfba5eee97eb64bf587","final_host":"nginx.org","final_path":"/agents.md","ms":65,"path":"/agents.md","html":true,"json":null,"present":false},"agent_skills":{"status":404,"content_type":"text/html","bytes":2078,"sha256":"3bc24a23c571aa75630ad6e6c6b44a981f381a68981e8cfba5eee97eb64bf587","final_host":"nginx.org","final_path":"/.well-known/agent-skills/index.json","ms":61,"path":"/.well-known/agent-skills/index.json","html":true,"json":null,"present":false},"mcp_card":{"status":404,"content_type":"text/html","bytes":2078,"sha256":"3bc24a23c571aa75630ad6e6c6b44a981f381a68981e8cfba5eee97eb64bf587","final_host":"nginx.org","final_path":"/.well-known/mcp/server-card.json","ms":68,"path":"/.well-known/mcp/server-card.json","html":true,"json":null,"present":false},"api_catalog":{"status":404,"content_type":"text/html","bytes":2078,"sha256":"3bc24a23c571aa75630ad6e6c6b44a981f381a68981e8cfba5eee97eb64bf587","final_host":"nginx.org","final_path":"/.well-known/api-catalog","ms":72,"path":"/.well-known/api-catalog","html":true,"json":null,"present":false},"a2a_card":{"status":404,"content_type":"text/html","bytes":2078,"sha256":"3bc24a23c571aa75630ad6e6c6b44a981f381a68981e8cfba5eee97eb64bf587","final_host":"nginx.org","final_path":"/.well-known/agent-card.json","ms":93,"path":"/.well-known/agent-card.json","html":true,"json":null,"present":false}},"rings":{"v":1,"candidates":[],"mcp":[],"a2a":null,"webmcp":{"model_context":false,"bridge":false,"mcp_json":false,"link_rel":false,"any":false},"handshakes":0,"get_path":{"status":404,"content_type":"text/html","ms":60,"error":null},"get_host":{"status":530,"content_type":"text/plain","ms":100,"error":null}},"sf":[1,0,0,0,0,0,0,0,0],"derived":{"reachable":true,"home_class":"answers","robots_served":true,"robots_sitemap":true,"content_signal":false,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":[],"blocks_any_ai":false,"files":{"llms_txt":false,"agents_md":false,"agent_skills":false,"mcp_card":false,"api_catalog":false,"a2a_card":false},"any_capability":false,"ai_refused":[],"ai_thin":[],"aipref":0,"aipref_measured":true,"sf":[1,0,0,0,0,0,0,0,0],"rings":{"mcp_declared":false,"mcp_live":false,"mcp_auth":false,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":3,"fh":"nginx.org","fp_other_host":[],"fp":"d5a07d05d1501da2","fp_bits":{"gptbot":0,"claudebot":0,"oaisearch":0,"perplexity":0}},"derive_v":5,"frame":"top10k","shard":3},{"v":1,"kind":"crawlcheck-census-record","run":18,"domain":"akam.net","rank":77,"measured_at":"2026-11-13T18:51:40.579Z","home":{"browser":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"akam.net","final_path":"/","ms":12,"identity":"Mobile browser","html":false},"gptbot":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"akam.net","final_path":"/","ms":1,"identity":"GPTBot","html":false},"claudebot":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"akam.net","final_path":"/","ms":0,"identity":"ClaudeBot","html":false},"oaisearch":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"akam.net","final_path":"/","ms":0,"identity":"OAI-SearchBot","html":false},"perplexity":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"akam.net","final_path":"/","ms":1,"identity":"PerplexityBot","html":false}},"robots":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"akam.net","final_path":"/robots.txt","ms":1,"html":false,"text":null,"agents":null},"sitemap":null,"files":null,"derived":{"reachable":false,"home_class":"no_site","robots_served":false,"robots_sitemap":false,"content_signal":false,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":[],"blocks_any_ai":false,"files":{"llms_txt":false,"agents_md":false,"agent_skills":false,"mcp_card":false,"api_catalog":false,"a2a_card":false},"any_capability":false,"ai_refused":[],"ai_thin":[],"rings":{"mcp_declared":false,"mcp_live":false,"mcp_auth":false,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":3,"fh":"akam.net","fp_other_host":[]},"derive_v":5,"frame":"top10k","shard":4},{"v":1,"kind":"crawlcheck-census-record","run":18,"domain":"tiktokcdn.com","rank":78,"measured_at":"2026-11-13T18:52:33.408Z","home":{"browser":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"tiktokcdn.com","final_path":"/","ms":4,"identity":"Mobile browser","html":false},"gptbot":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"tiktokcdn.com","final_path":"/","ms":1,"identity":"GPTBot","html":false},"claudebot":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"tiktokcdn.com","final_path":"/","ms":0,"identity":"ClaudeBot","html":false},"oaisearch":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"tiktokcdn.com","final_path":"/","ms":1,"identity":"OAI-SearchBot","html":false},"perplexity":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"tiktokcdn.com","final_path":"/","ms":0,"identity":"PerplexityBot","html":false}},"robots":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"tiktokcdn.com","final_path":"/robots.txt","ms":0,"html":false,"text":null,"agents":null},"sitemap":null,"files":null,"derived":{"reachable":false,"home_class":"no_site","robots_served":false,"robots_sitemap":false,"content_signal":false,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":[],"blocks_any_ai":false,"files":{"llms_txt":false,"agents_md":false,"agent_skills":false,"mcp_card":false,"api_catalog":false,"a2a_card":false},"any_capability":false,"ai_refused":[],"ai_thin":[],"rings":{"mcp_declared":false,"mcp_live":false,"mcp_auth":false,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":3,"fh":"tiktokcdn.com","fp_other_host":[]},"derive_v":5,"frame":"top10k","shard":5},{"v":1,"kind":"crawlcheck-census-record","run":18,"domain":"edgekey.net","rank":79,"measured_at":"2026-11-13T18:53:15.625Z","home":{"browser":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"edgekey.net","final_path":"/","ms":7,"identity":"Mobile browser","html":false},"gptbot":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"edgekey.net","final_path":"/","ms":1,"identity":"GPTBot","html":false},"claudebot":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"edgekey.net","final_path":"/","ms":0,"identity":"ClaudeBot","html":false},"oaisearch":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"edgekey.net","final_path":"/","ms":1,"identity":"OAI-SearchBot","html":false},"perplexity":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"edgekey.net","final_path":"/","ms":1,"identity":"PerplexityBot","html":false}},"robots":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"edgekey.net","final_path":"/robots.txt","ms":0,"html":false,"text":null,"agents":null},"sitemap":null,"files":null,"derived":{"reachable":false,"home_class":"no_site","robots_served":false,"robots_sitemap":false,"content_signal":false,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":[],"blocks_any_ai":false,"files":{"llms_txt":false,"agents_md":false,"agent_skills":false,"mcp_card":false,"api_catalog":false,"a2a_card":false},"any_capability":false,"ai_refused":[],"ai_thin":[],"rings":{"mcp_declared":false,"mcp_live":false,"mcp_auth":false,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":3,"fh":"edgekey.net","fp_other_host":[]},"derive_v":5,"frame":"top10k","shard":6},{"v":1,"kind":"crawlcheck-census-record","run":18,"domain":"baidu.com","rank":80,"measured_at":"2026-11-13T18:55:41.951Z","home":{"browser":{"status":200,"content_type":"text/html","bytes":724489,"sha256":"a96d3b18161525992fe75ea34eec15c3534c4ae274133d5b223c92b886bc7e9b","final_host":"www.baidu.com","final_path":"/","ms":1535,"identity":"Desktop browser (device re-check)","html":true,"fp":"ea77dfec638f13e7"},"gptbot":{"status":200,"content_type":"text/html","bytes":725720,"sha256":"557c39af4ff7cf2fd9877323a11f322f9e6fd7d1d2f1a0ddf9e9839961cfb88b","final_host":"www.baidu.com","final_path":"/","ms":740,"identity":"GPTBot","html":true,"fp":"ea77dfec638f13e7"},"claudebot":{"status":200,"content_type":"text/html","bytes":724273,"sha256":"a0388dbe06a04b33c77f70a49d2ec317940393781dfd4180427956b160e650ce","final_host":"www.baidu.com","final_path":"/","ms":745,"identity":"ClaudeBot","html":true,"fp":"ea77dfec638f13e7"},"oaisearch":{"status":200,"content_type":"text/html","bytes":725606,"sha256":"59a10468240f37720403cf64cfa7461a3e049e6b59f678e106d076faa9e14d18","final_host":"www.baidu.com","final_path":"/","ms":648,"identity":"OAI-SearchBot","html":true,"fp":"ea77dfec638f13e7"},"perplexity":{"status":200,"content_type":"text/html","bytes":725564,"sha256":"e201ca3ff4475bc278e7e58bbb1833be72072abd06f53f9d2b8b0eee72649a8f","final_host":"www.baidu.com","final_path":"/","ms":1523,"identity":"PerplexityBot","html":true,"fp":"ea77dfec638f13e7"},"browser_mobile":{"status":200,"content_type":"text/html","bytes":193062,"sha256":"baf664d17e3b6de7aa59914021e2464b56bfaa52a12488fb89ec36467cf6e822","final_host":"www.baidu.com","final_path":"/","ms":800,"identity":"Mobile browser","html":true,"fp":"02001e05754892c0"}},"robots":{"status":200,"content_type":"text/plain","bytes":2814,"sha256":"c28ebeffb7baf9bd6407109e80dc95d87c75c0eaecc20b23884cb33111cc5df0","final_host":"www.baidu.com","final_path":"/robots.txt","ms":1690,"html":false,"text":"User-agent: Baiduspider\nDisallow: /baidu\nDisallow: /s?\nDisallow: /ulink?\nDisallow: /link?\nDisallow: /home/news/data/\nDisallow: /bh\n\nUser-agent: Googlebot\nDisallow: /baidu\nDisallow: /s?\nDisallow: /shifen/\nDisallow: /homepage/\nDisallow: /cpro\nDisallow: /ulink?\nDisallow: /link?\nDisallow: /home/news/data/\nDisallow: /bh\n\nUser-agent: MSNBot\nDisallow: /baidu\nDisallow: /s?\nDisallow: /shifen/\nDisallow: /homepage/\nDisallow: /cpro\nDisallow: /ulink?\nDisallow: /link?\nDisallow: /home/news/data/\nDisallow: /bh\n\nUser-agent: Baiduspider-image\nDisallow: /baidu\nDisallow: /s?\nDisallow: /shifen/\nDisallow: /homepage/\nDisallow: /cpro\nDisallow: /ulink?\nDisallow: /link?\nDisallow: /home/news/data/\nDisallow: /bh\n\nUser-agent: YoudaoBot\nDisallow: /baidu\nDisallow: /s?\nDisallow: /shifen/\nDisallow: /homepage/\nDisallow: /cpro\nDisallow: /ulink?\nDisallow: /link?\nDisallow: /home/news/data/\nDisallow: /bh\n\nUser-agent: Sogou web spider\nDisallow: /baidu\nDisallow: /s?\nDisallow: /shifen/\nDisallow: /homepage/\nDisallow: /cpro\nDisallow: /ulink?\nDisallow: /link?\nDisallow: /home/news/data/\nDisallow: /bh\n\nUser-agent: Sogou inst spider\nDisallow: /baidu\nDisallow: /s?\nDisallow: /shifen/\nDisallow: /homepage/\nDisallow: /cpro\nDisallow: /ulink?\nDisallow: /link?\nDisallow: /home/news/data/\nDisallow: /bh\n\nUser-agent: Sogou spider2\nDisallow: /baidu\nDisallow: /s?\nDisallow: /shifen/\nDisallow: /homepage/\nDisallow: /cpro\nDisallow: /ulink?\nDisallow: /link?\nDisallow: /home/news/data/\nDisallow: /bh\n\nUser-agent: Sogou blog\nDisallow: /baidu\nDisallow: /s?\nDisallow: /shifen/\nDisallow: /homepage/\nDisallow: /cpro\nDisallow: /ulink?\nDisallow: /link?\nDisallow: /home/news/data/\nDisallow: /bh\n\nUser-agent: Sogou News Spider\nDisallow: /baidu\nDisallow: /s?\nDisallow: /shifen/\nDisallow: /homepage/\nDisallow: /cpro\nDisallow: /ulink?\nDisallow: /link?\nDisallow: /home/news/data/\nDisallow: /bh\n\nUser-agent: Sogou Orion spider\nDisallow: /baidu\nDisallow: /s?\nDisallow: /shifen/\nDisallow: /homepage/\nDisallow: /cpro\nDisallow: /ulink?\nDisallow: /link?\nDisallow: /home/news/data/\nDisallow: /bh\n\nUser-agent: ChinasoSpider\nDisallow: /baidu\nDisallow: /s?\nDisallow: /shifen/\nDisallow: /homepage/\nDisallow: /cpro\nDisallow: /ulink?\nDisallow: /link?\nDisallow: /home/news/data/\nDisallow: /bh\n\nUser-agent: Sosospider\nDisallow: /baidu\nDisallow: /s?\nDisallow: /shifen/\nDisallow: /homepage/\nDisallow: /cpro\nDisallow: /ulink?\nDisallow: /link?\nDisallow: /home/news/data/\nDisallow: /bh\n\n\nUser-agent: yisouspider\nDisallow: /baidu\nDisallow: /s?\nDisallow: /shifen/\nDisallow: /homepage/\nDisallow: /cpro\nDisallow: /ulink?\nDisallow: /link?\nDisallow: /home/news/data/\nDisallow: /bh\n\nUser-agent: EasouSpider\nDisallow: /baidu\nDisallow: /s?\nDisallow: /shifen/\nDisallow: /homepage/\nDisallow: /cpro\nDisallow: /ulink?\nDisallow: /link?\nDisallow: /home/news/data/\nDisallow: /bh\n\nUser-agent: *\nDisallow: /\n","agents":{"GPTBot":false,"OAI-SearchBot":false,"ChatGPT-User":false,"ClaudeBot":false,"Claude-SearchBot":false,"PerplexityBot":false,"Google-Extended":false,"Applebot-Extended":false,"CCBot":false,"Bytespider":false,"Meta-ExternalAgent":false,"Amazonbot":false}},"sitemap":{"status":404,"content_type":"text/html","bytes":209,"sha256":"636ca4bd838c8eeffcf8eaae1a56f72b6b93bb9df4f0d6e58333361968684800","final_host":"www.baidu.com","final_path":"/sitemap.xml","ms":1607,"url":"https://baidu.com/sitemap.xml","declared":false,"xml":false},"files":{"llms_txt":{"status":404,"content_type":"text/html","bytes":206,"sha256":"0da006306ca67b9d88367bc1a63929b073ed36fb046b9b61695f934bfa544b89","final_host":"www.baidu.com","final_path":"/llms.txt","ms":535,"path":"/llms.txt","html":true,"json":null,"present":false},"agents_md":{"status":404,"content_type":"text/html","bytes":207,"sha256":"a50408f4a5a4d457e4f46fe099a1286627188827df0a21eac4e3aea2d8f471d6","final_host":"www.baidu.com","final_path":"/agents.md","ms":520,"path":"/agents.md","html":true,"json":null,"present":false},"agent_skills":{"status":404,"content_type":"text/html","bytes":233,"sha256":"532bb81bdb57c440d614fcb44cbbcac627252208828f307c7943796457dfc49c","final_host":"www.baidu.com","final_path":"/.well-known/agent-skills/index.json","ms":521,"path":"/.well-known/agent-skills/index.json","html":true,"json":null,"present":false},"mcp_card":{"status":404,"content_type":"text/html","bytes":230,"sha256":"ff2106cf9a2975014654029d3a652abeccb0b8c216730f442ddabc48f763a431","final_host":"www.baidu.com","final_path":"/.well-known/mcp/server-card.json","ms":435,"path":"/.well-known/mcp/server-card.json","html":true,"json":null,"present":false},"api_catalog":{"status":404,"content_type":"text/html","bytes":221,"sha256":"923d0d4fc2d80a5fe8f23656d1f52a907c9fe33c7380164a03f29abd4b3979b6","final_host":"www.baidu.com","final_path":"/.well-known/api-catalog","ms":612,"path":"/.well-known/api-catalog","html":true,"json":null,"present":false},"a2a_card":{"status":404,"content_type":"text/html","bytes":225,"sha256":"8ec3ff7fbb679822567977a30006dbab75591df75733881c63ffe988553cf549","final_host":"www.baidu.com","final_path":"/.well-known/agent-card.json","ms":172,"path":"/.well-known/agent-card.json","html":true,"json":null,"present":false}},"device_variant":true,"rings":{"v":1,"candidates":[],"mcp":[],"a2a":null,"webmcp":{"model_context":false,"bridge":false,"mcp_json":false,"link_rel":false,"any":false},"handshakes":0,"get_path":{"status":404,"content_type":"text/html","ms":149,"error":null},"get_host":{"status":404,"content_type":"text/html","ms":5366,"error":null}},"sf":[4,0,0,0,0,0,3,0,0],"derived":{"reachable":true,"home_class":"answers","robots_served":true,"robots_sitemap":false,"content_signal":false,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":["GPTBot","OAI-SearchBot","ChatGPT-User","ClaudeBot","Claude-SearchBot","PerplexityBot","Google-Extended","Applebot-Extended","CCBot","Bytespider","Meta-ExternalAgent","Amazonbot"],"blocks_any_ai":true,"files":{"llms_txt":false,"agents_md":false,"agent_skills":false,"mcp_card":false,"api_catalog":false,"a2a_card":false},"any_capability":false,"ai_refused":[],"ai_thin":[],"aipref":0,"aipref_measured":true,"sf":[4,0,0,0,0,0,3,0,0],"rings":{"mcp_declared":false,"mcp_live":false,"mcp_auth":false,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":3,"fh":"www.baidu.com","fp_other_host":[],"fp":"ea77dfec638f13e7","fp_bits":{"gptbot":0,"claudebot":0,"oaisearch":0,"perplexity":0}},"derive_v":5,"frame":"top10k","shard":7},{"v":1,"kind":"crawlcheck-census-record","run":18,"domain":"workers.dev","rank":81,"measured_at":"2026-11-13T18:47:19.534Z","home":{"browser":{"status":200,"content_type":"text/html","bytes":757363,"sha256":"45962e0902fb60cb8ff41e4b9ed476df77f6b881c4c548147c51991dda2fba18","final_host":"www.cloudflare.com","final_path":"/products/workers","ms":459,"identity":"Mobile browser","html":true,"fp":"53623efc065a9de3"},"gptbot":{"status":200,"content_type":"text/html","bytes":757359,"sha256":"45962e0902fb60cb8ff41e4b9ed476df77f6b881c4c548147c51991dda2fba18","final_host":"www.cloudflare.com","final_path":"/products/workers","ms":458,"identity":"GPTBot","html":true,"fp":"53623efc065a9de3"},"claudebot":{"status":200,"content_type":"text/html","bytes":757361,"sha256":"45962e0902fb60cb8ff41e4b9ed476df77f6b881c4c548147c51991dda2fba18","final_host":"www.cloudflare.com","final_path":"/products/workers","ms":460,"identity":"ClaudeBot","html":true,"fp":"53623efc065a9de3"},"oaisearch":{"status":200,"content_type":"text/html","bytes":757359,"sha256":"45962e0902fb60cb8ff41e4b9ed476df77f6b881c4c548147c51991dda2fba18","final_host":"www.cloudflare.com","final_path":"/products/workers","ms":455,"identity":"OAI-SearchBot","html":true,"fp":"53623efc065a9de3"},"perplexity":{"status":200,"content_type":"text/html","bytes":757359,"sha256":"45962e0902fb60cb8ff41e4b9ed476df77f6b881c4c548147c51991dda2fba18","final_host":"www.cloudflare.com","final_path":"/products/workers","ms":459,"identity":"PerplexityBot","html":true,"fp":"53623efc065a9de3"}},"robots":{"status":200,"content_type":"text/html","bytes":756924,"sha256":"d6f4a98fa567a3183ed1bbf76c7e05344ccb7f9eb1b4518a3e5c0f57c44ac7ad","final_host":"www.cloudflare.com","final_path":"/products/workers","ms":435,"html":true,"text":null,"agents":null},"sitemap":{"status":200,"content_type":"text/html","bytes":756924,"sha256":"9af1a61d877ecd7e3d3ffe40ee82d1ab546a6d36a9202477a5dcca0eca5ce90f","final_host":"www.cloudflare.com","final_path":"/products/workers","ms":408,"url":"https://workers.dev/sitemap.xml","declared":false,"xml":false},"files":{"llms_txt":{"status":200,"content_type":"text/markdown","bytes":3314,"sha256":"096e84c5fb5f1b086aca292560dab76d533bee7e5c5daa55cabe2d720f36054e","final_host":"www.cloudflare.com","final_path":"/products/workers","ms":338,"path":"/llms.txt","html":false,"json":null,"present":true},"agents_md":{"status":200,"content_type":"text/markdown","bytes":3314,"sha256":"096e84c5fb5f1b086aca292560dab76d533bee7e5c5daa55cabe2d720f36054e","final_host":"www.cloudflare.com","final_path":"/products/workers","ms":339,"path":"/agents.md","html":false,"json":null,"present":true},"agent_skills":{"status":200,"content_type":"text/html","bytes":756924,"sha256":"9af1a61d877ecd7e3d3ffe40ee82d1ab546a6d36a9202477a5dcca0eca5ce90f","final_host":"www.cloudflare.com","final_path":"/products/workers","ms":532,"path":"/.well-known/agent-skills/index.json","html":true,"json":null,"present":false},"mcp_card":{"status":200,"content_type":"text/html","bytes":756924,"sha256":"d6f4a98fa567a3183ed1bbf76c7e05344ccb7f9eb1b4518a3e5c0f57c44ac7ad","final_host":"www.cloudflare.com","final_path":"/products/workers","ms":556,"path":"/.well-known/mcp/server-card.json","html":true,"json":null,"present":false},"api_catalog":{"status":200,"content_type":"text/html","bytes":756924,"sha256":"d6f4a98fa567a3183ed1bbf76c7e05344ccb7f9eb1b4518a3e5c0f57c44ac7ad","final_host":"www.cloudflare.com","final_path":"/products/workers","ms":545,"path":"/.well-known/api-catalog","html":true,"json":null,"present":false},"a2a_card":{"status":200,"content_type":"text/html","bytes":756924,"sha256":"d6f4a98fa567a3183ed1bbf76c7e05344ccb7f9eb1b4518a3e5c0f57c44ac7ad","final_host":"www.cloudflare.com","final_path":"/products/workers","ms":432,"path":"/.well-known/agent-card.json","html":true,"json":null,"present":false}},"rings":{"v":1,"candidates":[],"mcp":[],"a2a":null,"webmcp":{"model_context":false,"bridge":false,"mcp_json":false,"link_rel":false,"any":false},"handshakes":0,"get_path":{"status":200,"content_type":"text/html","ms":205,"error":null},"get_host":{"status":530,"content_type":"text/plain","ms":8,"error":null}},"sf":[4,0,0,0,0,0,1,0,1],"derived":{"reachable":true,"home_class":"answers","robots_served":false,"robots_sitemap":false,"content_signal":false,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":[],"blocks_any_ai":false,"files":{"llms_txt":true,"agents_md":true,"agent_skills":false,"mcp_card":false,"api_catalog":false,"a2a_card":false},"any_capability":false,"ai_refused":[],"ai_thin":[],"sf":[4,0,0,0,0,0,1,0,1],"rings":{"mcp_declared":false,"mcp_live":false,"mcp_auth":false,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":3,"fh":"www.cloudflare.com","fp_other_host":[],"fp":"53623efc065a9de3","fp_bits":{"gptbot":0,"claudebot":0,"oaisearch":0,"perplexity":0}},"derive_v":5,"frame":"top10k","shard":0},{"v":1,"kind":"crawlcheck-census-record","run":18,"domain":"tiktokv.com","rank":82,"measured_at":"2026-11-13T18:49:01.730Z","home":{"browser":{"status":404,"content_type":"text/html","bytes":546,"sha256":"5dd23ed6464f587908166860364362088446bf508f2793c4df0ad2119675c33e","final_host":"tiktokv.com","final_path":"/","ms":18,"identity":"Mobile browser","html":true},"gptbot":{"status":404,"content_type":"text/html","bytes":144,"sha256":"b282ac8512c5e029959282e6fd450dc119f5db4a072869eabf45e31440596248","final_host":"tiktokv.com","final_path":"/","ms":18,"identity":"GPTBot","html":true},"claudebot":{"status":404,"content_type":"text/html","bytes":144,"sha256":"b282ac8512c5e029959282e6fd450dc119f5db4a072869eabf45e31440596248","final_host":"tiktokv.com","final_path":"/","ms":26,"identity":"ClaudeBot","html":true},"oaisearch":{"status":404,"content_type":"text/html","bytes":144,"sha256":"b282ac8512c5e029959282e6fd450dc119f5db4a072869eabf45e31440596248","final_host":"tiktokv.com","final_path":"/","ms":21,"identity":"OAI-SearchBot","html":true},"perplexity":{"status":404,"content_type":"text/html","bytes":144,"sha256":"b282ac8512c5e029959282e6fd450dc119f5db4a072869eabf45e31440596248","final_host":"tiktokv.com","final_path":"/","ms":18,"identity":"PerplexityBot","html":true}},"robots":{"status":404,"content_type":"text/html","bytes":144,"sha256":"b282ac8512c5e029959282e6fd450dc119f5db4a072869eabf45e31440596248","final_host":"tiktokv.com","final_path":"/robots.txt","ms":31,"html":true,"text":null,"agents":null},"sitemap":null,"files":null,"derived":{"reachable":false,"home_class":"other","robots_served":false,"robots_sitemap":false,"content_signal":false,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":[],"blocks_any_ai":false,"files":{"llms_txt":false,"agents_md":false,"agent_skills":false,"mcp_card":false,"api_catalog":false,"a2a_card":false},"any_capability":false,"ai_refused":[],"ai_thin":[],"rings":{"mcp_declared":false,"mcp_live":false,"mcp_auth":false,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":3,"fh":"tiktokv.com","fp_other_host":[]},"derive_v":5,"frame":"top10k","shard":1},{"v":1,"kind":"crawlcheck-census-record","run":18,"domain":"chatgpt.com","rank":83,"measured_at":"2026-11-13T18:50:07.634Z","home":{"browser":{"status":200,"content_type":"text/html","bytes":460963,"sha256":"07dd3e5f16d20cf1240e7e95ee99f2791efcbe7f399fb5ba1b6ae0985f29094d","final_host":"chatgpt.com","final_path":"/","ms":583,"identity":"Mobile browser","html":true,"fp":"67454e6b59979aad"},"gptbot":{"status":200,"content_type":"text/html","bytes":448212,"sha256":"7dcb3c396e2c353a9fd5c833d6988968970a6249f8b399fef43578edf7908a17","final_host":"chatgpt.com","final_path":"/","ms":394,"identity":"GPTBot","html":true,"fp":"67454e6b59979aad"},"claudebot":{"status":200,"content_type":"text/html","bytes":450594,"sha256":"e4395bd59224f1e691598407dc308b44afb61acc4fa99bc1a206e8c308bc4d16","final_host":"chatgpt.com","final_path":"/","ms":493,"identity":"ClaudeBot","html":true,"fp":"67454e6b59979aad"},"oaisearch":{"status":200,"content_type":"text/html","bytes":450592,"sha256":"4104f22e76a64ca61604d5741f242a2d37b678264f98b51f5b009029c7a892fb","final_host":"chatgpt.com","final_path":"/","ms":495,"identity":"OAI-SearchBot","html":true,"fp":"67454e6b59979aad"},"perplexity":{"status":200,"content_type":"text/html","bytes":457000,"sha256":"32a6120be6d070eba35960ca7c7c1e3d99867a3a3dcd4593f81c16c68386b336","final_host":"chatgpt.com","final_path":"/","ms":543,"identity":"PerplexityBot","html":true,"fp":"67456e7b59979aad"}},"robots":{"status":200,"content_type":"text/plain","bytes":4573,"sha256":"7368cdfdeb325f204dc704c0696f0f9e4137d80440f596c1b1f71f872160ba04","final_host":"chatgpt.com","final_path":"/robots.txt","ms":13,"html":false,"text":"# https://www.robotstxt.org/robotstxt.html\n\nUser-agent: CCBot\nDisallow: /\n\nUser-agent: img2dataset\nDisallow: /\n\nUser-agent: Google-Extended\nDisallow: /\n\n# Prevent shared pages and GPTs from being used as Google Ads destinations\nUser-agent: AdsBot-Google\nUser-agent: AdsBot-Google-Mobile\nDisallow: /s/\nDisallow: /g/\n\nUser-agent: anthropic-ai\nDisallow: /\n\nUser-agent: Claude-Web\nDisallow: /\n\nUser-agent: Omgilibot\nDisallow: /\n\nUser-agent: Omgili\nDisallow: /\n\nUser-agent: FacebookBot\nDisallow: /\n\nUser-agent: Bytespider\nDisallow: /\n\nUser-agent: magpie-crawler\nDisallow: /\n\nUser-agent: PerplexityBot\nDisallow: /\n\nUser-agent: Perplexity‑User\nDisallow: /\n\n# General rules for all other bots\nUser-agent: *\n# Allow Googlebot to render the indexable homepage; production CSS and JavaScript live here.\nAllow: /unauth-mweb/assets/\n\n# Place allows first to avoid bots skipping after Disallow: /\n# Allow exactly the homepage\nAllow: /$\n# Allow the homepage with any query parameters\nAllow: /?*\nAllow: /api/share/og/\nAllow: /g/\nAllow: /s/\nAllow: /gg/v/\nAllow: /m/\nAllow: /share/\nAllow: /maps$\nAllow: /maps/\nAllow: /*/maps$\nAllow: /*/maps/\nAllow: /canvas/shared/\nAllow: /*/images\nAllow: /images\nAllow: /*/library\nAllow: /library\nAllow: /favicon.ico\nAllow: /assets/favicon\nAllow: /cdn/assets/favicon\nAllow: /cdn/assets/\nAllow: /auth/\nAllow: /gpts$\nAllow: /codex\nAllow: /*/codex\nAllow: /search$\nAllow: /backend-anon/\nAllow: /public-api/\nAllow: /sitemap.xml\nAllow: /marketing-sitemap.xml\nAllow: /writing-tools-sitemap.xml\nAllow: /100chats\nAllow: /api/public_content/\nAllow: /backend-api/public_content/\nAllow: /?ref=dotcom\n# Static Landing Pages\nAllow: /overview\nAllow: /*/overview\nAllow: /features\nAllow: /*/features\nAllow: /redeem$\nAllow: /redeem/\nAllow: /redeem?\nAllow: /health$\nAllow: /health/$\nAllow: /*/health$\nAllow: /*/health/$\nAllow: /voice$\nAllow: /voice/$\nAllow: /*/voice$\nAllow: /*/voice/$\nAllow: /apps\nAllow: /*/apps\nAllow: /finances\nAllow: /*/finances\nAllow: /shopping\nAllow: /*/shopping\nAllow: /use-cases\nAllow: /*/use-cases\nAllow: /learn\nAllow: /*/learn\nAllow: /business\nAllow: /*/business\nAllow: /*/business-hub\nAllow: /pricing\nAllow: /*/pricing\nAllow: /download\nAllow: /*/download\nAllow: /students\nAllow: /students/2026\nAllow: /*/college-students\nAllow: /college-students\nAllow: /contact-sales\nAllow: /*/contact-sales\nAllow: /100chats-project\nAllow: /*/100chats-project\nAllow: /merchants\nAllow: /*/merchants\nAllow: /parent-resources\nAllow: /*/parent-resources\nAllow: /plans\nAllow: /*/plans\nAllow: /plugins\nAllow: /translate\nAllow: /*/translate\nAllow: /writing\nAllow: /*/writing\nAllow: /ask/\nAllow: /*/ask\nAllow: /futures\nAllow: /*/futures\nAllow: /football\nAllow: /*/football\nAllow: /remote\nAllow: /*/remote\nAllow: /work\nAllow: /*/work\n# Allow crawlers to read the campaign page's noindex directive.\nAllow: /get-dot$\nAllow: /get-dot/$\nAllow: /get-dot?\nAllow: /get-dot/?\nAllow: /*/get-dot$\nAllow: /*/get-dot/$\nAllow: /*/get-dot?\nAllow: /*/get-dot/?\nAllow: /dots\nAllow: /*/dots\nAllow: /sites\nAllow: /*/sites\nAllow: /space\nAllow: /*/space\nAllow: /import-to-chatgpt\nAllow: /*/import-to-chatgpt\n# Exact locale specific homepages\nAllow: /am/$\nAllow: /ar/$\nAllow: /bg-BG/$\nAllow: /bn-BD/$\nAllow: /bs-BA/$\nAllow: /ca-ES/$\nAllow: /cs-CZ/$\nAllow: /da-DK/$\nAllow: /de-DE/$\nAllow: /el-GR/$\nAllow: /es-ES/$\nAllow: /es-419/$\nAllow: /et-EE/$\nAllow: /fa-IR/$\nAllow: /fi-FI/$\nAllow: /fr-FR/$\nAllow: /fr-CA/$\nAllow: /gu-IN/$\nAllow: /he-IL/$\nAllow: /hi-IN/$\nAllow: /hr-HR/$\nAllow: /hu-HU/$\nAllow: /hy-AM/$\nAllow: /id-ID/$\nAllow: /is-IS/$\nAllow: /it-IT/$\nAllow: /ja-JP/$\nAllow: /ka-GE/$\nAllow: /kk/$\nAllow: /kn-IN/$\nAllow: /ko-KR/$\nAllow: /lt/$\nAllow: /lv-LV/$\nAllow: /mk-MK/$\nAllow: /ml/$\nAllow: /mn/$\nAllow: /mr-IN/$\nAllow: /ms-MY/$\nAllow: /my-MM/$\nAllow: /nb-NO/$\nAllow: /nl-NL/$\nAllow: /pa/$\nAllow: /pl-PL/$\nAllow: /pt-BR/$\nAllow: /pt-PT/$\nAllow: /ro-RO/$\nAllow: /ru-RU/$\nAllow: /sk-SK/$\nAllow: /sl-SI/$\nAllow: /so-SO/$\nAllow: /sq-AL/$\nAllow: /sr-RS/$\nAllow: /sv-SE/$\nAllow: /sw-TZ/$\nAllow: /ta-IN/$\nAllow: /te-IN/$\nAllow: /th-TH/$\nAllow: /tl/$\nAllow: /tr-TR/$\nAllow: /uk-UA/$\nAllow: /ur/$\nAllow: /vi-VN/$\nAllow: /zh-CN/$\nAllow: /zh-TW/$\nAllow: /zh-HK/$\n\n# Now block everything else\nDisallow: /\n\n# Specific disallows (redundant for some bots, but still useful for those that respect precedence)\nDisallow: /auth/logout\nDisallow: /auth/login?*\nDisallow: /backend-anon/sentinel/*\nDisallow: /backend-anon/conversation$\nDisallow: /account-link/*\n\nSitemap: https://chatgpt.com/sitemap.xml\nSitemap: https://chatgpt.com/marketing-sitemap.xml\nSitemap: https://chatgpt.com/writing-tools-sitemap.xml\n","agents":{"GPTBot":true,"OAI-SearchBot":true,"ChatGPT-User":true,"ClaudeBot":true,"Claude-SearchBot":true,"PerplexityBot":false,"Google-Extended":false,"Applebot-Extended":true,"CCBot":false,"Bytespider":false,"Meta-ExternalAgent":true,"Amazonbot":true}},"sitemap":{"status":200,"content_type":"application/xml","bytes":4949,"sha256":"675d73e5d9b94382ea5cd98eb58d84b64560bd0b8350e49370881b172af0395e","final_host":"chatgpt.com","final_path":"/sitemap.xml","ms":30,"url":"https://chatgpt.com/sitemap.xml","declared":true,"xml":true},"files":{"llms_txt":{"status":404,"content_type":null,"bytes":0,"sha256":"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855","final_host":"chatgpt.com","final_path":"/llms.txt","ms":57,"path":"/llms.txt","html":false,"json":null,"present":false},"agents_md":{"status":404,"content_type":null,"bytes":0,"sha256":"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855","final_host":"chatgpt.com","final_path":"/agents.md","ms":45,"path":"/agents.md","html":false,"json":null,"present":false},"agent_skills":{"status":404,"content_type":null,"bytes":0,"sha256":"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855","final_host":"chatgpt.com","final_path":"/.well-known/agent-skills/index.json","ms":35,"path":"/.well-known/agent-skills/index.json","html":false,"json":null,"present":false},"mcp_card":{"status":404,"content_type":null,"bytes":0,"sha256":"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855","final_host":"chatgpt.com","final_path":"/.well-known/mcp/server-card.json","ms":28,"path":"/.well-known/mcp/server-card.json","html":false,"json":null,"present":false},"api_catalog":{"status":200,"content_type":"text/html","bytes":722536,"sha256":"88eb9568b19028caabef998dd5869d35824cd3987bf54d6bf7446e0aaf4a91dd","final_host":"chatgpt.com","final_path":"/.well-known/api-catalog","ms":172,"path":"/.well-known/api-catalog","html":true,"json":null,"present":false},"a2a_card":{"status":404,"content_type":null,"bytes":0,"sha256":"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855","final_host":"chatgpt.com","final_path":"/.well-known/agent-card.json","ms":44,"path":"/.well-known/agent-card.json","html":false,"json":null,"present":false}},"rings":{"v":1,"candidates":[],"mcp":[],"a2a":null,"webmcp":{"model_context":false,"bridge":false,"mcp_json":false,"link_rel":false,"any":false},"handshakes":0,"get_path":{"status":200,"content_type":"text/html","ms":243,"error":null},"get_host":{"status":530,"content_type":"text/plain","ms":61,"error":null}},"sf":[4,0,0,0,0,0,0,0,1],"derived":{"reachable":true,"home_class":"answers","robots_served":true,"robots_sitemap":true,"content_signal":false,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":["PerplexityBot","Google-Extended","CCBot","Bytespider"],"blocks_any_ai":true,"files":{"llms_txt":false,"agents_md":false,"agent_skills":false,"mcp_card":false,"api_catalog":false,"a2a_card":false},"any_capability":false,"ai_refused":[],"ai_thin":[],"aipref":0,"aipref_measured":true,"sf":[4,0,0,0,0,0,0,0,1],"rings":{"mcp_declared":false,"mcp_live":false,"mcp_auth":false,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":3,"fh":"chatgpt.com","fp_other_host":[],"fp":"67454e6b59979aad","fp_bits":{"gptbot":0,"claudebot":0,"oaisearch":0,"perplexity":2}},"derive_v":5,"frame":"top10k","shard":2},{"v":1,"kind":"crawlcheck-census-record","run":18,"domain":"yandex.net","rank":84,"measured_at":"2026-11-13T18:51:23.228Z","home":{"browser":{"status":200,"content_type":"text/html","bytes":475053,"sha256":"424e61b14aa539b25eb95549ef9b202777df09ad28da8df6b808c1635eba0038","final_host":"ya.ru","final_path":"/","ms":1087,"identity":"Desktop browser (device re-check)","html":true,"fp":"9f632e9a74d51c22"},"gptbot":{"status":200,"content_type":"text/html","bytes":456051,"sha256":"8bac73d0c57dae772f431795ffc202287e507751ca37118efe8e311df30477b4","final_host":"ya.ru","final_path":"/","ms":3602,"identity":"GPTBot","html":true,"fp":"9f632e9a74d51c22"},"claudebot":{"status":200,"content_type":"text/html","bytes":456041,"sha256":"49c2d1a7a20bba25e5f6f42a016fcdf15cd99dd98158617648759467e882d11f","final_host":"ya.ru","final_path":"/","ms":3222,"identity":"ClaudeBot","html":true,"fp":"9f632e9a74d51c22"},"oaisearch":{"status":200,"content_type":"text/html","bytes":456075,"sha256":"ca760353bce4d6375a9d606b3642660e6d83eeaf2dc19af9bb17b9e678017c79","final_host":"ya.ru","final_path":"/","ms":4076,"identity":"OAI-SearchBot","html":true,"fp":"9f632e9a74d51c22"},"perplexity":{"status":200,"content_type":"text/html","bytes":455981,"sha256":"1a572f96ba757389830e6d9a914a92dbf9e7b7045469e0550625a46357078f42","final_host":"ya.ru","final_path":"/","ms":2867,"identity":"PerplexityBot","html":true,"fp":"9f632e9a74d51c22"},"browser_mobile":{"status":200,"content_type":"text/html","bytes":376960,"sha256":"5e8336efa4438420fa498262b246c8f57388763075e20e8e03a5293ab96d243f","final_host":"ya.ru","final_path":"/","ms":4001,"identity":"Mobile browser","html":true}},"robots":{"status":404,"content_type":null,"bytes":9,"sha256":"0019dfc4b32d63c1392aa264aed2253c1e0c2fb09216f8e2cc269bbfb8bb49b5","final_host":"yandex.net","final_path":"/robots.txt","ms":1117,"html":false,"text":null,"agents":null},"sitemap":{"status":404,"content_type":"text/html","bytes":11808,"sha256":"a6489e4d360e44df9cc51d7cced67e97067d5cec062f9496f12dcc19df9cd6f7","final_host":"ya.ru","final_path":"/sitemap.xml","ms":621,"url":"https://yandex.net/sitemap.xml","declared":false,"xml":false},"files":{"llms_txt":{"status":404,"content_type":"text/plain","bytes":9,"sha256":"e3ebaa16dd9d9b9fc107c42183fb6cf9d22927e1af03dbbdfa0ccc38e4e4ac31","final_host":"ya.ru","final_path":"/llms.txt","ms":452,"path":"/llms.txt","html":false,"json":null,"present":false},"agents_md":{"status":404,"content_type":"text/plain","bytes":9,"sha256":"e3ebaa16dd9d9b9fc107c42183fb6cf9d22927e1af03dbbdfa0ccc38e4e4ac31","final_host":"ya.ru","final_path":"/agents.md","ms":452,"path":"/agents.md","html":false,"json":null,"present":false},"agent_skills":{"status":404,"content_type":null,"bytes":9,"sha256":"0019dfc4b32d63c1392aa264aed2253c1e0c2fb09216f8e2cc269bbfb8bb49b5","final_host":"yandex.net","final_path":"/.well-known/agent-skills/index.json","ms":213,"path":"/.well-known/agent-skills/index.json","html":false,"json":null,"present":false},"mcp_card":{"status":404,"content_type":null,"bytes":9,"sha256":"0019dfc4b32d63c1392aa264aed2253c1e0c2fb09216f8e2cc269bbfb8bb49b5","final_host":"yandex.net","final_path":"/.well-known/mcp/server-card.json","ms":212,"path":"/.well-known/mcp/server-card.json","html":false,"json":null,"present":false},"api_catalog":{"status":404,"content_type":null,"bytes":9,"sha256":"0019dfc4b32d63c1392aa264aed2253c1e0c2fb09216f8e2cc269bbfb8bb49b5","final_host":"yandex.net","final_path":"/.well-known/api-catalog","ms":202,"path":"/.well-known/api-catalog","html":false,"json":null,"present":false},"a2a_card":{"status":404,"content_type":null,"bytes":9,"sha256":"0019dfc4b32d63c1392aa264aed2253c1e0c2fb09216f8e2cc269bbfb8bb49b5","final_host":"yandex.net","final_path":"/.well-known/agent-card.json","ms":629,"path":"/.well-known/agent-card.json","html":false,"json":null,"present":false}},"device_variant":true,"rings":{"v":1,"candidates":[],"mcp":[],"a2a":null,"webmcp":{"model_context":false,"bridge":false,"mcp_json":false,"link_rel":false,"any":false},"handshakes":0,"get_path":{"status":404,"content_type":"application/json","ms":572,"error":null},"get_host":{"status":null,"content_type":null,"ms":8000,"error":"timeout"}},"sf":[4,0,0,0,0,0,1,0,0],"derived":{"reachable":true,"home_class":"answers","robots_served":false,"robots_sitemap":false,"content_signal":false,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":[],"blocks_any_ai":false,"files":{"llms_txt":false,"agents_md":false,"agent_skills":false,"mcp_card":false,"api_catalog":false,"a2a_card":false},"any_capability":false,"ai_refused":[],"ai_thin":[],"sf":[4,0,0,0,0,0,1,0,0],"rings":{"mcp_declared":false,"mcp_live":false,"mcp_auth":false,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":3,"fh":"ya.ru","fp_other_host":[],"fp":"9f632e9a74d51c22","fp_bits":{"gptbot":0,"claudebot":0,"oaisearch":0,"perplexity":0}},"derive_v":5,"frame":"top10k","shard":3},{"v":1,"kind":"crawlcheck-census-record","run":18,"domain":"cloudflare-dns.com","rank":85,"measured_at":"2026-11-13T18:52:00.314Z","home":{"browser":{"status":200,"content_type":"text/html","bytes":229418,"sha256":"cb173d7e74602e093bb23f2f179313958328583f302d5623b9259c3ac1fb7e38","final_host":"one.one.one.one","final_path":"/dns","ms":21,"identity":"Mobile browser","html":true,"fp":"155b3d20036ed672"},"gptbot":{"status":200,"content_type":"text/html","bytes":229418,"sha256":"cb173d7e74602e093bb23f2f179313958328583f302d5623b9259c3ac1fb7e38","final_host":"one.one.one.one","final_path":"/dns","ms":19,"identity":"GPTBot","html":true,"fp":"155b3d20036ed672"},"claudebot":{"status":200,"content_type":"text/html","bytes":229418,"sha256":"cb173d7e74602e093bb23f2f179313958328583f302d5623b9259c3ac1fb7e38","final_host":"one.one.one.one","final_path":"/dns","ms":19,"identity":"ClaudeBot","html":true,"fp":"155b3d20036ed672"},"oaisearch":{"status":200,"content_type":"text/html","bytes":229418,"sha256":"cb173d7e74602e093bb23f2f179313958328583f302d5623b9259c3ac1fb7e38","final_host":"one.one.one.one","final_path":"/dns","ms":19,"identity":"OAI-SearchBot","html":true,"fp":"155b3d20036ed672"},"perplexity":{"status":200,"content_type":"text/html","bytes":229418,"sha256":"cb173d7e74602e093bb23f2f179313958328583f302d5623b9259c3ac1fb7e38","final_host":"one.one.one.one","final_path":"/dns","ms":19,"identity":"PerplexityBot","html":true,"fp":"155b3d20036ed672"}},"robots":{"status":200,"content_type":"text/plain","bytes":1248,"sha256":"8fa3036c68bfcbd32365f6225d24333264093b7cd38d306e106b4dbdc934fd5b","final_host":"one.one.one.one","final_path":"/robots.txt","ms":16,"html":false,"text":"# As a condition of accessing this website, you agree to abide by the following\n# content signals:\n\n# (a)  If a content-signal = yes, you may collect content for the corresponding\n#      use.\n# (b)  If a content-signal = no, you may not collect content for the\n#      corresponding use.\n# (c)  If the website operator does not include a content signal for a\n#      corresponding use, the website operator neither grants nor restricts\n#      permission via content signal with respect to the corresponding use.\n\n# The content signals and their meanings are:\n\n# search:   building a search index and providing search results (e.g., returning\n#           hyperlinks and short excerpts from your website's contents). Search does not\n#           include providing AI-generated search summaries.\n# ai-input: inputting content into one or more AI models (e.g., retrieval\n#           augmented generation, grounding, or other real-time taking of content for\n#           generative AI search answers).\n# ai-train: training or fine-tuning AI models.\n\n# ANY RESTRICTIONS EXPRESSED VIA CONTENT SIGNALS ARE EXPRESS RESERVATIONS OF\n# RIGHTS UNDER ARTICLE 4 OF THE EUROPEAN UNION DIRECTIVE 2019/790 ON COPYRIGHT\n# AND RELATED RIGHTS IN THE DIGITAL SINGLE MARKET.\n","agents":{"GPTBot":true,"OAI-SearchBot":true,"ChatGPT-User":true,"ClaudeBot":true,"Claude-SearchBot":true,"PerplexityBot":true,"Google-Extended":true,"Applebot-Extended":true,"CCBot":true,"Bytespider":true,"Meta-ExternalAgent":true,"Amazonbot":true}},"sitemap":{"status":404,"content_type":"text/html","bytes":1285,"sha256":"8017fa1f7764a80571cc0a99c7dbdfaa33dad3ec31f19239d56d38b0a6f421fe","final_host":"one.one.one.one","final_path":"/sitemap.xml","ms":19,"url":"https://cloudflare-dns.com/sitemap.xml","declared":false,"xml":false},"files":{"llms_txt":{"status":404,"content_type":"text/html","bytes":1285,"sha256":"8017fa1f7764a80571cc0a99c7dbdfaa33dad3ec31f19239d56d38b0a6f421fe","final_host":"one.one.one.one","final_path":"/llms.txt","ms":7,"path":"/llms.txt","html":true,"json":null,"present":false},"agents_md":{"status":404,"content_type":"text/html","bytes":1285,"sha256":"8017fa1f7764a80571cc0a99c7dbdfaa33dad3ec31f19239d56d38b0a6f421fe","final_host":"one.one.one.one","final_path":"/agents.md","ms":7,"path":"/agents.md","html":true,"json":null,"present":false},"agent_skills":{"status":404,"content_type":"text/html","bytes":1285,"sha256":"8017fa1f7764a80571cc0a99c7dbdfaa33dad3ec31f19239d56d38b0a6f421fe","final_host":"one.one.one.one","final_path":"/.well-known/agent-skills/index.json","ms":22,"path":"/.well-known/agent-skills/index.json","html":true,"json":null,"present":false},"mcp_card":{"status":404,"content_type":"text/html","bytes":1285,"sha256":"8017fa1f7764a80571cc0a99c7dbdfaa33dad3ec31f19239d56d38b0a6f421fe","final_host":"one.one.one.one","final_path":"/.well-known/mcp/server-card.json","ms":21,"path":"/.well-known/mcp/server-card.json","html":true,"json":null,"present":false},"api_catalog":{"status":404,"content_type":"text/html","bytes":1285,"sha256":"8017fa1f7764a80571cc0a99c7dbdfaa33dad3ec31f19239d56d38b0a6f421fe","final_host":"one.one.one.one","final_path":"/.well-known/api-catalog","ms":20,"path":"/.well-known/api-catalog","html":true,"json":null,"present":false},"a2a_card":{"status":404,"content_type":"text/html","bytes":1285,"sha256":"8017fa1f7764a80571cc0a99c7dbdfaa33dad3ec31f19239d56d38b0a6f421fe","final_host":"one.one.one.one","final_path":"/.well-known/agent-card.json","ms":27,"path":"/.well-known/agent-card.json","html":true,"json":null,"present":false}},"rings":{"v":1,"candidates":[],"mcp":[],"a2a":null,"webmcp":{"model_context":false,"bridge":false,"mcp_json":false,"link_rel":false,"any":false},"handshakes":0,"get_path":{"status":404,"content_type":"text/html","ms":28,"error":null},"get_host":{"status":404,"content_type":"text/html","ms":25,"error":null}},"sf":[1,0,0,0,0,0,2,0,1],"derived":{"reachable":true,"home_class":"answers","robots_served":true,"robots_sitemap":false,"content_signal":false,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":[],"blocks_any_ai":false,"files":{"llms_txt":false,"agents_md":false,"agent_skills":false,"mcp_card":false,"api_catalog":false,"a2a_card":false},"any_capability":false,"ai_refused":[],"ai_thin":[],"aipref":0,"aipref_measured":true,"sf":[1,0,0,0,0,0,2,0,1],"rings":{"mcp_declared":false,"mcp_live":false,"mcp_auth":false,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":3,"fh":"one.one.one.one","fp_other_host":[],"fp":"155b3d20036ed672","fp_bits":{"gptbot":0,"claudebot":0,"oaisearch":0,"perplexity":0}},"derive_v":5,"frame":"top10k","shard":4},{"v":1,"kind":"crawlcheck-census-record","run":18,"domain":"opera.com","rank":86,"measured_at":"2026-11-13T18:53:20.684Z","home":{"browser":{"status":200,"content_type":"text/html","bytes":196488,"sha256":"5b6623ff26dfa243cf29eb23c9f4e1f4eb0bcc22011984d027a90a0ed9ffd609","final_host":"www.opera.com","final_path":"/","ms":3098,"identity":"Mobile browser","html":true,"fp":"aa2201a8171172bc"},"gptbot":{"status":200,"content_type":"text/html","bytes":193302,"sha256":"a4e90ac2f2f14aa5cdcde0d52d329c9f3f9a890b906ad893cbb1f900ed774f3b","final_host":"www.opera.com","final_path":"/","ms":3118,"identity":"GPTBot","html":true,"fp":"aa3201a8153172bc"},"claudebot":{"status":200,"content_type":"text/html","bytes":193302,"sha256":"a4e90ac2f2f14aa5cdcde0d52d329c9f3f9a890b906ad893cbb1f900ed774f3b","final_host":"www.opera.com","final_path":"/","ms":3117,"identity":"ClaudeBot","html":true,"fp":"aa3201a8153172bc"},"oaisearch":{"status":200,"content_type":"text/html","bytes":193302,"sha256":"a4e90ac2f2f14aa5cdcde0d52d329c9f3f9a890b906ad893cbb1f900ed774f3b","final_host":"www.opera.com","final_path":"/","ms":3101,"identity":"OAI-SearchBot","html":true,"fp":"aa3201a8153172bc"},"perplexity":{"status":200,"content_type":"text/html","bytes":193278,"sha256":"101c01a6d3271d12d29c697cc489378b5cff62e9c1a3f146406ba261a3cb05fa","final_host":"www.opera.com","final_path":"/","ms":3093,"identity":"PerplexityBot","html":true,"fp":"aa3201a8153172bc"}},"robots":{"status":200,"content_type":"text/plain","bytes":994,"sha256":"0c6b878d90d9783f0a0972e38888f8dfef8393e317be80a266a82f3da548f20f","final_host":"www.opera.com","final_path":"/robots.txt","ms":2695,"html":false,"text":"User-agent: *\nAllow: /help/tutorials/\nAllow: /browser/tutorials/\nAllow: /support/tutorials/\nAllow: /lp/br-themes\nAllow: /pt-br/lp/br-themes\nAllow: /lp/video-streaming\nAllow: /*/lp/video-streaming\nDisallow: /o/\nDisallow: /*/o/\nDisallow: /abtest/\nDisallow: /*/abtest/\nDisallow: /api/\nDisallow: /client/\nDisallow: /downloadassets/\nDisallow: /*/downloadassets/\nDisallow: /download/get*\nDisallow: /get$\nDisallow: /get/\nDisallow: /promoassets/\nDisallow: /*/promoassets/\nDisallow: /campaign/\nDisallow: /*/campaign/\nDisallow: /suite/\nDisallow: /*/suite/\nDisallow: /info/\nDisallow: /*/info/\nDisallow: /computer/thanks\nDisallow: /*/computer/thanks\nDisallow: /computer/thanks-partner\nDisallow: /*/computer/thanks-partner\nDisallow: /ofa/\nDisallow: /*/ofa/\nDisallow: /oma/\nDisallow: /*/oma/\nDisallow: */tlp/*\nDisallow: */lp/*\nDisallow: /tutorials\nDisallow: /*/tutorials\nDisallow: /opera/features\nDisallow: /*/opera/features\nDisallow: /opera-coupons-uninstall\nSitemap: https://www.opera.com/sitemap-index.xml","agents":{"GPTBot":true,"OAI-SearchBot":true,"ChatGPT-User":true,"ClaudeBot":true,"Claude-SearchBot":true,"PerplexityBot":true,"Google-Extended":true,"Applebot-Extended":true,"CCBot":true,"Bytespider":true,"Meta-ExternalAgent":true,"Amazonbot":true}},"sitemap":{"status":200,"content_type":"application/xml","bytes":314,"sha256":"8eb2cdb83e21bab426f66f7b493ba60caa5b818a2c1a4f6e8b6fbbc1992dc718","final_host":"www.opera.com","final_path":"/sitemap-index.xml","ms":1026,"url":"https://www.opera.com/sitemap-index.xml","declared":true,"xml":true},"files":{"llms_txt":{"status":200,"content_type":"text/plain","bytes":11613,"sha256":"abefbde9fb5c1ef40e063c0d21aeff38c71e1f9899b3b7cc8d924a89deffb013","final_host":"www.opera.com","final_path":"/llms.txt","ms":489,"path":"/llms.txt","html":false,"json":null,"present":true},"agents_md":{"status":404,"content_type":"text/html","bytes":117760,"sha256":"a69ca050e42b2b4b47c61265f5fc0e53bdc4776d18187f2f92ba4652b3adaba7","final_host":"www.opera.com","final_path":"/agents.md","ms":1487,"path":"/agents.md","html":true,"json":null,"present":false},"agent_skills":{"status":404,"content_type":"text/html","bytes":117812,"sha256":"35adebb86067a99a0daa438f31ca390ff932694fbc679475a838dd982045c520","final_host":"www.opera.com","final_path":"/.well-known/agent-skills/index.json","ms":1475,"path":"/.well-known/agent-skills/index.json","html":true,"json":null,"present":false},"mcp_card":{"status":404,"content_type":"text/html","bytes":117806,"sha256":"04e0674b696651ca732bd095c2ef74db96eb540f1f03ebc9a2c53bea30b830c0","final_host":"www.opera.com","final_path":"/.well-known/mcp/server-card.json","ms":486,"path":"/.well-known/mcp/server-card.json","html":true,"json":null,"present":false},"api_catalog":{"status":404,"content_type":"text/html","bytes":117788,"sha256":"c67f0728407c8d85d5ac45077b09c6418eede342220b321cb80815235f636509","final_host":"www.opera.com","final_path":"/.well-known/api-catalog","ms":480,"path":"/.well-known/api-catalog","html":true,"json":null,"present":false},"a2a_card":{"status":404,"content_type":"text/html","bytes":117796,"sha256":"8bd1d7da81552813d43a64dbcc73dfd11772fabeaca52b65a02f0c04eb13e5e6","final_host":"www.opera.com","final_path":"/.well-known/agent-card.json","ms":1914,"path":"/.well-known/agent-card.json","html":true,"json":null,"present":false}},"rings":{"v":1,"candidates":[],"mcp":[],"a2a":null,"webmcp":{"model_context":false,"bridge":false,"mcp_json":false,"link_rel":false,"any":false},"handshakes":0,"get_path":{"status":404,"content_type":"text/html","ms":495,"error":null},"get_host":{"status":530,"content_type":"text/plain","ms":241,"error":null}},"sf":[1,0,0,0,0,0,1,0,0],"derived":{"reachable":true,"home_class":"answers","robots_served":true,"robots_sitemap":true,"content_signal":false,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":[],"blocks_any_ai":false,"files":{"llms_txt":true,"agents_md":false,"agent_skills":false,"mcp_card":false,"api_catalog":false,"a2a_card":false},"any_capability":false,"ai_refused":[],"ai_thin":[],"aipref":0,"aipref_measured":true,"sf":[1,0,0,0,0,0,1,0,0],"rings":{"mcp_declared":false,"mcp_live":false,"mcp_auth":false,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":3,"fh":"www.opera.com","fp_other_host":[],"fp":"aa2201a8171172bc","fp_bits":{"gptbot":3,"claudebot":3,"oaisearch":3,"perplexity":3}},"derive_v":5,"frame":"top10k","shard":5},{"v":1,"kind":"crawlcheck-census-record","run":18,"domain":"vimeo.com","rank":87,"measured_at":"2026-11-13T18:53:42.755Z","home":{"browser":{"status":200,"content_type":"text/html","bytes":1758715,"sha256":"ab56ae01ae91b55d4123f14d57723cd0a172a74bb7c0e432790f52076880587a","final_host":"vimeo.com","final_path":"/","ms":62,"identity":"Mobile browser","html":true,"fp":"e699fe5c0d05a34a"},"gptbot":{"status":200,"content_type":"text/html","bytes":1758715,"sha256":"ab56ae01ae91b55d4123f14d57723cd0a172a74bb7c0e432790f52076880587a","final_host":"vimeo.com","final_path":"/","ms":103,"identity":"GPTBot","html":true,"fp":"e699fe5c0d05a34a"},"claudebot":{"status":200,"content_type":"text/html","bytes":1758715,"sha256":"ab56ae01ae91b55d4123f14d57723cd0a172a74bb7c0e432790f52076880587a","final_host":"vimeo.com","final_path":"/","ms":62,"identity":"ClaudeBot","html":true,"fp":"e699fe5c0d05a34a"},"oaisearch":{"status":200,"content_type":"text/html","bytes":1758715,"sha256":"ab56ae01ae91b55d4123f14d57723cd0a172a74bb7c0e432790f52076880587a","final_host":"vimeo.com","final_path":"/","ms":89,"identity":"OAI-SearchBot","html":true,"fp":"e699fe5c0d05a34a"},"perplexity":{"status":200,"content_type":"text/html","bytes":1758715,"sha256":"ab56ae01ae91b55d4123f14d57723cd0a172a74bb7c0e432790f52076880587a","final_host":"vimeo.com","final_path":"/","ms":62,"identity":"PerplexityBot","html":true,"fp":"e699fe5c0d05a34a"}},"robots":{"status":200,"content_type":"text/plain","bytes":30242,"sha256":"b076c41a641a33acc075b9275c9edb0cb100308adcf93a9ee4ca326a005b81a9","final_host":"vimeo.com","final_path":"/robots.txt","ms":61,"html":false,"text":"# Notice: The use of robots, scrapers, or other automated means to access Vimeo \n# without the express written permission of Vimeo is strictly prohibited.\n# See https://vimeo.com/terms#acceptable_use_policy\n# Vimeo permits limited automated access to certain public-facing pages by approved services,\n# including search engines and select AI assistants, in accordance with this robots.txt file.\n# Access to user-generated content, including videos, user profiles, comments, and metadata, \n# is not permitted for scraping, harvesting, or use in training machine learning models.\n# For more on Vimeo's position on AI and content usage, see https://vimeo.com/blog/post/vimeos-position-on-ai\n\nUser-agent: *\nDisallow: /*/format:thumbnail\nDisallow: /download/\nDisallow: /*/download?\nDisallow: /couchmode/\nDisallow: /groups/*/forum/*\nDisallow: /musicstore/preview\nDisallow: /musicstore/download\nDisallow: /forgot_password\nDisallow: /help/forgot_login\nDisallow: /help/verify_email\nDisallow: /help/zendesk_sso\nDisallow: /help/sso\nDisallow: /user/verify\nDisallow: /search/\nDisallow: /search?\nDisallow: /search\\?\nDisallow: /_\nDisallow: /*mkc=*&\nDisallow: /*&mkc=*\nDisallow: /*gl=*&\nDisallow: /*&gl=*\nDisallow: /_global_notifications\nDisallow: /*?*referrer_vuid=\nDisallow: /*?*vimeo_search\nDisallow: /store/\nDisallow: /*?*_gl=\nDisallow: /*/collections/channels\nDisallow: /manage/videos/\nDisallow: /flarepoint/\nDisallow: /*/likes \nDisallow: /reviews/\nDisallow: /*/sort:*/*:\nDisallow: /ablincoln/\nDisallow: /*tag:\nDisallow: /*/videos/\nDisallow: /legal/~gitbook/\nDisallow: /manage/showcase\nAllow: /api/*/likes \nDisallow: /api/v3/me$\nDisallow: /api/v3/me/\nDisallow: /api/v3/me?\nDisallow: /api/v3/modworld\nDisallow: /api/v3/*/comments\nAllow: /_rv/viewer\nAllow: /_next/viewer\nAllow: /_rv/gtm\nAllow: /_rv/jwt\nAllow: /_next/jwt\nAllow: /_rv/title\nAllow: /_next/title\n# Clip Page Parameters\nDisallow: /*?_=\nDisallow: /*?_hsenc=\nDisallow: /*?_hsmi=\nDisallow: /*?*&_=\nDisallow: /*?*&_hsenc=\nDisallow: /*?*&_hsmi=\nDisallow: /*?*&1=\nDisallow: /*?*&activityReferer=\nDisallow: /*?*&adcPathname=\nDisallow: /*?*&ajax=\nDisallow: /*?*&allowfullscreen=\nDisallow: /*?*&autohide=\nDisallow: /*?*&autopause=\nDisallow: /*?*&background=\nDisallow: /*?*&badge=\nDisallow: /*?*&brid=\nDisallow: /*?*&byline=\nDisallow: /*?*&callback=\nDisallow: /*?*&cb=\nDisallow: /*?*&cc_load_policy=\nDisallow: /*?*&ceid=\nDisallow: /*?*&cjdata=\nDisallow: /*?*&cjevent=\nDisallow: /*?*&clickid=\nDisallow: /*?*&clipId=\nDisallow: /*?*&color=\nDisallow: /*?*&consentEnabled=\nDisallow: /*?*&content=\nDisallow: /*?*&context=\nDisallow: /*?*&controls=\nDisallow: /*?*&d=\nDisallow: /*?*&data=\nDisallow: /*?*&dnt=\nDisallow: /*?*&ei=\nDisallow: /*?*&ej=\nDisallow: /*?*&embed_email_provider=\nDisallow: /*?*&emci=\nDisallow: /*?*&emdi=\nDisallow: /*?*&enablejsapi=\nDisallow: /*?*&f1=\nDisallow: /*?*&filter_location=\nDisallow: /*?*&filter_skills=\nDisallow: /*?*&fit=\nDisallow: /*?*&flashvars=\nDisallow: /*?*&foe=\nDisallow: /*?*&format=\nDisallow: /*?*&from=\nDisallow: /*?*&fromApp=\nDisallow: /*?*&fullscreen=\nDisallow: /*?*&h=\nDisallow: /*?*&hashKey=\nDisallow: /*?*&hd=\nDisallow: /*?*&height=\nDisallow: /*?*&hss_meta=\nDisallow: /*?*&id=\nDisallow: /*?*&iframe=\nDisallow: /*?*&itok=\nDisallow: /*?*&keepEmbedCode=\nDisallow: /*?*&keyboard=\nDisallow: /*?1=\nDisallow: /*?activityReferer=\nDisallow: /*?adcPathname=\nDisallow: /*?ajax=\nDisallow: /*?allowfullscreen=\nDisallow: /*?autohide=\nDisallow: /*?autopause=\nDisallow: /*?background=\nDisallow: /*?badge=\nDisallow: /*?brid=\nDisallow: /*?byline=\nDisallow: /*?callback=\nDisallow: /*?cb=\nDisallow: /*?cc_load_policy=\nDisallow: /*?ceid=\nDisallow: /*?cjdata=\nDisallow: /*?cjevent=\nDisallow: /*?clickid=\nDisallow: /*?clipId=\nDisallow: /*?color=\nDisallow: /*?consentEnabled=\nDisallow: /*?content=\nDisallow: /*?context=\nDisallow: /*?controls=\nDisallow: /*?d=\nDisallow: /*?data=\nDisallow: /*?dnt=\nDisallow: /*?ei=\nDisallow: /*?ej=\nDisallow: /*?embed_email_provider=\nDisallow: /*?emci=\nDisallow: /*?emdi=\nDisallow: /*?enablejsapi=\nDisallow: /*?f1=\nDisallow: /*?filter_location=\nDisallow: /*?filter_skills=\nDisallow: /*?fit=\nDisallow: /*?flashvars=\nDisallow: /*?foe=\nDisallow: /*?format=\nDisallow: /*?from=\nDisallow: /*?fromApp=\nDisallow: /*?fullscreen=\nDisallow: /*?h=\nDisallow: /*?hashKey=\nDisallow: /*?hd=\nDisallow: /*?height=\nDisallow: /*?hss_meta=\nDisallow: /*?id=\nDisallow: /*?iframe=\nDisallow: /*?itok=\nDisallow: /*?keepEmbedCode=\nDisallow: /*?keyboard=\nDisallow: /*?*&%3Bfe=\nDisallow: /*?*&%3Bfl=\nDisallow: /*?*&kref=\nDisallow: /*?*&kuid=\nDisallow: /*?*&lang=\nDisallow: /*?*&lightbox[modal]=\nDisallow: /*?*&linkId=\nDisallow: /*?*&lite=\nDisallow: /*?*&login=\nDisallow: /*?*&loop=\nDisallow: /*?*&maxheight=\nDisallow: /*?*&maxw=\nDisallow: /*?*&maxwidth=\nDisallow: /*?*&mc_cid=\nDisallow: /*?*&method=\nDisallow: /*?*&mibextid=\nDisallow: /*?*&modestbranding=\nDisallow: /*?*&ms=\nDisallow: /*?*&msockid=\nDisallow: /*?*&mute=\nDisallow: /*?*&odnBg=\nDisallow: /*?*&odnHeight=\nDisallow: /*?*&odnWidth=\nDisallow: /*?*&output=\nDisallow: /*?*&outro=\nDisallow: /*?*&per_page=\nDisallow: /*?*&pgroup=\nDisallow: /*?*&platform=\nDisallow: /*?*&play=\nDisallow: /*?*&player_id=\nDisallow: /*?*&playerapiid=\nDisallow: /*?*&playlist=\nDisallow: /*?*&playsinline=\nDisallow: /*?*&portrait=\nDisallow: /*?*&progress_bar=\nDisallow: /*?*&quality_selector=\nDisallow: /*?*&quality=\nDisallow: /*?*&ra=\nDisallow: /*?*&random=\nDisallow: /*?*&redirect=\nDisallow: /*?*&rel=\nDisallow: /*?*&resize=\nDisallow: /*?*&responsive=\nDisallow: /*?*&rmxv=\nDisallow: /*?*&rt=\nDisallow: /*?*&rv=\nDisallow: /*?*&s=\nDisallow: /*?*&sa=\nDisallow: /*?*&scrlybrkr=\nDisallow: /*?*&sd=\nDisallow: /*?*&server=\nDisallow: /*?*&sfmc_id=\nDisallow: /*?*&show_byline=\nDisallow: /*?*&show_portrait=\nDisallow: /*?*&show_title=\nDisallow: /*?*&showinfo=\nDisallow: /*?*&sig=\nDisallow: /*?*&signup=\nDisallow: /*?*&sizes=\nDisallow: /*?*&sl_key=\nDisallow: /*?*&sort=\nDisallow: /*?*&speed=\nDisallow: /*?*&sr=\nDisallow: /*?*&ssl=\nDisallow: /*?*&ssp_iabi=\nDisallow: /*?*&start=\nDisallow: /*?*&streamid=\nDisallow: /*?*&sv=\nDisallow: /*?*&syclid=\nDisallow: /*?*&tblci=\nDisallow: /*?*&test=\nDisallow: /*?*&texttrack=\nDisallow: /*?*&title=\nDisallow: /*?*&tq=\nDisallow: /*?*&transparent=\nDisallow: /*?*&trk=\nDisallow: /*?*&type=\nDisallow: /*?*&url=\nDisallow: /*?*&user_id=\nDisallow: /*?*&usg=\nDisallow: /*?*&utm_date=\nDisallow: /*?*&utm_id=\nDisallow: /*?*&v=\nDisallow: /*?*&version=\nDisallow: /*?*&volumeOff=\nDisallow: /*?*&vq=\nDisallow: /*?*&w=\nDisallow: /*?*&ysclid=\nDisallow: /*?%3Bfe=\nDisallow: /*?%3Bfl=\nDisallow: /*?kref=\nDisallow: /*?kuid=\nDisallow: /*?lang=\nDisallow: /*?lightbox[modal]=\nDisallow: /*?linkId=\nDisallow: /*?lite=\nDisallow: /*?login=\nDisallow: /*?loop=\nDisallow: /*?maxheight=\nDisallow: /*?maxw=\nDisallow: /*?maxwidth=\nDisallow: /*?mc_cid=\nDisallow: /*?method=\nDisallow: /*?mibextid=\nDisallow: /*?modestbranding=\nDisallow: /*?ms=\nDisallow: /*?msockid=\nDisallow: /*?mute=\nDisallow: /*?odnBg=\nDisallow: /*?odnHeight=\nDisallow: /*?odnWidth=\nDisallow: /*?output=\nDisallow: /*?outro=\nDisallow: /*?per_page=\nDisallow: /*?pgroup=\nDisallow: /*?platform=\nDisallow: /*?play=\nDisallow: /*?player_id=\nDisallow: /*?playerapiid=\nDisallow: /*?playlist=\nDisallow: /*?playsinline=\nDisallow: /*?portrait=\nDisallow: /*?progress_bar=\nDisallow: /*?quality_selector=\nDisallow: /*?quality=\nDisallow: /*?ra=\nDisallow: /*?random=\nDisallow: /*?redirect=\nDisallow: /*?rel=\nDisallow: /*?resize=\nDisallow: /*?responsive=\nDisallow: /*?rmxv=\nDisallow: /*?rt=\nDisallow: /*?rv=\nDisallow: /*?s=\nDisallow: /*?sa=\nDisallow: /*?scrlybrkr=\nDisallow: /*?sd=\nDisallow: /*?server=\nDisallow: /*?sfmc_id=\nDisallow: /*?show_byline=\nDisallow: /*?show_portrait=\nDisallow: /*?show_title=\nDisallow: /*?showinfo=\nDisallow: /*?sig=\nDisallow: /*?signup=\nDisallow: /*?sizes=\nDisallow: /*?sl_key=\nDisallow: /*?sort=\nDisallow: /*?speed=\nDisallow: /*?sr=\nDisallow: /*?ssl=\nDisallow: /*?ssp_iabi=\nDisallow: /*?start=\nDisallow: /*?streamid=\nDisallow: /*?sv=\nDisallow: /*?syclid=\nDisallow: /*?tblci=\nDisallow: /*?test=\nDisallow: /*?texttrack=\nDisallow: /*?title=\nDisallow: /*?tq=\nDisallow: /*?transparent=\nDisallow: /*?trk=\nDisallow: /*?type=\nDisallow: /*?url=\nDisallow: /*?user_id=\nDisallow: /*?usg=\nDisallow: /*?utm_date=\nDisallow: /*?utm_id=\nDisallow: /*?v=\nDisallow: /*?version=\nDisallow: /*?volumeOff=\nDisallow: /*?vq=\nDisallow: /*?w=\nDisallow: /*?ysclid=\nAllow: /*/*?_=\nAllow: /*/*?_hsenc=\nAllow: /*/*?_hsmi=\nAllow: /*/*?*&_=\nAllow: /*/*?*&_hsenc=\nAllow: /*/*?*&_hsmi=\nAllow: /*/*?*&1=\nAllow: /*/*?*&activityReferer=\nAllow: /*/*?*&adcPathname=\nAllow: /*/*?*&ajax=\nAllow: /*/*?*&allowfullscreen=\nAllow: /*/*?*&autohide=\nAllow: /*/*?*&autopause=\nAllow: /*/*?*&background=\nAllow: /*/*?*&badge=\nAllow: /*/*?*&brid=\nAllow: /*/*?*&byline=\nAllow: /*/*?*&callback=\nAllow: /*/*?*&cb=\nAllow: /*/*?*&cc_load_policy=\nAllow: /*/*?*&ceid=\nAllow: /*/*?*&cjdata=\nAllow: /*/*?*&cjevent=\nAllow: /*/*?*&clickid=\nAllow: /*/*?*&clipId=\nAllow: /*/*?*&color=\nAllow: /*/*?*&consentEnabled=\nAllow: /*/*?*&content=\nAllow: /*/*?*&context=\nAllow: /*/*?*&controls=\nAllow: /*/*?*&d=\nAllow: /*/*?*&data=\nAllow: /*/*?*&dnt=\nAllow: /*/*?*&ei=\nAllow: /*/*?*&ej=\nAllow: /*/*?*&embed_email_provider=\nAllow: /*/*?*&emci=\nAllow: /*/*?*&emdi=\nAllow: /*/*?*&enablejsapi=\nAllow: /*/*?*&f1=\nAllow: /*/*?*&filter_location=\nAllow: /*/*?*&filter_skills=\nAllow: /*/*?*&fit=\nAllow: /*/*?*&flashvars=\nAllow: /*/*?*&foe=\nAllow: /*/*?*&format=\nAllow: /*/*?*&from=\nAllow: /*/*?*&fromApp=\nAllow: /*/*?*&fullscreen=\nAllow: /*/*?*&h=\nAllow: /*/*?*&hashKey=\nAllow: /*/*?*&hd=\nAllow: /*/*?*&height=\nAllow: /*/*?*&hss_meta=\nAllow: /*/*?*&id=\nAllow: /*/*?*&iframe=\nAllow: /*/*?*&itok=\nAllow: /*/*?*&keepEmbedCode=\nAllow: /*/*?*&keyboard=\nAllow: /*/*?1=\nAllow: /*/*?activityReferer=\nAllow: /*/*?adcPathname=\nAllow: /*/*?ajax=\nAllow: /*/*?allowfullscreen=\nAllow: /*/*?autohide=\nAllow: /*/*?autopause=\nAllow: /*/*?background=\nAllow: /*/*?badge=\nAllow: /*/*?brid=\nAllow: /*/*?byline=\nAllow: /*/*?callback=\nAllow: /*/*?cb=\nAllow: /*/*?cc_load_policy=\nAllow: /*/*?ceid=\nAllow: /*/*?cjdata=\nAllow: /*/*?cjevent=\nAllow: /*/*?clickid=\nAllow: /*/*?clipId=\nAllow: /*/*?color=\nAllow: /*/*?consentEnabled=\nAllow: /*/*?content=\nAllow: /*/*?context=\nAllow: /*/*?controls=\nAllow: /*/*?d=\nAllow: /*/*?data=\nAllow: /*/*?dnt=\nAllow: /*/*?ei=\nAllow: /*/*?ej=\nAllow: /*/*?embed_email_provider=\nAllow: /*/*?emci=\nAllow: /*/*?emdi=\nAllow: /*/*?enablejsapi=\nAllow: /*/*?f1=\nAllow: /*/*?filter_location=\nAllow: /*/*?filter_skills=\nAllow: /*/*?fit=\nAllow: /*/*?flashvars=\nAllow: /*/*?foe=\nAllow: /*/*?format=\nAllow: /*/*?from=\nAllow: /*/*?fromApp=\nAllow: /*/*?fullscreen=\nAllow: /*/*?h=\nAllow: /*/*?hashKey=\nAllow: /*/*?hd=\nAllow: /*/*?height=\nAllow: /*/*?hss_meta=\nAllow: /*/*?id=\nAllow: /*/*?iframe=\nAllow: /*/*?itok=\nAllow: /*/*?keepEmbedCode=\nAllow: /*/*?keyboard=\nAllow: /*/*?*&%3Bfe=\nAllow: /*/*?*&%3Bfl=\nAllow: /*/*?*&kref=\nAllow: /*/*?*&kuid=\nAllow: /*/*?*&lang=\nAllow: /*/*?*&lightbox[modal]=\nAllow: /*/*?*&linkId=\nAllow: /*/*?*&lite=\nAllow: /*/*?*&login=\nAllow: /*/*?*&loop=\nAllow: /*/*?*&maxheight=\nAllow: /*/*?*&maxw=\nAllow: /*/*?*&maxwidth=\nAllow: /*/*?*&mc_cid=\nAllow: /*/*?*&method=\nAllow: /*/*?*&mibextid=\nAllow: /*/*?*&modestbranding=\nAllow: /*/*?*&ms=\nAllow: /*/*?*&msockid=\nAllow: /*/*?*&mute=\nAllow: /*/*?*&odnBg=\nAllow: /*/*?*&odnHeight=\nAllow: /*/*?*&odnWidth=\nAllow: /*/*?*&output=\nAllow: /*/*?*&outro=\nAllow: /*/*?*&per_page=\nAllow: /*/*?*&pgroup=\nAllow: /*/*?*&platform=\nAllow: /*/*?*&play=\nAllow: /*/*?*&player_id=\nAllow: /*/*?*&playerapiid=\nAllow: /*/*?*&playlist=\nAllow: /*/*?*&playsinline=\nAllow: /*/*?*&portrait=\nAllow: /*/*?*&progress_bar=\nAllow: /*/*?*&quality_selector=\nAllow: /*/*?*&quality=\nAllow: /*/*?*&ra=\nAllow: /*/*?*&random=\nAllow: /*/*?*&redirect=\nAllow: /*/*?*&rel=\nAllow: /*/*?*&resize=\nAllow: /*/*?*&responsive=\nAllow: /*/*?*&rmxv=\nAllow: /*/*?*&rt=\nAllow: /*/*?*&rv=\nAllow: /*/*?*&s=\nAllow: /*/*?*&sa=\nAllow: /*/*?*&scrlybrkr=\nAllow: /*/*?*&sd=\nAllow: /*/*?*&server=\nAllow: /*/*?*&sfmc_id=\nAllow: /*/*?*&show_byline=\nAllow: /*/*?*&show_portrait=\nAllow: /*/*?*&show_title=\nAllow: /*/*?*&showinfo=\nAllow: /*/*?*&sig=\nAllow: /*/*?*&signup=\nAllow: /*/*?*&sizes=\nAllow: /*/*?*&sl_key=\nAllow: /*/*?*&sort=\nAllow: /*/*?*&speed=\nAllow: /*/*?*&sr=\nAllow: /*/*?*&ssl=\nAllow: /*/*?*&ssp_iabi=\nAllow: /*/*?*&start=\nAllow: /*/*?*&streamid=\nAllow: /*/*?*&sv=\nAllow: /*/*?*&syclid=\nAllow: /*/*?*&tblci=\nAllow: /*/*?*&test=\nAllow: /*/*?*&texttrack=\nAllow: /*/*?*&title=\nAllow: /*/*?*&tq=\nAllow: /*/*?*&transparent=\nAllow: /*/*?*&trk=\nAllow: /*/*?*&type=\nAllow: /*/*?*&url=\nAllow: /*/*?*&user_id=\nAllow: /*/*?*&usg=\nAllow: /*/*?*&utm_date=\nAllow: /*/*?*&utm_id=\nAllow: /*/*?*&v=\nAllow: /*/*?*&version=\nAllow: /*/*?*&volumeOff=\nAllow: /*/*?*&vq=\nAllow: /*/*?*&w=\nAllow: /*/*?*&ysclid=\nAllow: /*/*?%3Bfe=\nAllow: /*/*?%3Bfl=\nAllow: /*/*?kref=\nAllow: /*/*?kuid=\nAllow: /*/*?lang=\nAllow: /*/*?lightbox[modal]=\nAllow: /*/*?linkId=\nAllow: /*/*?lite=\nAllow: /*/*?login=\nAllow: /*/*?loop=\nAllow: /*/*?maxheight=\nAllow: /*/*?maxw=\nAllow: /*/*?maxwidth=\nAllow: /*/*?mc_cid=\nAllow: /*/*?method=\nAllow: /*/*?mibextid=\nAllow: /*/*?modestbranding=\nAllow: /*/*?ms=\nAllow: /*/*?msockid=\nAllow: /*/*?mute=\nAllow: /*/*?odnBg=\nAllow: /*/*?odnHeight=\nAllow: /*/*?odnWidth=\nAllow: /*/*?output=\nAllow: /*/*?outro=\nAllow: /*/*?per_page=\nAllow: /*/*?pgroup=\nAllow: /*/*?platform=\nAllow: /*/*?play=\nAllow: /*/*?player_id=\nAllow: /*/*?playerapiid=\nAllow: /*/*?playlist=\nAllow: /*/*?playsinline=\nAllow: /*/*?portrait=\nAllow: /*/*?progress_bar=\nAllow: /*/*?quality_selector=\nAllow: /*/*?quality=\nAllow: /*/*?ra=\nAllow: /*/*?random=\nAllow: /*/*?redirect=\nAllow: /*/*?rel=\nAllow: /*/*?resize=\nAllow: /*/*?responsive=\nAllow: /*/*?rmxv=\nAllow: /*/*?rt=\nAllow: /*/*?rv=\nAllow: /*/*?s=\nAllow: /*/*?sa=\nAllow: /*/*?scrlybrkr=\nAllow: /*/*?sd=\nAllow: /*/*?server=\nAllow: /*/*?sfmc_id=\nAllow: /*/*?show_byline=\nAllow: /*/*?show_portrait=\nAllow: /*/*?show_title=\nAllow: /*/*?showinfo=\nAllow: /*/*?sig=\nAllow: /*/*?signup=\nAllow: /*/*?sizes=\nAllow: /*/*?sl_key=\nAllow: /*/*?sort=\nAllow: /*/*?speed=\nAllow: /*/*?sr=\nAllow: /*/*?ssl=\nAllow: /*/*?ssp_iabi=\nAllow: /*/*?start=\nAllow: /*/*?streamid=\nAllow: /*/*?sv=\nAllow: /*/*?syclid=\nAllow: /*/*?tblci=\nAllow: /*/*?test=\nAllow: /*/*?texttrack=\nAllow: /*/*?title=\nAllow: /*/*?tq=\nAllow: /*/*?transparent=\nAllow: /*/*?trk=\nAllow: /*/*?type=\nAllow: /*/*?url=\nAllow: /*/*?user_id=\nAllow: /*/*?usg=\nAllow: /*/*?utm_date=\nAllow: /*/*?utm_id=\nAllow: /*/*?v=\nAllow: /*/*?version=\nAllow: /*/*?volumeOff=\nAllow: /*/*?vq=\nAllow: /*/*?w=\nAllow: /*/*?ysclid=\nCrawl-delay: 2\n\nUser-agent: ia_archiver\nDisallow: /api/v3\nDisallow: /*/format:thumbnail\nDisallow: /download/\nDisallow: /*/download?\nDisallow: /couchmode/\nDisallow: /groups/*/forum/*\nDisallow: /musicstore/preview\nDisallow: /musicstore/download\nDisallow: /forgot_password\nDisallow: /help/forgot_login\nDisallow: /help/verify_email\nDisallow: /help/zendesk_sso\nDisallow: /help/sso\nDisallow: /user/verify\nDisallow: /search/\nDisallow: /search?\nDisallow: /search\\?\nDisallow: /_\nDisallow: /*mkc=*&\nDisallow: /*&mkc=*\nDisallow: /*gl=*&\nDisallow: /*&gl=*\nDisallow: /_global_notifications\nDisallow: /*?*referrer_vuid=\nDisallow: /*?*vimeo_search\nDisallow: /store/\nDisallow: /*?*_gl=\nDisallow: /*/collections/channels\nDisallow: /manage/videos/\nDisallow: /flarepoint/\nDisallow: /*/likes \nDisallow: /reviews/\nDisallow: /*/sort:*/*:\nDisallow: /ablincoln/\nDisallow: /*tag:\nDisallow: /*/videos/\nDisallow: /legal/~gitbook/\nDisallow: /manage/showcase\nDisallow: /api/v3/me$\nDisallow: /api/v3/me/\nDisallow: /api/v3/me?\nDisallow: /api/v3/modworld\nDisallow: /api/v3/*/comments\n# Clip Page Parameters\nDisallow: /*?_=\nDisallow: /*?_hsenc=\nDisallow: /*?_hsmi=\nDisallow: /*?*&_=\nDisallow: /*?*&_hsenc=\nDisallow: /*?*&_hsmi=\nDisallow: /*?*&1=\nDisallow: /*?*&activityReferer=\nDisallow: /*?*&adcPathname=\nDisallow: /*?*&ajax=\nDisallow: /*?*&allowfullscreen=\nDisallow: /*?*&autohide=\nDisallow: /*?*&autopause=\nDisallow: /*?*&background=\nDisallow: /*?*&badge=\nDisallow: /*?*&brid=\nDisallow: /*?*&byline=\nDisallow: /*?*&callback=\nDisallow: /*?*&cb=\nDisallow: /*?*&cc_load_policy=\nDisallow: /*?*&ceid=\nDisallow: /*?*&cjdata=\nDisallow: /*?*&cjevent=\nDisallow: /*?*&clickid=\nDisallow: /*?*&clipId=\nDisallow: /*?*&color=\nDisallow: /*?*&consentEnabled=\nDisallow: /*?*&content=\nDisallow: /*?*&context=\nDisallow: /*?*&controls=\nDisallow: /*?*&d=\nDisallow: /*?*&data=\nDisallow: /*?*&dnt=\nDisallow: /*?*&ei=\nDisallow: /*?*&ej=\nDisallow: /*?*&embed_email_provider=\nDisallow: /*?*&emci=\nDisallow: /*?*&emdi=\nDisallow: /*?*&enablejsapi=\nDisallow: /*?*&f1=\nDisallow: /*?*&filter_location=\nDisallow: /*?*&filter_skills=\nDisallow: /*?*&fit=\nDisallow: /*?*&flashvars=\nDisallow: /*?*&foe=\nDisallow: /*?*&format=\nDisallow: /*?*&from=\nDisallow: /*?*&fromApp=\nDisallow: /*?*&fullscreen=\nDisallow: /*?*&h=\nDisallow: /*?*&hashKey=\nDisallow: /*?*&hd=\nDisallow: /*?*&height=\nDisallow: /*?*&hss_meta=\nDisallow: /*?*&id=\nDisallow: /*?*&iframe=\nDisallow: /*?*&itok=\nDisallow: /*?*&keepEmbedCode=\nDisallow: /*?*&keyboard=\nDisallow: /*?1=\nDisallow: /*?activityReferer=\nDisallow: /*?adcPathname=\nDisallow: /*?ajax=\nDisallow: /*?allowfullscreen=\nDisallow: /*?autohide=\nDisallow: /*?autopause=\nDisallow: /*?background=\nDisallow: /*?badge=\nDisallow: /*?brid=\nDisallow: /*?byline=\nDisallow: /*?callback=\nDisallow: /*?cb=\nDisallow: /*?cc_load_policy=\nDisallow: /*?ceid=\nDisallow: /*?cjdata=\nDisallow: /*?cjevent=\nDisallow: /*?clickid=\nDisallow: /*?clipId=\nDisallow: /*?color=\nDisallow: /*?consentEnabled=\nDisallow: /*?content=\nDisallow: /*?context=\nDisallow: /*?controls=\nDisallow: /*?d=\nDisallow: /*?data=\nDisallow: /*?dnt=\nDisallow: /*?ei=\nDisallow: /*?ej=\nDisallow: /*?embed_email_provider=\nDisallow: /*?emci=\nDisallow: /*?emdi=\nDisallow: /*?enablejsapi=\nDisallow: /*?f1=\nDisallow: /*?filter_location=\nDisallow: /*?filter_skills=\nDisallow: /*?fit=\nDisallow: /*?flashvars=\nDisallow: /*?foe=\nDisallow: /*?format=\nDisallow: /*?from=\nDisallow: /*?fromApp=\nDisallow: /*?fullscreen=\nDisallow: /*?h=\nDisallow: /*?hashKey=\nDisallow: /*?hd=\nDisallow: /*?height=\nDisallow: /*?hss_meta=\nDisallow: /*?id=\nDisallow: /*?iframe=\nDisallow: /*?itok=\nDisallow: /*?keepEmbedCode=\nDisallow: /*?keyboard=\nDisallow: /*?*&%3Bfe=\nDisallow: /*?*&%3Bfl=\nDisallow: /*?*&kref=\nDisallow: /*?*&kuid=\nDisallow: /*?*&lang=\nDisallow: /*?*&lightbox[modal]=\nDisallow: /*?*&linkId=\nDisallow: /*?*&lite=\nDisallow: /*?*&login=\nDisallow: /*?*&loop=\nDisallow: /*?*&maxheight=\nDisallow: /*?*&maxw=\nDisallow: /*?*&maxwidth=\nDisallow: /*?*&mc_cid=\nDisallow: /*?*&method=\nDisallow: /*?*&mibextid=\nDisallow: /*?*&modestbranding=\nDisallow: /*?*&ms=\nDisallow: /*?*&msockid=\nDisallow: /*?*&mute=\nDisallow: /*?*&odnBg=\nDisallow: /*?*&odnHeight=\nDisallow: /*?*&odnWidth=\nDisallow: /*?*&output=\nDisallow: /*?*&outro=\nDisallow: /*?*&per_page=\nDisallow: /*?*&pgroup=\nDisallow: /*?*&platform=\nDisallow: /*?*&play=\nDisallow: /*?*&player_id=\nDisallow: /*?*&playerapiid=\nDisallow: /*?*&playlist=\nDisallow: /*?*&playsinline=\nDisallow: /*?*&portrait=\nDisallow: /*?*&progress_bar=\nDisallow: /*?*&quality_selector=\nDisallow: /*?*&quality=\nDisallow: /*?*&ra=\nDisallow: /*?*&random=\nDisallow: /*?*&redirect=\nDisallow: /*?*&rel=\nDisallow: /*?*&resize=\nDisallow: /*?*&responsive=\nDisallow: /*?*&rmxv=\nDisallow: /*?*&rt=\nDisallow: /*?*&rv=\nDisallow: /*?*&s=\nDisallow: /*?*&sa=\nDisallow: /*?*&scrlybrkr=\nDisallow: /*?*&sd=\nDisallow: /*?*&server=\nDisallow: /*?*&sfmc_id=\nDisallow: /*?*&show_byline=\nDisallow: /*?*&show_portrait=\nDisallow: /*?*&show_title=\nDisallow: /*?*&showinfo=\nDisallow: /*?*&sig=\nDisallow: /*?*&signup=\nDisallow: /*?*&sizes=\nDisallow: /*?*&sl_key=\nDisallow: /*?*&sort=\nDisallow: /*?*&speed=\nDisallow: /*?*&sr=\nDisallow: /*?*&ssl=\nDisallow: /*?*&ssp_iabi=\nDisallow: /*?*&start=\nDisallow: /*?*&streamid=\nDisallow: /*?*&sv=\nDisallow: /*?*&syclid=\nDisallow: /*?*&tblci=\nDisallow: /*?*&test=\nDisallow: /*?*&texttrack=\nDisallow: /*?*&title=\nDisallow: /*?*&tq=\nDisallow: /*?*&transparent=\nDisallow: /*?*&trk=\nDisallow: /*?*&type=\nDisallow: /*?*&url=\nDisallow: /*?*&user_id=\nDisallow: /*?*&usg=\nDisallow: /*?*&utm_date=\nDisallow: /*?*&utm_id=\nDisallow: /*?*&v=\nDisallow: /*?*&version=\nDisallow: /*?*&volumeOff=\nDisallow: /*?*&vq=\nDisallow: /*?*&w=\nDisallow: /*?*&ysclid=\nDisallow: /*?%3Bfe=\nDisallow: /*?%3Bfl=\nDisallow: /*?kref=\nDisallow: /*?kuid=\nDisallow: /*?lang=\nDisallow: /*?lightbox[modal]=\nDisallow: /*?linkId=\nDisallow: /*?lite=\nDisallow: /*?login=\nDisallow: /*?loop=\nDisallow: /*?maxheight=\nDisallow: /*?maxw=\nDisallow: /*?maxwidth=\nDisallow: /*?mc_cid=\nDisallow: /*?method=\nDisallow: /*?mibextid=\nDisallow: /*?modestbranding=\nDisallow: /*?ms=\nDisallow: /*?msockid=\nDisallow: /*?mute=\nDisallow: /*?odnBg=\nDisallow: /*?odnHeight=\nDisallow: /*?odnWidth=\nDisallow: /*?output=\nDisallow: /*?outro=\nDisallow: /*?per_page=\nDisallow: /*?pgroup=\nDisallow: /*?platform=\nDisallow: /*?play=\nDisallow: /*?player_id=\nDisallow: /*?playerapiid=\nDisallow: /*?playlist=\nDisallow: /*?playsinline=\nDisallow: /*?portrait=\nDisallow: /*?progress_bar=\nDisallow: /*?quality_selector=\nDisallow: /*?quality=\nDisallow: /*?ra=\nDisallow: /*?random=\nDisallow: /*?redirect=\nDisallow: /*?rel=\nDisallow: /*?resize=\nDisallow: /*?responsive=\nDisallow: /*?rmxv=\nDisallow: /*?rt=\nDisallow: /*?rv=\nDisallow: /*?s=\nDisallow: /*?sa=\nDisallow: /*?scrlybrkr=\nDisallow: /*?sd=\nDisallow: /*?server=\nDisallow: /*?sfmc_id=\nDisallow: /*?show_byline=\nDisallow: /*?show_portrait=\nDisallow: /*?show_title=\nDisallow: /*?showinfo=\nDisallow: /*?sig=\nDisallow: /*?signup=\nDisallow: /*?sizes=\nDisallow: /*?sl_key=\nDisallow: /*?sort=\nDisallow: /*?speed=\nDisallow: /*?sr=\nDisallow: /*?ssl=\nDisallow: /*?ssp_iabi=\nDisallow: /*?start=\nDisallow: /*?streamid=\nDisallow: /*?sv=\nDisallow: /*?syclid=\nDisallow: /*?tblci=\nDisallow: /*?test=\nDisallow: /*?texttrack=\nDisallow: /*?title=\nDisallow: /*?tq=\nDisallow: /*?transparent=\nDisallow: /*?trk=\nDisallow: /*?type=\nDisallow: /*?url=\nDisallow: /*?user_id=\nDisallow: /*?usg=\nDisallow: /*?utm_date=\nDisallow: /*?utm_id=\nDisallow: /*?v=\nDisallow: /*?version=\nDisallow: /*?volumeOff=\nDisallow: /*?vq=\nDisallow: /*?w=\nDisallow: /*?ysclid=\nCrawl-delay: 2\n\nUser-agent: SeznamBot\nRequest-rate: 1/10s\nDisallow: */format:thumbnail\nDisallow: /download/\nDisallow: /*/download?\nDisallow: /couchmode/\nDisallow: /_global_notifications\nDisallow: /musicstore/preview\nDisallow: /musicstore/download\nDisallow: /forgot_password\nDisallow: /help/forgot_login\nDisallow: /help/verify_email\nDisallow: /help/zendesk_sso\nDisallow: /help/sso\nDisallow: /api/\nDisallow: /flarepoint/\nDisallow: /user/verify\nDisallow: /search/\nDisallow: /search?\nDisallow: /search\\?\nDisallow: /_*\n\nUser-agent: msnbot\nCrawl-delay: 5\n\nUser-agent: Bingbot\nCrawl-delay: 5\n\nUser-agent: AdIdxBot\nCrawl-delay: 5\n\nUser-agent: BingPreview\nCrawl-delay: 5\n\nUser-agent: Yandex\nCrawl-delay: 2\n\n# Block Open AI - Allowing specific marketing/informational paths\nUser-agent: GPTBot\nDisallow: /\nAllow: /features/\nAllow: /solutions/\nAllow: /product/\nAllow: /video-uploader\nAllow: /upgrade-plan\nAllow: /enterprise/\nAllow: /integrations/\nAllow: /blog/\nAllow: /create/\nAllow: /ott/\nAllow: /customers/\nAllow: /events/\nAllow: /campaigns/\nAllow: /about/\nAllow: /vimeo-academy/\nAllow: /$\nAllow: /es/features/\nAllow: /es/solutions/\nAllow: /es/product/\nAllow: /es/video-uploader\nAllow: /es/enterprise/\nAllow: /es/integrations/\nAllow: /es/blog/\nAllow: /es/create/\nAllow: /es/ott/\nAllow: /es/customers/\nAllow: /es/events/\nAllow: /es/campaigns/\nAllow: /es/about/\nAllow: /es/vimeo-academy/\nAllow: /es/$\nAllow: /fr/features/\nAllow: /fr/solutions/\nAllow: /fr/product/\nAllow: /fr/video-uploader\nAllow: /fr/enterprise/\nAllow: /fr/integrations/\nAllow: /fr/blog/\nAllow: /fr/create/\nAllow: /fr/ott/\nAllow: /fr/customers/\nAllow: /fr/events/\nAllow: /fr/campaigns/\nAllow: /fr/about/\nAllow: /fr/vimeo-academy/\nAllow: /fr/$\nAllow: /de/features/\nAllow: /de/solutions/\nAllow: /de/product/\nAllow: /de/video-uploader\nAllow: /de/enterprise/\nAllow: /de/integrations/\nAllow: /de/blog/\nAllow: /de/create/\nAllow: /de/ott/\nAllow: /de/customers/\nAllow: /de/events/\nAllow: /de/campaigns/\nAllow: /de/about/\nAllow: /de/vimeo-academy/\nAllow: /de/$\nAllow: /pt-br/features/\nAllow: /pt-br/solutions/\nAllow: /pt-br/product/\nAllow: /pt-BR/video-uploader\nAllow: /pt-br/enterprise/\nAllow: /pt-br/integrations/\nAllow: /pt-br/blog/\nAllow: /pt-br/create/\nAllow: /pt-br/ott/\nAllow: /pt-br/customers/\nAllow: /pt-br/events/\nAllow: /pt-br/campaigns/\nAllow: /pt-br/about/\nAllow: /pt-br/vimeo-academy/\nAllow: /pt-br/$\nAllow: /ko/features/\nAllow: /ko/solutions/\nAllow: /ko/product/\nAllow: /ko/video-uploader\nAllow: /ko/enterprise/\nAllow: /ko/integrations/\nAllow: /ko/blog/\nAllow: /ko/create/\nAllow: /ko/ott/\nAllow: /ko/customers/\nAllow: /ko/events/\nAllow: /ko/campaigns/\nAllow: /ko/about/\nAllow: /ko/vimeo-academy/\nAllow: /ko/$\nAllow: /jp/features/\nAllow: /jp/solutions/\nAllow: /jp/product/\nAllow: /jp/video-uploader\nAllow: /jp/enterprise/\nAllow: /jp/integrations/\nAllow: /jp/blog/\nAllow: /jp/create/\nAllow: /jp/ott/\nAllow: /jp/customers/\nAllow: /jp/events/\nAllow: /jp/campaigns/\nAllow: /jp/about/\nAllow: /jp/vimeo-academy/\nAllow: /jp/$\n\n# Block Google (Gemini) - Allowing specific marketing/informational paths\nUser-agent: Google-Extended\nDisallow: /\nAllow: /features/\nAllow: /solutions/\nAllow: /product/\nAllow: /video-uploader\nAllow: /upgrade-plan\nAllow: /enterprise/\nAllow: /integrations/\nAllow: /blog/\nAllow: /create/\nAllow: /ott/\nAllow: /customers/\nAllow: /events/\nAllow: /campaigns/\nAllow: /about/\nAllow: /vimeo-academy/\nAllow: /$\nAllow: /es/features/\nAllow: /es/solutions/\nAllow: /es/product/\nAllow: /es/video-uploader\nAllow: /es/enterprise/\nAllow: /es/integrations/\nAllow: /es/blog/\nAllow: /es/create/\nAllow: /es/ott/\nAllow: /es/customers/\nAllow: /es/events/\nAllow: /es/campaigns/\nAllow: /es/about/\nAllow: /es/vimeo-academy/\nAllow: /es/$\nAllow: /fr/features/\nAllow: /fr/solutions/\nAllow: /fr/product/\nAllow: /fr/video-uploader\nAllow: /fr/enterprise/\nAllow: /fr/integrations/\nAllow: /fr/blog/\nAllow: /fr/create/\nAllow: /fr/ott/\nAllow: /fr/customers/\nAllow: /fr/events/\nAllow: /fr/campaigns/\nAllow: /fr/about/\nAllow: /fr/vimeo-academy/\nAllow: /fr/$\nAllow: /de/features/\nAllow: /de/solutions/\nAllow: /de/product/\nAllow: /de/video-uploader\nAllow: /de/enterprise/\nAllow: /de/integrations/\nAllow: /de/blog/\nAllow: /de/create/\nAllow: /de/ott/\nAllow: /de/customers/\nAllow: /de/events/\nAllow: /de/campaigns/\nAllow: /de/about/\nAllow: /de/vimeo-academy/\nAllow: /de/$\nAllow: /pt-br/features/\nAllow: /pt-br/solutions/\nAllow: /pt-br/product/\nAllow: /pt-BR/video-uploader\nAllow: /pt-br/enterprise/\nAllow: /pt-br/integrations/\nAllow: /pt-br/blog/\nAllow: /pt-br/create/\nAllow: /pt-br/ott/\nAllow: /pt-br/customers/\nAllow: /pt-br/events/\nAllow: /pt-br/campaigns/\nAllow: /pt-br/about/\nAllow: /pt-br/vimeo-academy/\nAllow: /pt-br/$\nAllow: /ko/features/\nAllow: /ko/solutions/\nAllow: /ko/product/\nAllow: /ko/video-uploader\nAllow: /ko/enterprise/\nAllow: /ko/integrations/\nAllow: /ko/blog/\nAllow: /ko/create/\nAllow: /ko/ott/\nAllow: /ko/customers/\nAllow: /ko/events/\nAllow: /ko/campaigns/\nAllow: /ko/about/\nAllow: /ko/vimeo-academy/\nAllow: /ko/$\nAllow: /jp/features/\nAllow: /jp/solutions/\nAllow: /jp/product/\nAllow: /jp/video-uploader\nAllow: /jp/enterprise/\nAllow: /jp/integrations/\nAllow: /jp/blog/\nAllow: /jp/create/\nAllow: /jp/ott/\nAllow: /jp/customers/\nAllow: /jp/events/\nAllow: /jp/campaigns/\nAllow: /jp/about/\nAllow: /jp/vimeo-academy/\nAllow: /jp/$\n\n# Block Claude - Allowing specific marketing/informational paths\nUser-agent: anthropic-ai\nUser-agent: Claude-Web\nUser-agent: Claudebot\nDisallow: /\nAllow: /features/\nAllow: /solutions/\nAllow: /product/\nAllow: /video-uploader\nAllow: /upgrade-plan\nAllow: /enterprise/\nAllow: /integrations/\nAllow: /blog/\nAllow: /create/\nAllow: /ott/\nAllow: /customers/\nAllow: /events/\nAllow: /campaigns/\nAllow: /about/\nAllow: /vimeo-academy/\nAllow: /$\nAllow: /es/features/\nAllow: /es/solutions/\nAllow: /es/product/\nAllow: /es/video-uploader\nAllow: /es/enterprise/\nAllow: /es/integrations/\nAllow: /es/blog/\nAllow: /es/create/\nAllow: /es/ott/\nAllow: /es/customers/\nAllow: /es/events/\nAllow: /es/campaigns/\nAllow: /es/about/\nAllow: /es/vimeo-academy/\nAllow: /es/$\nAllow: /fr/features/\nAllow: /fr/solutions/\nAllow: /fr/product/\nAllow: /fr/video-uploader\nAllow: /fr/enterprise/\nAllow: /fr/integrations/\nAllow: /fr/blog/\nAllow: /fr/create/\nAllow: /fr/ott/\nAllow: /fr/customers/\nAllow: /fr/events/\nAllow: /fr/campaigns/\nAllow: /fr/about/\nAllow: /fr/vimeo-academy/\nAllow: /fr/$\nAllow: /de/features/\nAllow: /de/solutions/\nAllow: /de/product/\nAllow: /de/video-uploader\nAllow: /de/enterprise/\nAllow: /de/integrations/\nAllow: /de/blog/\nAllow: /de/create/\nAllow: /de/ott/\nAllow: /de/customers/\nAllow: /de/events/\nAllow: /de/campaigns/\nAllow: /de/about/\nAllow: /de/vimeo-academy/\nAllow: /de/$\nAllow: /pt-br/features/\nAllow: /pt-br/solutions/\nAllow: /pt-br/product/\nAllow: /pt-BR/video-uploader\nAllow: /pt-br/enterprise/\nAllow: /pt-br/integrations/\nAllow: /pt-br/blog/\nAllow: /pt-br/create/\nAllow: /pt-br/ott/\nAllow: /pt-br/customers/\nAllow: /pt-br/events/\nAllow: /pt-br/campaigns/\nAllow: /pt-br/about/\nAllow: /pt-br/vimeo-academy/\nAllow: /pt-br/$\nAllow: /ko/features/\nAllow: /ko/solutions/\nAllow: /ko/product/\nAllow: /ko/video-uploader\nAllow: /ko/enterprise/\nAllow: /ko/integrations/\nAllow: /ko/blog/\nAllow: /ko/create/\nAllow: /ko/ott/\nAllow: /ko/customers/\nAllow: /ko/events/\nAllow: /ko/campaigns/\nAllow: /ko/about/\nAllow: /ko/vimeo-academy/\nAllow: /ko/$\nAllow: /jp/features/\nAllow: /jp/solutions/\nAllow: /jp/product/\nAllow: /jp/video-uploader\nAllow: /jp/enterprise/\nAllow: /jp/integrations/\nAllow: /jp/blog/\nAllow: /jp/create/\nAllow: /jp/ott/\nAllow: /jp/customers/\nAllow: /jp/events/\nAllow: /jp/campaigns/\nAllow: /jp/about/\nAllow: /jp/vimeo-academy/\nAllow: /jp/$\n\n# Block CommonCrawl\nUser-agent: CCBot\nUser-agent: CC-Crawler\nUser-agent: Common Crawl\nUser-agent: CommonCrawl\nDisallow: /\n\n# Block Diffbot\nUser-agent: Diffbot\nDisallow: /\n\n# Block Meta (Facebook)\nUser-agent: FacebookBot\nDisallow: /\n\n# Block ByteDance\nUser-agent: Bytespider\nDisallow: /\n\n# Block Webz.io\nUser-agent: Omgilibot\nDisallow: /\nUser-agent: Omgili\nDisallow: /\n\n# Block ImagesiftBot\nUser-agent: ImagesiftBot\nDisallow: /\n\n# Block Meltwater\nUser-agent: Meltwater\nDisallow: /\n\n# Block Seekr\nUser-agent: Seekr\nDisallow: /\n\n# Block Grok\nUser-agent: Grok\nDisallow: /\n\n# Sitemap declaration\nSitemap: https://vimeo.com/marketing.xml.gz\nSitemap: https://vimeo.com/sitemap/videos.xml.gz\n\n","agents":{"GPTBot":true,"OAI-SearchBot":true,"ChatGPT-User":true,"ClaudeBot":true,"Claude-SearchBot":true,"PerplexityBot":true,"Google-Extended":true,"Applebot-Extended":true,"CCBot":false,"Bytespider":false,"Meta-ExternalAgent":true,"Amazonbot":true}},"sitemap":{"status":200,"content_type":"application/xml","bytes":2759,"sha256":"08fda1eef38912abaf1492e14f8ca441096d71a0982c989fd5c41699bb9fba1e","final_host":"vimeo.com","final_path":"/marketing.xml.gz","ms":10,"url":"https://vimeo.com/marketing.xml.gz","declared":true,"xml":true},"files":{"llms_txt":{"status":404,"content_type":"text/html","bytes":163525,"sha256":"8e144e45bdbd7e78ccbb8f06f2174f023f3cc09793e9c359f4bd0645033e324d","final_host":"vimeo.com","final_path":"/llms.txt","ms":81,"path":"/llms.txt","html":true,"json":null,"present":false},"agents_md":{"status":404,"content_type":"text/html","bytes":111658,"sha256":"2da2dea97aee88aec10b9117a1a5d1fb043d65b0f374cb3266dd17c04e45b236","final_host":"vimeo.com","final_path":"/agents.md","ms":89,"path":"/agents.md","html":true,"json":null,"present":false},"agent_skills":{"status":404,"content_type":"text/html","bytes":163525,"sha256":"ed44a929de2c48130e9fa398c655d4fe218d7fa88c3fc6ac7e303e165209ccf1","final_host":"vimeo.com","final_path":"/.well-known/agent-skills/index.json","ms":98,"path":"/.well-known/agent-skills/index.json","html":true,"json":null,"present":false},"mcp_card":{"status":404,"content_type":"text/html","bytes":163525,"sha256":"ed44a929de2c48130e9fa398c655d4fe218d7fa88c3fc6ac7e303e165209ccf1","final_host":"vimeo.com","final_path":"/.well-known/mcp/server-card.json","ms":102,"path":"/.well-known/mcp/server-card.json","html":true,"json":null,"present":false},"api_catalog":{"status":404,"content_type":"text/html","bytes":163525,"sha256":"8e144e45bdbd7e78ccbb8f06f2174f023f3cc09793e9c359f4bd0645033e324d","final_host":"vimeo.com","final_path":"/.well-known/api-catalog","ms":126,"path":"/.well-known/api-catalog","html":true,"json":null,"present":false},"a2a_card":{"status":404,"content_type":"text/html","bytes":163525,"sha256":"ed44a929de2c48130e9fa398c655d4fe218d7fa88c3fc6ac7e303e165209ccf1","final_host":"vimeo.com","final_path":"/.well-known/agent-card.json","ms":115,"path":"/.well-known/agent-card.json","html":true,"json":null,"present":false}},"rings":{"v":1,"candidates":[{"url":"https://mcp.vimeo.com/mcp","source":"conventional_host"}],"mcp":[{"url":"https://mcp.vimeo.com/mcp","source":"conventional_host","outcome":"auth_required","status":401,"ms":21,"initialized":false,"protocol_version":null,"server":null,"tools":null,"tools_sha256":null,"auth_header":true}],"a2a":null,"webmcp":{"model_context":false,"bridge":false,"mcp_json":false,"link_rel":false,"any":false},"handshakes":1,"get_path":{"status":200,"content_type":"text/html","ms":246,"error":null},"get_host":{"status":401,"content_type":"text/plain","ms":121,"error":null}},"sf":[4,0,0,0,0,0,2,0,1],"derived":{"reachable":true,"home_class":"answers","robots_served":true,"robots_sitemap":true,"content_signal":false,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":["CCBot","Bytespider"],"blocks_any_ai":true,"files":{"llms_txt":false,"agents_md":false,"agent_skills":false,"mcp_card":false,"api_catalog":false,"a2a_card":false},"any_capability":false,"ai_refused":[],"ai_thin":[],"aipref":0,"aipref_measured":true,"sf":[4,0,0,0,0,0,2,0,1],"rings":{"mcp_declared":true,"mcp_live":false,"mcp_auth":true,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":3,"fh":"vimeo.com","fp_other_host":[],"fp":"e699fe5c0d05a34a","fp_bits":{"gptbot":0,"claudebot":0,"oaisearch":0,"perplexity":0}},"derive_v":5,"frame":"top10k","shard":6},{"v":1,"kind":"crawlcheck-census-record","run":18,"domain":"gvt2.com","rank":88,"measured_at":"2026-11-13T18:56:28.476Z","home":{"browser":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"gvt2.com","final_path":"/","ms":52,"identity":"Mobile browser","html":false},"gptbot":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"gvt2.com","final_path":"/","ms":52,"identity":"GPTBot","html":false},"claudebot":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"gvt2.com","final_path":"/","ms":51,"identity":"ClaudeBot","html":false},"oaisearch":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"gvt2.com","final_path":"/","ms":51,"identity":"OAI-SearchBot","html":false},"perplexity":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"gvt2.com","final_path":"/","ms":51,"identity":"PerplexityBot","html":false}},"robots":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"gvt2.com","final_path":"/robots.txt","ms":51,"html":false,"text":null,"agents":null},"sitemap":null,"files":null,"derived":{"reachable":false,"home_class":"no_site","robots_served":false,"robots_sitemap":false,"content_signal":false,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":[],"blocks_any_ai":false,"files":{"llms_txt":false,"agents_md":false,"agent_skills":false,"mcp_card":false,"api_catalog":false,"a2a_card":false},"any_capability":false,"ai_refused":[],"ai_thin":[],"rings":{"mcp_declared":false,"mcp_live":false,"mcp_auth":false,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":3,"fh":"gvt2.com","fp_other_host":[]},"derive_v":5,"frame":"top10k","shard":7},{"v":1,"kind":"crawlcheck-census-record","run":18,"domain":"cdninstagram.com","rank":89,"measured_at":"2026-11-13T18:47:19.534Z","home":{"browser":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"cdninstagram.com","final_path":"/","ms":45,"identity":"Mobile browser","html":false},"gptbot":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"cdninstagram.com","final_path":"/","ms":25,"identity":"GPTBot","html":false},"claudebot":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"cdninstagram.com","final_path":"/","ms":22,"identity":"ClaudeBot","html":false},"oaisearch":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"cdninstagram.com","final_path":"/","ms":21,"identity":"OAI-SearchBot","html":false},"perplexity":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"cdninstagram.com","final_path":"/","ms":21,"identity":"PerplexityBot","html":false}},"robots":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"cdninstagram.com","final_path":"/robots.txt","ms":20,"html":false,"text":null,"agents":null},"sitemap":null,"files":null,"derived":{"reachable":false,"home_class":"no_site","robots_served":false,"robots_sitemap":false,"content_signal":false,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":[],"blocks_any_ai":false,"files":{"llms_txt":false,"agents_md":false,"agent_skills":false,"mcp_card":false,"api_catalog":false,"a2a_card":false},"any_capability":false,"ai_refused":[],"ai_thin":[],"rings":{"mcp_declared":false,"mcp_live":false,"mcp_auth":false,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":3,"fh":"cdninstagram.com","fp_other_host":[]},"derive_v":5,"frame":"top10k","shard":0},{"v":1,"kind":"crawlcheck-census-record","run":18,"domain":"openai.com","rank":90,"measured_at":"2026-11-13T18:49:01.730Z","home":{"browser":{"status":403,"content_type":"text/html","bytes":9861,"sha256":"87636fb61d1973527a43c4910f2a1e35ca36b63ec1fa5de22e6ac1b0aba9b1a3","final_host":"openai.com","final_path":"/","ms":8,"identity":"Mobile browser","html":true},"gptbot":{"status":403,"content_type":"text/html","bytes":9861,"sha256":"0dd40661d5c035cff418866f49603c788a9028cadbce38432af8e312714ce0b2","final_host":"openai.com","final_path":"/","ms":8,"identity":"GPTBot","html":true},"claudebot":{"status":403,"content_type":"text/html","bytes":9775,"sha256":"424a35b257229491b0a88b53c6675088fa6b1242e761ddb5ed8d2412fe3f54cc","final_host":"openai.com","final_path":"/","ms":8,"identity":"ClaudeBot","html":true},"oaisearch":{"status":403,"content_type":"text/html","bytes":9882,"sha256":"a6c2c2424e92f7100e82325abc5b55c045ed26583cb995d0fbad4e29cebae645","final_host":"openai.com","final_path":"/","ms":5,"identity":"OAI-SearchBot","html":true},"perplexity":{"status":403,"content_type":"text/html","bytes":9968,"sha256":"d3c03e0f05dcae187c884b7e2270528060df774aac9a22d9ccce38b254586f63","final_host":"openai.com","final_path":"/","ms":1,"identity":"PerplexityBot","html":true}},"robots":{"status":200,"content_type":"text/plain","bytes":98,"sha256":"c53100563b9d289efeebb6f826a20c566578e1337e3096e298eb9da5ecc2e498","final_host":"openai.com","final_path":"/robots.txt","ms":4,"html":false,"text":"User-agent: *\nAllow: /\nDisallow: /microsoft-for-startups/\n\nSitemap: https://openai.com/sitemap.xml","agents":{"GPTBot":true,"OAI-SearchBot":true,"ChatGPT-User":true,"ClaudeBot":true,"Claude-SearchBot":true,"PerplexityBot":true,"Google-Extended":true,"Applebot-Extended":true,"CCBot":true,"Bytespider":true,"Meta-ExternalAgent":true,"Amazonbot":true}},"sitemap":null,"files":null,"derived":{"reachable":false,"home_class":"refused_us","robots_served":false,"robots_sitemap":false,"content_signal":false,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":[],"blocks_any_ai":false,"files":{"llms_txt":false,"agents_md":false,"agent_skills":false,"mcp_card":false,"api_catalog":false,"a2a_card":false},"any_capability":false,"ai_refused":[],"ai_thin":[],"rings":{"mcp_declared":false,"mcp_live":false,"mcp_auth":false,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":3,"fh":"openai.com","fp_other_host":[]},"derive_v":5,"frame":"top10k","shard":1},{"v":1,"kind":"crawlcheck-census-record","run":18,"domain":"f5.com","rank":91,"measured_at":"2026-11-13T18:50:07.634Z","home":{"browser":{"status":200,"content_type":"text/html","bytes":596914,"sha256":"6f8c996912ba3a918eb85b27e0ab8e60f0c7bdc88c6aabe0be8c3dcd6918af5d","final_host":"www.f5.com","final_path":"/","ms":1220,"identity":"Mobile browser","html":true,"fp":"6e38f356021bc19b"},"gptbot":{"status":200,"content_type":"text/html","bytes":596914,"sha256":"6f8c996912ba3a918eb85b27e0ab8e60f0c7bdc88c6aabe0be8c3dcd6918af5d","final_host":"www.f5.com","final_path":"/","ms":840,"identity":"GPTBot","html":true,"fp":"6e38f356021bc19b"},"claudebot":{"status":200,"content_type":"text/html","bytes":596914,"sha256":"6f8c996912ba3a918eb85b27e0ab8e60f0c7bdc88c6aabe0be8c3dcd6918af5d","final_host":"www.f5.com","final_path":"/","ms":739,"identity":"ClaudeBot","html":true,"fp":"6e38f356021bc19b"},"oaisearch":{"status":200,"content_type":"text/html","bytes":596914,"sha256":"6f8c996912ba3a918eb85b27e0ab8e60f0c7bdc88c6aabe0be8c3dcd6918af5d","final_host":"www.f5.com","final_path":"/","ms":830,"identity":"OAI-SearchBot","html":true,"fp":"6e38f356021bc19b"},"perplexity":{"status":200,"content_type":"text/html","bytes":596914,"sha256":"6f8c996912ba3a918eb85b27e0ab8e60f0c7bdc88c6aabe0be8c3dcd6918af5d","final_host":"www.f5.com","final_path":"/","ms":687,"identity":"PerplexityBot","html":true,"fp":"6e38f356021bc19b"}},"robots":{"status":200,"content_type":"text/plain","bytes":156,"sha256":"eca371645a1484bc5d7dfc8e150f188b7e70d7754e6c00c1770de364f38e1401","final_host":"www.f5.com","final_path":"/robots.txt","ms":782,"html":false,"text":"# *\nUser-agent: *\nAllow: /\n\n# Host\nHost: https://www.f5.com\n\n# Sitemaps\nSitemap: https://www.f5.com/sitemap.xml\nSitemap: https://www.f5.com/labs/sitemap.xml","agents":{"GPTBot":true,"OAI-SearchBot":true,"ChatGPT-User":true,"ClaudeBot":true,"Claude-SearchBot":true,"PerplexityBot":true,"Google-Extended":true,"Applebot-Extended":true,"CCBot":true,"Bytespider":true,"Meta-ExternalAgent":true,"Amazonbot":true}},"sitemap":{"status":200,"content_type":"application/xml","bytes":1287,"sha256":"2705ee74dea18c947ac642a4a07d1e985e48869398a4e6bf48a56879ef38f266","final_host":"www.f5.com","final_path":"/sitemap.xml","ms":116,"url":"https://www.f5.com/sitemap.xml","declared":true,"xml":true},"files":{"llms_txt":{"status":404,"content_type":"text/html","bytes":166790,"sha256":"9bc202e39c90e75e8ce50395982c7f9054fd53664744d797d9ddc067568ec0d1","final_host":"www.f5.com","final_path":"/llms.txt","ms":1520,"path":"/llms.txt","html":true,"json":null,"present":false},"agents_md":{"status":404,"content_type":"text/html","bytes":166707,"sha256":"1402e2624e285d100d74376c8fa9699e5a6ebf66348e3344a60eb15bdd0b8bf8","final_host":"www.f5.com","final_path":"/agents.md","ms":1319,"path":"/agents.md","html":true,"json":null,"present":false},"agent_skills":{"status":404,"content_type":"text/html","bytes":11333,"sha256":"0d7fbf17a8a42de1036be6b5dec4e7fe82111a3cb6b920fb8bba1a4a15255258","final_host":"www.f5.com","final_path":"/.well-known/agent-skills/index.json","ms":326,"path":"/.well-known/agent-skills/index.json","html":true,"json":null,"present":false},"mcp_card":{"status":404,"content_type":"text/html","bytes":11324,"sha256":"155f794d3c5f2b6004291a09e2fc31f4652df737540894941f4d3bf8dc721b8c","final_host":"www.f5.com","final_path":"/.well-known/mcp/server-card.json","ms":287,"path":"/.well-known/mcp/server-card.json","html":true,"json":null,"present":false},"api_catalog":{"status":404,"content_type":"text/html","bytes":11293,"sha256":"12debf650dbe3dd9b46159f0a05fffbe0758a81f65609dbdca6e2901caeb4476","final_host":"www.f5.com","final_path":"/.well-known/api-catalog","ms":258,"path":"/.well-known/api-catalog","html":true,"json":null,"present":false},"a2a_card":{"status":404,"content_type":"text/html","bytes":11305,"sha256":"4e809310babcc40c5bd89ad33fb762fa5feb3670f44b39851fa8140e23283f95","final_host":"www.f5.com","final_path":"/.well-known/agent-card.json","ms":281,"path":"/.well-known/agent-card.json","html":true,"json":null,"present":false}},"rings":{"v":1,"candidates":[],"mcp":[],"a2a":null,"webmcp":{"model_context":false,"bridge":false,"mcp_json":false,"link_rel":false,"any":false},"handshakes":0,"get_path":{"status":404,"content_type":"text/html","ms":1553,"error":null},"get_host":{"status":530,"content_type":"text/plain","ms":290,"error":null}},"sf":[1,0,0,0,0,0,0,0,0],"derived":{"reachable":true,"home_class":"answers","robots_served":true,"robots_sitemap":true,"content_signal":false,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":[],"blocks_any_ai":false,"files":{"llms_txt":false,"agents_md":false,"agent_skills":false,"mcp_card":false,"api_catalog":false,"a2a_card":false},"any_capability":false,"ai_refused":[],"ai_thin":[],"aipref":0,"aipref_measured":true,"sf":[1,0,0,0,0,0,0,0,0],"rings":{"mcp_declared":false,"mcp_live":false,"mcp_auth":false,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":3,"fh":"www.f5.com","fp_other_host":[],"fp":"6e38f356021bc19b","fp_bits":{"gptbot":0,"claudebot":0,"oaisearch":0,"perplexity":0}},"derive_v":5,"frame":"top10k","shard":2},{"v":1,"kind":"crawlcheck-census-record","run":18,"domain":"gwfb.net","rank":92,"measured_at":"2026-11-13T18:51:23.228Z","home":{"browser":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"gwfb.net","final_path":"/","ms":371,"identity":"Mobile browser","html":false},"gptbot":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"gwfb.net","final_path":"/","ms":1,"identity":"GPTBot","html":false},"claudebot":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"gwfb.net","final_path":"/","ms":1,"identity":"ClaudeBot","html":false},"oaisearch":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"gwfb.net","final_path":"/","ms":2,"identity":"OAI-SearchBot","html":false},"perplexity":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"gwfb.net","final_path":"/","ms":0,"identity":"PerplexityBot","html":false}},"robots":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"gwfb.net","final_path":"/robots.txt","ms":1,"html":false,"text":null,"agents":null},"sitemap":null,"files":null,"derived":{"reachable":false,"home_class":"no_site","robots_served":false,"robots_sitemap":false,"content_signal":false,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":[],"blocks_any_ai":false,"files":{"llms_txt":false,"agents_md":false,"agent_skills":false,"mcp_card":false,"api_catalog":false,"a2a_card":false},"any_capability":false,"ai_refused":[],"ai_thin":[],"rings":{"mcp_declared":false,"mcp_live":false,"mcp_auth":false,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":3,"fh":"gwfb.net","fp_other_host":[]},"derive_v":5,"frame":"top10k","shard":3},{"v":1,"kind":"crawlcheck-census-record","run":18,"domain":"sentry.io","rank":93,"measured_at":"2026-11-13T18:52:00.314Z","home":{"browser":{"status":200,"content_type":"text/html","bytes":625105,"sha256":"a43eb50533a5a44f4c80032db65a55b76bb3b81aeefbb29a62b63597b84eed1c","final_host":"sentry.io","final_path":"/welcome","ms":205,"identity":"Mobile browser","html":true,"fp":"4c20af501f70cf77"},"gptbot":{"status":200,"content_type":"text/markdown","bytes":19540,"sha256":"b52e76af70e3830da7de994c72d70255f681f73b74065f240a804736f3a3a80d","final_host":"sentry.io","final_path":"/welcome","ms":160,"identity":"GPTBot","html":false},"claudebot":{"status":200,"content_type":"text/markdown","bytes":19540,"sha256":"b52e76af70e3830da7de994c72d70255f681f73b74065f240a804736f3a3a80d","final_host":"sentry.io","final_path":"/welcome","ms":175,"identity":"ClaudeBot","html":false},"oaisearch":{"status":200,"content_type":"text/markdown","bytes":19540,"sha256":"b52e76af70e3830da7de994c72d70255f681f73b74065f240a804736f3a3a80d","final_host":"sentry.io","final_path":"/welcome","ms":159,"identity":"OAI-SearchBot","html":false},"perplexity":{"status":200,"content_type":"text/markdown","bytes":19540,"sha256":"b52e76af70e3830da7de994c72d70255f681f73b74065f240a804736f3a3a80d","final_host":"sentry.io","final_path":"/welcome","ms":175,"identity":"PerplexityBot","html":false}},"robots":{"status":200,"content_type":"text/plain","bytes":140,"sha256":"c0f7d803d97db6c0bf75804b97eed4a533f7249c27994e253ad4fa403652da96","final_host":"sentry.io","final_path":"/robots.txt","ms":52,"html":false,"text":"User-agent: *\nContent-Signal: search=yes, ai-input=yes, ai-train=yes\nDisallow: /api/\nAllow: /\n\nSitemap: https://sentry.io/sitemap-index.xml\n","agents":{"GPTBot":true,"OAI-SearchBot":true,"ChatGPT-User":true,"ClaudeBot":true,"Claude-SearchBot":true,"PerplexityBot":true,"Google-Extended":true,"Applebot-Extended":true,"CCBot":true,"Bytespider":true,"Meta-ExternalAgent":true,"Amazonbot":true}},"sitemap":{"status":200,"content_type":"application/xml","bytes":372,"sha256":"8bfba18ac85d5dbb7f46917c1e54f491c1b2a0df8559c46e9d92307ef1d41a89","final_host":"sentry.io","final_path":"/sitemap-index.xml","ms":288,"url":"https://sentry.io/sitemap-index.xml","declared":true,"xml":true},"files":{"llms_txt":{"status":200,"content_type":"text/plain","bytes":93571,"sha256":"ad086d7028cf74d7e820ef1e5ff44135e7a75f6942e2d6043f30f412a07514ba","final_host":"sentry.io","final_path":"/llms.txt","ms":250,"path":"/llms.txt","html":false,"json":null,"present":true},"agents_md":{"status":200,"content_type":"text/markdown","bytes":976,"sha256":"d74022d0efda2763dde039ddcf1155c5d0a4783227ba9d88a47eed66435a7683","final_host":"sentry.io","final_path":"/agents.md","ms":89,"path":"/agents.md","html":false,"json":null,"present":true},"agent_skills":{"status":200,"content_type":"application/json","bytes":13,"sha256":"0b513961955003886d251c9091e61f5dc648adbb93b431b1a951b08e0f6201ac","final_host":"sentry.io","final_path":"/.well-known/agent-skills/index.json","ms":40,"path":"/.well-known/agent-skills/index.json","html":false,"json":true,"present":true},"mcp_card":{"status":200,"content_type":"application/json","bytes":234,"sha256":"5fb322b52608b7465c0ca90eb50ef8cea2e63a0d657a32f35cc2765023551ae2","final_host":"sentry.io","final_path":"/.well-known/mcp/server-card.json","ms":47,"path":"/.well-known/mcp/server-card.json","html":false,"json":true,"present":true},"api_catalog":{"status":200,"content_type":"application/linkset+json","bytes":179,"sha256":"e906b19b2175c78512435aa44daef69d900227c4f8e2e225186a626f5d98733f","final_host":"sentry.io","final_path":"/.well-known/api-catalog","ms":40,"path":"/.well-known/api-catalog","html":false,"json":true,"present":true},"a2a_card":{"status":200,"content_type":"text/html","bytes":27400,"sha256":"3664dda74d3919391c028319a57447d94cc5fac2f90ac1d05cdba72eecdb478b","final_host":"sentry.io","final_path":"/auth/login/.well-known","ms":149,"path":"/.well-known/agent-card.json","html":true,"json":null,"present":false}},"rings":{"v":1,"candidates":[{"url":"https://mcp.sentry.dev/mcp","source":"mcp_card"},{"url":"https://sentry.io","source":"mcp_card"},{"url":"https://mcp.sentry.dev","source":"agents_md"},{"url":"https://mcp.sentry.dev/","source":"homepage_link"}],"mcp":[{"url":"https://mcp.sentry.dev/mcp","source":"mcp_card","outcome":"auth_required","status":401,"ms":16,"initialized":false,"protocol_version":null,"server":null,"tools":null,"tools_sha256":null,"auth_header":true},{"url":"https://sentry.io","source":"mcp_card","outcome":"auth_required","status":403,"ms":38,"initialized":false,"protocol_version":null,"server":null,"tools":null,"tools_sha256":null,"auth_header":false},{"url":"https://mcp.sentry.dev","source":"agents_md","outcome":"http_error","status":405,"ms":12,"initialized":false,"protocol_version":null,"server":null,"tools":null,"tools_sha256":null,"auth_header":false}],"a2a":null,"webmcp":{"model_context":false,"bridge":false,"mcp_json":false,"link_rel":false,"any":false},"handshakes":3,"get_path":{"status":200,"content_type":"text/html","ms":142,"error":null},"get_host":{"status":200,"content_type":"text/html","ms":337,"error":null}},"sf":[1,0,0,0,0,0,1,0,0],"derived":{"reachable":true,"home_class":"answers","robots_served":true,"robots_sitemap":true,"content_signal":true,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":[],"blocks_any_ai":false,"files":{"llms_txt":true,"agents_md":true,"agent_skills":true,"mcp_card":true,"api_catalog":true,"a2a_card":false},"any_capability":true,"ai_refused":[],"ai_thin":["gptbot","claudebot","oaisearch","perplexity"],"aipref":0,"aipref_measured":true,"sf":[1,0,0,0,0,0,1,0,0],"rings":{"mcp_declared":true,"mcp_live":false,"mcp_auth":true,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":3,"fh":"sentry.io","fp_other_host":[],"fp":"4c20af501f70cf77","fp_bits":{}},"derive_v":5,"frame":"top10k","shard":4},{"v":1,"kind":"crawlcheck-census-record","run":18,"domain":"a2z.com","rank":94,"measured_at":"2026-11-13T18:53:20.684Z","home":{"browser":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"a2z.com","final_path":"/","ms":71,"identity":"Mobile browser","html":false},"gptbot":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"a2z.com","final_path":"/","ms":1,"identity":"GPTBot","html":false},"claudebot":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"a2z.com","final_path":"/","ms":0,"identity":"ClaudeBot","html":false},"oaisearch":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"a2z.com","final_path":"/","ms":1,"identity":"OAI-SearchBot","html":false},"perplexity":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"a2z.com","final_path":"/","ms":0,"identity":"PerplexityBot","html":false}},"robots":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"a2z.com","final_path":"/robots.txt","ms":1,"html":false,"text":null,"agents":null},"sitemap":null,"files":null,"derived":{"reachable":false,"home_class":"no_site","robots_served":false,"robots_sitemap":false,"content_signal":false,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":[],"blocks_any_ai":false,"files":{"llms_txt":false,"agents_md":false,"agent_skills":false,"mcp_card":false,"api_catalog":false,"a2a_card":false},"any_capability":false,"ai_refused":[],"ai_thin":[],"rings":{"mcp_declared":false,"mcp_live":false,"mcp_auth":false,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":3,"fh":"a2z.com","fp_other_host":[]},"derive_v":5,"frame":"top10k","shard":5},{"v":1,"kind":"crawlcheck-census-record","run":18,"domain":"nginx.com","rank":95,"measured_at":"2026-11-13T18:53:42.755Z","home":{"browser":{"status":200,"content_type":"text/html","bytes":434951,"sha256":"e6d0417bcfbd353d349abd4d8c85fd24456b363c1de2de17df7bbbd9e3f41986","final_host":"www.f5.com","final_path":"/products/nginx","ms":1770,"identity":"Mobile browser","html":true,"fp":"31793d5e224b6812"},"gptbot":{"status":null,"error":"timeout","ms":8000,"identity":"GPTBot","html":null},"claudebot":{"status":null,"error":"timeout","ms":8000,"identity":"ClaudeBot","html":null},"oaisearch":{"status":null,"error":"timeout","ms":8000,"identity":"OAI-SearchBot","html":null},"perplexity":{"status":null,"error":"timeout","ms":8000,"identity":"PerplexityBot","html":null}},"robots":{"status":null,"error":"timeout","ms":8000,"html":null,"text":null,"agents":null},"sitemap":{"status":200,"content_type":"text/html","bytes":434951,"sha256":"58194062b9acb2f809b88a7cd92756e5e109396f3149a0ab10cf8c4239c57058","final_host":"www.f5.com","final_path":"/products/nginx","ms":274,"url":"https://nginx.com/sitemap.xml","declared":false,"xml":false},"files":{"llms_txt":{"status":200,"content_type":"text/html","bytes":434951,"sha256":"58194062b9acb2f809b88a7cd92756e5e109396f3149a0ab10cf8c4239c57058","final_host":"www.f5.com","final_path":"/products/nginx","ms":262,"path":"/llms.txt","html":true,"json":null,"present":false},"agents_md":{"status":200,"content_type":"text/html","bytes":434951,"sha256":"58194062b9acb2f809b88a7cd92756e5e109396f3149a0ab10cf8c4239c57058","final_host":"www.f5.com","final_path":"/products/nginx","ms":359,"path":"/agents.md","html":true,"json":null,"present":false},"agent_skills":{"status":200,"content_type":"text/html","bytes":434951,"sha256":"58194062b9acb2f809b88a7cd92756e5e109396f3149a0ab10cf8c4239c57058","final_host":"www.f5.com","final_path":"/products/nginx","ms":259,"path":"/.well-known/agent-skills/index.json","html":true,"json":null,"present":false},"mcp_card":{"status":200,"content_type":"text/html","bytes":434951,"sha256":"58194062b9acb2f809b88a7cd92756e5e109396f3149a0ab10cf8c4239c57058","final_host":"www.f5.com","final_path":"/products/nginx","ms":355,"path":"/.well-known/mcp/server-card.json","html":true,"json":null,"present":false},"api_catalog":{"status":200,"content_type":"text/html","bytes":434951,"sha256":"58194062b9acb2f809b88a7cd92756e5e109396f3149a0ab10cf8c4239c57058","final_host":"www.f5.com","final_path":"/products/nginx","ms":213,"path":"/.well-known/api-catalog","html":true,"json":null,"present":false},"a2a_card":{"status":200,"content_type":"text/html","bytes":434951,"sha256":"58194062b9acb2f809b88a7cd92756e5e109396f3149a0ab10cf8c4239c57058","final_host":"www.f5.com","final_path":"/products/nginx","ms":221,"path":"/.well-known/agent-card.json","html":true,"json":null,"present":false}},"rings":{"v":1,"candidates":[],"mcp":[],"a2a":null,"webmcp":{"model_context":false,"bridge":false,"mcp_json":false,"link_rel":false,"any":false},"handshakes":0,"get_path":{"status":200,"content_type":"text/html","ms":261,"error":null},"get_host":{"status":530,"content_type":"text/plain","ms":1039,"error":null}},"sf":[1,0,0,0,0,0,0,0,0],"derived":{"reachable":true,"home_class":"answers","robots_served":false,"robots_sitemap":false,"content_signal":false,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":[],"blocks_any_ai":false,"files":{"llms_txt":false,"agents_md":false,"agent_skills":false,"mcp_card":false,"api_catalog":false,"a2a_card":false},"any_capability":false,"ai_refused":["gptbot","claudebot","oaisearch","perplexity"],"ai_thin":[],"sf":[1,0,0,0,0,0,0,0,0],"rings":{"mcp_declared":false,"mcp_live":false,"mcp_auth":false,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":3,"fh":"www.f5.com","fp_other_host":[],"fp":"31793d5e224b6812","fp_bits":{}},"derive_v":5,"frame":"top10k","shard":6},{"v":1,"kind":"crawlcheck-census-record","run":18,"domain":"ytimg.com","rank":96,"measured_at":"2026-11-13T18:56:28.476Z","home":{"browser":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"ytimg.com","final_path":"/","ms":12,"identity":"Mobile browser","html":false},"gptbot":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"ytimg.com","final_path":"/","ms":12,"identity":"GPTBot","html":false},"claudebot":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"ytimg.com","final_path":"/","ms":12,"identity":"ClaudeBot","html":false},"oaisearch":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"ytimg.com","final_path":"/","ms":12,"identity":"OAI-SearchBot","html":false},"perplexity":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"ytimg.com","final_path":"/","ms":11,"identity":"PerplexityBot","html":false}},"robots":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"ytimg.com","final_path":"/robots.txt","ms":11,"html":false,"text":null,"agents":null},"sitemap":null,"files":null,"derived":{"reachable":false,"home_class":"no_site","robots_served":false,"robots_sitemap":false,"content_signal":false,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":[],"blocks_any_ai":false,"files":{"llms_txt":false,"agents_md":false,"agent_skills":false,"mcp_card":false,"api_catalog":false,"a2a_card":false},"any_capability":false,"ai_refused":[],"ai_thin":[],"rings":{"mcp_declared":false,"mcp_live":false,"mcp_auth":false,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":3,"fh":"ytimg.com","fp_other_host":[]},"derive_v":5,"frame":"top10k","shard":7},{"v":1,"kind":"crawlcheck-census-record","run":18,"domain":"mozilla.org","rank":97,"measured_at":"2026-11-13T18:47:19.534Z","home":{"browser":{"status":200,"content_type":"text/html","bytes":47881,"sha256":"bc35049d7968e241ffd00cf5050b6116acaa771110bfc7d7ecc373744496c36f","final_host":"www.mozilla.org","final_path":"/","ms":648,"identity":"Mobile browser","html":true,"fp":"4b53d53d5d2e04c8"},"gptbot":{"status":200,"content_type":"text/html","bytes":47881,"sha256":"bc35049d7968e241ffd00cf5050b6116acaa771110bfc7d7ecc373744496c36f","final_host":"www.mozilla.org","final_path":"/","ms":648,"identity":"GPTBot","html":true,"fp":"4b53d53d5d2e04c8"},"claudebot":{"status":200,"content_type":"text/html","bytes":47881,"sha256":"bc35049d7968e241ffd00cf5050b6116acaa771110bfc7d7ecc373744496c36f","final_host":"www.mozilla.org","final_path":"/","ms":648,"identity":"ClaudeBot","html":true,"fp":"4b53d53d5d2e04c8"},"oaisearch":{"status":200,"content_type":"text/html","bytes":47881,"sha256":"bc35049d7968e241ffd00cf5050b6116acaa771110bfc7d7ecc373744496c36f","final_host":"www.mozilla.org","final_path":"/","ms":648,"identity":"OAI-SearchBot","html":true,"fp":"4b53d53d5d2e04c8"},"perplexity":{"status":200,"content_type":"text/html","bytes":47881,"sha256":"bc35049d7968e241ffd00cf5050b6116acaa771110bfc7d7ecc373744496c36f","final_host":"www.mozilla.org","final_path":"/","ms":647,"identity":"PerplexityBot","html":true,"fp":"4b53d53d5d2e04c8"}},"robots":{"status":200,"content_type":"text/plain","bytes":66,"sha256":"cf0fd2842659297b0d6fce407bed86bf424ae6666051b881dbd72e871d687fe3","final_host":"www.mozilla.org","final_path":"/robots.txt","ms":532,"html":false,"text":"user-agent: *\ndisallow: /*/newsletter/existing/\ndisallow: /*/etc/\n","agents":{"GPTBot":true,"OAI-SearchBot":true,"ChatGPT-User":true,"ClaudeBot":true,"Claude-SearchBot":true,"PerplexityBot":true,"Google-Extended":true,"Applebot-Extended":true,"CCBot":true,"Bytespider":true,"Meta-ExternalAgent":true,"Amazonbot":true}},"sitemap":{"status":404,"content_type":"text/html","bytes":9035,"sha256":"8934e06d3cb29ec574f903b4a2b549b2da7ac6a29ca6bf807012ff34f285a3cb","final_host":"www.mozilla.org","final_path":"/en-us/sitemap.xml","ms":905,"url":"https://mozilla.org/sitemap.xml","declared":false,"xml":false},"files":{"llms_txt":{"status":404,"content_type":"text/html","bytes":9032,"sha256":"540c3866c8e266b26bacf130bc7458b2507f68d9f4836757853ec2462b8167a9","final_host":"www.mozilla.org","final_path":"/en-us/llms.txt","ms":544,"path":"/llms.txt","html":true,"json":null,"present":false},"agents_md":{"status":404,"content_type":"text/html","bytes":9033,"sha256":"00644bd683424baff2686668942b532733a18349f9c918bdb24c2e61a70df98a","final_host":"www.mozilla.org","final_path":"/en-us/agents.md","ms":552,"path":"/agents.md","html":true,"json":null,"present":false},"agent_skills":{"status":404,"content_type":"text/html","bytes":9059,"sha256":"7cf0b6eafb6e497d055d70b757d8e7aaf2291c7f570f2b1289a2a59ba612ede6","final_host":"www.mozilla.org","final_path":"/.well-known/agent-skills/index.json","ms":319,"path":"/.well-known/agent-skills/index.json","html":true,"json":null,"present":false},"mcp_card":{"status":404,"content_type":"text/html","bytes":9056,"sha256":"f0aab44eeeaeb2893507fa9b3c0b7886abea14705f36c82f3c08b259e084bfb5","final_host":"www.mozilla.org","final_path":"/.well-known/mcp/server-card.json","ms":353,"path":"/.well-known/mcp/server-card.json","html":true,"json":null,"present":false},"api_catalog":{"status":404,"content_type":"text/html","bytes":9047,"sha256":"7c26b85f525cd8e05fc5f5da516d49b238f950823f16beb5d4a4fd7acd49ce4f","final_host":"www.mozilla.org","final_path":"/.well-known/api-catalog","ms":307,"path":"/.well-known/api-catalog","html":true,"json":null,"present":false},"a2a_card":{"status":404,"content_type":"text/html","bytes":9051,"sha256":"8a0731003e470cc64051e5fb03f845ee6629c5af04883d9824f576228cf26fc2","final_host":"www.mozilla.org","final_path":"/.well-known/agent-card.json","ms":357,"path":"/.well-known/agent-card.json","html":true,"json":null,"present":false}},"rings":{"v":1,"candidates":[],"mcp":[],"a2a":null,"webmcp":{"model_context":false,"bridge":false,"mcp_json":false,"link_rel":false,"any":false},"handshakes":0,"get_path":{"status":404,"content_type":"text/html","ms":737,"error":null},"get_host":{"status":530,"content_type":"text/plain","ms":5,"error":null}},"sf":[1,0,0,0,0,0,0,0,0],"derived":{"reachable":true,"home_class":"answers","robots_served":true,"robots_sitemap":false,"content_signal":false,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":[],"blocks_any_ai":false,"files":{"llms_txt":false,"agents_md":false,"agent_skills":false,"mcp_card":false,"api_catalog":false,"a2a_card":false},"any_capability":false,"ai_refused":[],"ai_thin":[],"aipref":0,"aipref_measured":true,"sf":[1,0,0,0,0,0,0,0,0],"rings":{"mcp_declared":false,"mcp_live":false,"mcp_auth":false,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":3,"fh":"www.mozilla.org","fp_other_host":[],"fp":"4b53d53d5d2e04c8","fp_bits":{"gptbot":0,"claudebot":0,"oaisearch":0,"perplexity":0}},"derive_v":5,"frame":"top10k","shard":0},{"v":1,"kind":"crawlcheck-census-record","run":18,"domain":"highwebmedia.com","rank":98,"measured_at":"2026-11-13T18:49:01.730Z","home":{"browser":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"highwebmedia.com","final_path":"/","ms":5,"identity":"Mobile browser","html":false},"gptbot":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"highwebmedia.com","final_path":"/","ms":5,"identity":"GPTBot","html":false},"claudebot":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"highwebmedia.com","final_path":"/","ms":5,"identity":"ClaudeBot","html":false},"oaisearch":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"highwebmedia.com","final_path":"/","ms":4,"identity":"OAI-SearchBot","html":false},"perplexity":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"highwebmedia.com","final_path":"/","ms":1,"identity":"PerplexityBot","html":false}},"robots":{"status":530,"content_type":"text/plain","bytes":17,"sha256":"71310e1f2fd12c6e8cf5d3ce9ff20e37774d35782b657f4c0c4be8beaa971ecc","final_host":"highwebmedia.com","final_path":"/robots.txt","ms":0,"html":false,"text":null,"agents":null},"sitemap":null,"files":null,"derived":{"reachable":false,"home_class":"no_site","robots_served":false,"robots_sitemap":false,"content_signal":false,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":[],"blocks_any_ai":false,"files":{"llms_txt":false,"agents_md":false,"agent_skills":false,"mcp_card":false,"api_catalog":false,"a2a_card":false},"any_capability":false,"ai_refused":[],"ai_thin":[],"rings":{"mcp_declared":false,"mcp_live":false,"mcp_auth":false,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":3,"fh":"highwebmedia.com","fp_other_host":[]},"derive_v":5,"frame":"top10k","shard":1},{"v":1,"kind":"crawlcheck-census-record","run":18,"domain":"yandex.ru","rank":99,"measured_at":"2026-11-13T18:50:07.634Z","home":{"browser":{"status":200,"content_type":"text/html","bytes":3061,"sha256":"9574cb686bbb41cc37bb9a329e778cc5579a34402db9f49ea7c9eaf41a8b6a53","final_host":"sso.passport.yandex.ru","final_path":"/push","ms":986,"identity":"Mobile browser","html":true},"gptbot":{"status":200,"content_type":"text/html","bytes":2901,"sha256":"bec231d32e841f069d54e5033710917a810495e51e280838ecf19789ffcaf77e","final_host":"sso.passport.yandex.ru","final_path":"/push","ms":977,"identity":"GPTBot","html":true},"claudebot":{"status":200,"content_type":"text/html","bytes":2901,"sha256":"974a3a115c3027d04be5ee8a910e07fa9bd7f00e82f8160d8d58bad40da24896","final_host":"sso.passport.yandex.ru","final_path":"/push","ms":981,"identity":"ClaudeBot","html":true},"oaisearch":{"status":200,"content_type":"text/html","bytes":2901,"sha256":"4a9b68a4948d80b6ab9d161a5d2f4f54f3606f951f0793e8f33970f6b9971ad3","final_host":"sso.passport.yandex.ru","final_path":"/push","ms":982,"identity":"OAI-SearchBot","html":true},"perplexity":{"status":200,"content_type":"text/html","bytes":2900,"sha256":"30039e3bb9d43a02b40202f3c18e5f386cce6a60c075fb5588412faf385e43e6","final_host":"sso.passport.yandex.ru","final_path":"/push","ms":891,"identity":"PerplexityBot","html":true}},"robots":{"status":200,"content_type":"text/plain","bytes":40657,"sha256":"c51a208183d07eb761c7dea99d104021b60c0840f91d43194f016aa5a44a3f46","final_host":"yandex.ru","final_path":"/robots.txt","ms":168,"html":false,"text":"# yandex.ru\nUser-agent: *\nDisallow: /?\nDisallow: /403.html\nDisallow: /404.html\nDisallow: /500.html\nDisallow: /adddata\nDisallow: /adresa-segmentator\nDisallow: /advanced_engl.html\nDisallow: /advertising\nDisallow: /alice/term/\nDisallow: /alice/*/term/\nDisallow: /alice/old-browser\nDisallow: /alice/store/orders/\nDisallow: /alice*?\nDisallow: /alice/*&\nDisallow: /alice/chat/\nDisallow: /alice?share=\nDisallow: /all-supported-params\nDisallow: /articles\nDisallow: /aura/\nAllow: /aura/promo/\nDisallow: /blog/*?text=\nDisallow: /blog/*?tag=\nDisallow: /blog/*/*/*\nDisallow: /blog/punto\nDisallow: /MAPS/\nDisallow: /MaPS/\nDisallow: /MapS/\nDisallow: /Maps/\nDisallow: /mApS/\nDisallow: /harita/business/\nDisallow: /maps/business/\nDisallow: /maps-api/docs/js-api/reference-guide/instruction-gxs.html\nDisallow: /maps-api/docs/js-api/reference-guide/instruction-rs.html\nDisallow: /catalog/?text=\nDisallow: /chat\nDisallow: /cgi-bin/\nDisallow: /cgi/\nDisallow: /clck\nDisallow: /collections/\nDisallow: /company/*.rss\nDisallow: /company/search\nDisallow: /comments/*\nDisallow: /courier\nDisallow: /conflagexp\nDisallow: /cy\nDisallow: /cycounter\nDisallow: /dzen\nDisallow: /experiments.xml\nDisallow: /formfeedback\nDisallow: /fresh\nDisallow: /goto_issue/\nDisallow: /goto_rubric/\nDisallow: /gundem\nDisallow: /i/\nDisallow: /images-data\nDisallow: /images.html\nDisallow: /images/*\nDisallow: /images-apphost/*\nAllow: /images/$\nAllow: /images/smart/$\nAllow: /images/touch/$\nAllow: /images/touch/?\nAllow: /images/search?text=*&source=morda*\nAllow: /images/search?rpt=imageview\nAllow: /images/touch/search?text=*&source=morda*\nAllow: /images/touch/search?rpt=imageview\nAllow: /images/pad/search?text=*&source=morda*\nAllow: /images/pad/search?rpt=imageview\nDisallow: /index_m\nDisallow: /iznanka/\nDisallow: /int_search_summary\nDisallow: /delivery_tracking\nDisallow: /jobs/skill-diagnostic/private/*\nDisallow: /keyboard_qwerty.html\nDisallow: /logotypes\nDisallow: /business/widget\nDisallow: /local/\nDisallow: /lab/neurodetector\nDisallow: /lab/habr-detector\nDisallow: /more_samples\nDisallow: /msearch\nDisallow: /map-constructor/loader*\nDisallow: /maps/print/*\nAllow: /maps/*?lang=ru$\nAllow: /maps/*?lang=uk$\nAllow: /maps/*?lang=kk$\nAllow: /maps/*?lang=uz$\nDisallow: /nmaps/*?\nDisallow: /mapeditor/*?\nDisallow: /metro/*?from\nDisallow: /masters\nDisallow: /services/executor/\nDisallow: /n?\nDisallow: /n$\nDisallow: /n/\nDisallow: /neuralsearch/api\nDisallow: /norobot\nDisallow: /opensearch.xml\nDisallow: /padsearch\nDisallow: /person\nDisallow: /podpiska/login.pl\nDisallow: /pogoda/search\nDisallow: /pogoda/404\nDisallow: /pogoda/0\nDisallow: /polling\nDisallow: /poll/\nDisallow: /portal/set/\nDisallow: /portal/tune/\nDisallow: /prefetch\nDisallow: /promo/*goodbye*\nDisallow: /promo/diskelement\nDisallow: /promo/launcher/mgoodbye/*\nDisallow: /promo/*welcome*\nDisallow: /promo/yobject/changelog*\nDisallow: /project/eda/partnership_services/$\nDisallow: /q/profile/*/answers\nDisallow: /q/profile/*/questions\nDisallow: /q/profile/*/verifies\nDisallow: /q/profile/*/articles\nDisallow: /q/profile/*/subscribers\nDisallow: /q/profile/*/org_answers\nDisallow: /q/user/*/answers\nDisallow: /q/user/*/questions\nDisallow: /q/user/*/verifies\nDisallow: /q/user/*/articles\nDisallow: /q/user/*/subscribers\nDisallow: /q/user/*/org_answers\nDisallow: /soft/bm/goodbye\nDisallow: /soft/chrome/ext-install/*\nDisallow: /soft/chrome/searchline-install/*\nDisallow: /soft/*goodbye*\nDisallow: /soft/punto/mac/uninstall/*\nDisallow: /soft/punto/win/uninstall/*\nDisallow: /soft/*welcome*\nDisallow: /soft/win/?*\nDisallow: /quotes\nDisallow: /realty*?*pinned_offer_id=\nDisallow: /realty*?*pinned_nb_id=\nDisallow: /realty/search\nDisallow: /realty*?*text=\nDisallow: /realty/result\nDisallow: /realty/map_pins\nDisallow: /realty/map_dust\nDisallow: /realty/callme\nDisallow: /redir\nDisallow: /region_map\nDisallow: /regions_list.xml\nDisallow: /regions.html?\nDisallow: /s/\nDisallow: /save\nDisallow: /safety/?\nDisallow: /safety?\nDisallow: /search\nAllow: /search/entity/touch/\nDisallow: /setup\nDisallow: /showcaptcha\nDisallow: /sitesearch\nDisallow: /sl/*.html\nDisallow: /soft/extensions/goodbye\nDisallow: /sportagent\nDisallow: /storeclick\nDisallow: /storerequest\nDisallow: /subscribe/confirm.pl\nDisallow: /subscribe/view.pl\nAllow: /support/\nDisallow: /support/direct-images\nDisallow: /support/direct-tooltips\nDisallow: /support/distr\nDisallow: /support/dsp\nDisallow: /support/fe4be44a295cc679e19bf0b8f133083d\nDisallow: /support/maps-beta\nDisallow: /support/market-images\nDisallow: /support/security\nDisallow: /support/webmaster-images\nDisallow: /support/*zout_\nDisallow: /support/search-results/\nDisallow: /support/praktikum/flow.html\nDisallow: /support/praktikum-english/\nDisallow: /telsearch\nDisallow: /themes\nDisallow: /toggle-experiment\nDisallow: /touchsearch\nDisallow: /tune*retpath=\nDisallow: /turbo\nAllow: /ugcpub/\nDisallow: /ugcpub/cabinet\nDisallow: /v$\nDisallow: /viewconfig$\nDisallow: /images/v$\nDisallow: /images/viewconfig$\nAllow: /tutor\nDisallow: /tutor/admin\nDisallow: /tutor/search/problems\nDisallow: /versions\nDisallow: /video/*\nAllow: /video/$\nAllow: /video/?\nAllow: /video/sitemap\nAllow: /video/movies/entity\nAllow: /video/touch/movies/entity\nAllow: /video/touch/$\nAllow: /video/touch/?\nAllow: /video/preview\nAllow: /video/touch/preview\nAllow: /video/search?text=video$\nAllow: /video/touch/search?text=video$\nAllow: /video/search?text=%D0%B2%D0%B8%D0%B4%D0%B5%D0%BE$\nAllow: /video/touch/search?text=%D0%B2%D0%B8%D0%B4%D0%B5%D0%BE$\nAllow: /video/video_sitemap_*.xml\nAllow: /video/movies_sitemap_*.xml\nDisallow: /xmlsearch\nDisallow: /yandsearch\nDisallow: /yca/cy\nDisallow: /soft/distribution\nDisallow: /soft/?*\nDisallow: /promo/launcher/?*\nDisallow: /opera/?*\nDisallow: /firefox/?*\nDisallow: /soft/bm/?*\nDisallow: /soft/browsers/?*\nDisallow: /soft/punto/?*\nDisallow: /ie/?*\nDisallow: /soft/*goodbye*\nDisallow: /adult\nDisallow: /addappterless\nDisallow: /archive/api/image.jpg*\nAllow: /archive$\nAllow: /archive/project/sovsport$\nDisallow: /all?\nDisallow: /games/*?*reqId*\nDisallow: /games/*?*pageId*\nDisallow: /games/*?experimental=\nDisallow: /games/*?*app-id=\nDisallow: /games/*?*game-id=\nDisallow: /games/*?*div-rtx-reqid=\nDisallow: /games/*?*header=\nDisallow: /games/*?*from=\nDisallow: /games/*?*clid=\nDisallow: /games/*?*deviceid=\nDisallow: /games/*?*flags=\nDisallow: /games/*?*inapp_id=\nDisallow: /games/*?*k50id=\nDisallow: /games/*?*lang=\nDisallow: /games/*?*payload=\nDisallow: /games/*?*pfid=\nDisallow: /games/*?*promo_id=\nDisallow: /games/*?*promo_intent=\nDisallow: /games/*?*referrer=\nDisallow: /games/*?*utm_\nDisallow: /games/*?*ypr=\nDisallow: /familysearch\nDisallow: /schoolsearch\nDisallow: /yandpage\nDisallow: *maps/covid19*\nDisallow: /promo/market/mobile/app*?from=\nDisallow: /support2/*zout_\nDisallow: /activity/\nDisallow: /sport\nDisallow: /support2/sport\n\nAllow: /medicine\nDisallow: /medicine/search\nDisallow: /medicine?text=\nDisallow: /finance\nAllow: /finance/reviews\nAllow: /finance/banks/*/reviews\nDisallow: /finance/reviews*?*\nDisallow: /finance/banks/*/reviews*?\n\nSitemap: https://yandex.ru/support/sitemap.xml\nSitemap: https://yandex.ru/blog/sitemap.xml\nSitemap: https://frontend.s3.yandex.net/health/sitemap.xml\nSitemap: https://yandex.ru/uslugi/sitemap/sitemap.xml.gz\nSitemap: https://yandex.ru/turbo/public/sitemap.xml\nSitemap: https://yandex.ru/tutor/sitemap/?sitemap=sitemap.xml\nSitemap: https://yandex.ru/turbo/talk-sitemap-index.xml\nSitemap: https://yandex.ru/games/sitemaps/sitemap.index.xml\nSitemap: https://yandex.ru/q/sitemap/sitemap-index.xml\nSitemap: https://yandex.ru/images/sitemap.xml\nSitemap: https://yandex.ru/medicine/sitemaps/sitemap.xml\nSitemap: https://yandex.ru/finance/sitemaps/sitemap.xml\nSitemap: https://yandex.ru/yandexapp/ru/callerid/sitemap.xml\nSitemap: https://yandex.ru/video/sitemap_index.xml\n\nUser-agent: Bingbot\nDisallow: /?\nDisallow: /403.html\nDisallow: /404.html\nDisallow: /500.html\nDisallow: /adddata\nDisallow: /adresa-segmentator\nDisallow: /advanced_engl.html\nDisallow: /advertising\nDisallow: /alice/term/\nDisallow: /alice/*/term/\nDisallow: /alice/old-browser\nDisallow: /alice/store/orders/\nDisallow: /alice*?\nDisallow: /alice/*&\nDisallow: /alice/chat/\nDisallow: /alice?share=\nDisallow: /all-supported-params\nDisallow: /articles\nDisallow: /aura/\nAllow: /aura/promo/\nDisallow: /blog/*?text=\nDisallow: /blog/*?tag=\nDisallow: /blog/*/*/*\nDisallow: /blog/punto\nDisallow: /MAPS/\nDisallow: /MaPS/\nDisallow: /MapS/\nDisallow: /Maps/\nDisallow: /mApS/\nDisallow: /maps-api/docs/js-api/reference-guide/instruction-gxs.html\nDisallow: /maps-api/docs/js-api/reference-guide/instruction-rs.html\nDisallow: /catalog/?text=\nDisallow: /chat\nDisallow: /cgi-bin/\nDisallow: /cgi/\nDisallow: /clck\nDisallow: /collections/\nDisallow: /company/*.rss\nDisallow: /company/search\nDisallow: /comments/*\nDisallow: /courier\nDisallow: /conflagexp\nDisallow: /cy\nDisallow: /cycounter\nDisallow: /dzen\nDisallow: /experiments.xml\nDisallow: /formfeedback\nDisallow: /fresh\nDisallow: /goto_issue/\nDisallow: /goto_rubric/\nDisallow: /i/\nDisallow: /images-data\nDisallow: /images.html\nDisallow: /images/*\nDisallow: /images-apphost/*\nAllow: /images/$\nAllow: /images/smart/$\nAllow: /images/touch/$\nAllow: /images/touch/?\nAllow: /images/search?text=*&source=morda*\nAllow: /images/search?rpt=imageview\nAllow: /images/touch/search?text=*&source=morda*\nAllow: /images/touch/search?rpt=imageview\nAllow: /images/pad/search?text=*&source=morda*\nAllow: /images/pad/search?rpt=imageview\nDisallow: /index_m\nDisallow: /iznanka/\nDisallow: /int_search_summary\nDisallow: /delivery_tracking\nDisallow: /gundem\nDisallow: /jobs/skill-diagnostic/private/*\nDisallow: /keyboard_qwerty.html\nDisallow: /logotypes\nDisallow: /business/widget\nDisallow: /local/\nDisallow: /lab/neurodetector\nDisallow: /lab/habr-detector\nDisallow: /more_samples\nDisallow: /msearch\nDisallow: /map-constructor/loader*\nDisallow: /maps/print/*\nAllow: /maps/*?lang=ru$\nAllow: /maps/*?lang=uk$\nAllow: /maps/*?lang=kk$\nAllow: /maps/*?lang=uz$\nDisallow: /nmaps/*?\nDisallow: /mapeditor/*?\nDisallow: /metro/*?from\nDisallow: /masters\nDisallow: /services/executor/\nDisallow: /n?\nDisallow: /n$\nDisallow: /n/\nDisallow: /norobot\nDisallow: /opensearch.xml\nDisallow: /padsearch\nDisallow: /person\nDisallow: /podpiska/login.pl\nDisallow: /pogoda/search\nDisallow: /pogoda/404\nDisallow: /polling\nDisallow: /poll/\nDisallow: /portal/set/\nDisallow: /portal/tune/\nDisallow: /prefetch\nDisallow: /promo/*goodbye*\nDisallow: /promo/diskelement\nDisallow: /promo/launcher/mgoodbye/*\nDisallow: /promo/*welcome*\nDisallow: /promo/yobject/changelog*\nDisallow: /project/eda/partnership_services/$\nDisallow: /q/profile/*/answers\nDisallow: /q/profile/*/questions\nDisallow: /q/profile/*/verifies\nDisallow: /q/profile/*/articles\nDisallow: /q/profile/*/subscribers\nDisallow: /q/profile/*/org_answers\nDisallow: /q/user/*/answers\nDisallow: /q/user/*/questions\nDisallow: /q/user/*/verifies\nDisallow: /q/user/*/articles\nDisallow: /q/user/*/subscribers\nDisallow: /q/user/*/org_answers\nDisallow: /soft/bm/goodbye\nDisallow: /soft/chrome/ext-install/*\nDisallow: /soft/chrome/searchline-install/*\nDisallow: /soft/*goodbye*\nDisallow: /soft/punto/mac/uninstall/*\nDisallow: /soft/punto/win/uninstall/*\nDisallow: /soft/*welcome*\nDisallow: /soft/win/?*\nDisallow: /quotes\nDisallow: /realty*?*pinned_offer_id=\nDisallow: /realty*?*pinned_nb_id=\nDisallow: /realty/search\nDisallow: /realty*?*text=\nDisallow: /realty/result\nDisallow: /realty/map_pins\nDisallow: /realty/map_dust\nDisallow: /realty/callme\nDisallow: /redir\nDisallow: /region_map\nDisallow: /regions_list.xml\nDisallow: /regions.html?\nDisallow: /s/\nDisallow: /save\nDisallow: /safety/?\nDisallow: /safety?\nDisallow: /search\nAllow: /search/entity/touch/\nAllow: /search?products_mode=1\nAllow: /search/?products_mode=1\nAllow: /search?*&products_mode=1\nAllow: /search/?*&products_mode=1\nAllow: /search/touch?products_mode=1\nAllow: /search/touch/?products_mode=1\nAllow: /search/touch?*&products_mode=1\nAllow: /search/touch/?*&products_mode=1\nAllow: /search/products\nAllow: /products/$\nAllow: /products$\nDisallow: /setup\nDisallow: /showcaptcha\nDisallow: /sitesearch\nDisallow: /sl/*.html\nDisallow: /soft/extensions/goodbye\nDisallow: /sportagent\nDisallow: /storeclick\nDisallow: /storerequest\nDisallow: /subscribe/confirm.pl\nDisallow: /subscribe/view.pl\nAllow: /support/\nDisallow: /support/direct-images\nDisallow: /support/direct-tooltips\nDisallow: /support/distr\nDisallow: /support/dsp\nDisallow: /support/fe4be44a295cc679e19bf0b8f133083d\nDisallow: /support/maps-beta\nDisallow: /support/market-images\nDisallow: /support/security\nDisallow: /support/webmaster-images\nDisallow: /support/*zout_\nDisallow: /support/search-results/\nDisallow: /support/praktikum/flow.html\nDisallow: /support/praktikum-english/\nDisallow: /telsearch\nDisallow: /themes\nDisallow: /toggle-experiment\nDisallow: /touchsearch\nDisallow: /tune*retpath=\nAllow: /ugcpub/\nDisallow: /ugcpub/cabinet\nDisallow: /v$\nDisallow: /viewconfig$\nDisallow: /images/v$\nDisallow: /images/viewconfig$\nAllow: /tutor\nDisallow: /tutor/admin\nDisallow: /tutor/search/problems\nDisallow: /turbo\nDisallow: /versions\nDisallow: /video/*\nAllow: /video/$\nAllow: /video/?\nAllow: /video/sitemap\nAllow: /video/movies/entity\nAllow: /video/touch/movies/entity\nAllow: /video/touch/$\nAllow: /video/touch/?\nAllow: /video/preview\nAllow: /video/touch/preview\nAllow: /video/search?text=video$\nAllow: /video/touch/search?text=video$\nAllow: /video/search?text=%D0%B2%D0%B8%D0%B4%D0%B5%D0%BE$\nAllow: /video/touch/search?text=%D0%B2%D0%B8%D0%B4%D0%B5%D0%BE$\nAllow: /video/video_sitemap_*.xml\nAllow: /video/movies_sitemap_*.xml\nDisallow: /xmlsearch\nDisallow: /yandsearch\nDisallow: /yca/cy\nDisallow: /soft/distribution\nDisallow: /soft/?*\nDisallow: /promo/launcher/?*\nDisallow: /opera/?*\nDisallow: /firefox/?*\nDisallow: /soft/bm/?*\nDisallow: /soft/browsers/?*\nDisallow: /soft/punto/?*\nDisallow: /ie/?*\nDisallow: /soft/*goodbye*\nDisallow: /adult\nDisallow: /addappterless\nDisallow: /archive/api/image.jpg*\nAllow: /archive$\nAllow: /archive/project/sovsport$\nDisallow: /all?\nDisallow: /games/*?*reqId*\nDisallow: /games/*?*pageId*\nDisallow: /games/*?experimental=\nDisallow: /games/*?*app-id=\nDisallow: /games/*?*game-id=\nDisallow: /games/*?*div-rtx-reqid=\nDisallow: /games/*?*header=\nDisallow: /games/*?*from=\nDisallow: /games/*?*clid=\nDisallow: /games/*?*deviceid=\nDisallow: /games/*?*flags=\nDisallow: /games/*?*inapp_id=\nDisallow: /games/*?*k50id=\nDisallow: /games/*?*lang=\nDisallow: /games/*?*payload=\nDisallow: /games/*?*pfid=\nDisallow: /games/*?*promo_id=\nDisallow: /games/*?*promo_intent=\nDisallow: /games/*?*referrer=\nDisallow: /games/*?*utm_\nDisallow: /games/*?*ypr=\nDisallow: /products\nDisallow: /familysearch\nDisallow: /schoolsearch\nDisallow: /yandpage\nDisallow: *maps/covid19*\nDisallow: /promo/market/mobile/app*?from=\nDisallow: /support2/*zout_\nDisallow: /activity/\nDisallow: /sport\nDisallow: /support2/sport*\n\nAllow: /medicine\nDisallow: /medicine/search\nDisallow: /medicine?text=\nDisallow: /finance*?\nDisallow: /finance/assistant\n\nUser-agent: Yandex\nDisallow: /?\nDisallow: /403.html\nDisallow: /404.html\nDisallow: /500.html\nDisallow: /adddata\nDisallow: /adresa-segmentator\nDisallow: /advanced_engl.html\nDisallow: /advertising\nDisallow: /adv/news/vystavka-prodazha-na-poiske-poyavilas-tovarnaya-galereya\nDisallow: /adv?nw=\nDisallow: /adv?from=\nDisallow: /adv?email_form=\nDisallow: /adv?ref=\nDisallow: /alice/term/\nDisallow: /alice/*/term/\nDisallow: /alice/old-browser\nDisallow: /alice/chat/\nDisallow: /alice?share=\nDisallow: /alice*?\nDisallow: /all-supported-params\nDisallow: /articles\nDisallow: /aura/\nAllow: /aura/promo/\nDisallow: /blog/*?text=\nDisallow: /blog/*?tag=\nDisallow: /blog/*/*/*\nDisallow: /blog/punto\nDisallow: /MAPS/\nDisallow: /MaPS/\nDisallow: /MapS/\nDisallow: /Maps/\nDisallow: /mApS/\nDisallow: /catalog/?text=\nDisallow: /chat\nDisallow: /cgi-bin/\nDisallow: /cgi/\nDisallow: /clck\nDisallow: /collections/\nDisallow: /company/*.rss\nDisallow: /company/search\nDisallow: /business/widget\nDisallow: /comments/*\nDisallow: /courier\nDisallow: /products\nDisallow: /conflagexp\nDisallow: /cy\nDisallow: /cycounter\nDisallow: /dzen\nDisallow: /experiments.xml\nDisallow: /formfeedback\nDisallow: /fresh\nDisallow: /games/*?experimental=\nDisallow: /goto_issue/\nDisallow: /goto_rubric/\nDisallow: /i/\nDisallow: /images-data\nDisallow: /images.html\nDisallow: /images/*\nAllow: /images/$\nAllow: /images/smart/$\nAllow: /images/touch/$\nAllow: /images/touch/?\nAllow: /images/search?text=*&source=morda*\nAllow: /images/search?rpt=imageview\nAllow: /images/touch/search?text=*&source=morda*\nAllow: /images/touch/search?rpt=imageview\nAllow: /images/pad/search?text=*&source=morda*\nAllow: /images/pad/search?rpt=imageview\nDisallow: /index_m\nDisallow: /iznanka/\nDisallow: /int_search_summary\nDisallow: /delivery_tracking\nDisallow: /jobs/skill-diagnostic/private/*\nDisallow: /keyboard_qwerty.html\nDisallow: /logotypes\nDisallow: /local/\nDisallow: /lab/neurodetector\nDisallow: /lab/habr-detector\nDisallow: /more_samples\nDisallow: /msearch\nDisallow: /map-constructor/loader*\nDisallow: /maps/print/*\nDisallow: /harita/business/\nDisallow: /maps/business/\nDisallow: /maps-api/docs/js-api/reference-guide/instruction-gxs.html\nDisallow: /maps-api/docs/js-api/reference-guide/instruction-rs.html\nAllow: /maps/*?lang=ru$\nAllow: /maps/*?lang=uk$\nAllow: /maps/org/\nDisallow: /nmaps/*?\nDisallow: /mapeditor/*?\nDisallow: /metro/*?from\nDisallow: /masters\nDisallow: /services\nDisallow: /neuralsearch/api\nDisallow: /n?\nDisallow: /n$\nDisallow: /n/\nDisallow: /norobot\nDisallow: /opensearch.xml\nDisallow: /padsearch\nDisallow: /person\nDisallow: /podpiska/login.pl\nDisallow: /pogoda/amp/*\nDisallow: /pogoda/search\nDisallow: /pogoda/404\nDisallow: /pogoda/0\nDisallow: /polling\nDisallow: /poll/\nDisallow: /portal/set/\nDisallow: /portal/tune/\nDisallow: /promo/*goodbye*\nDisallow: /promo/diskelement\nDisallow: /promo/launcher/mgoodbye/*\nDisallow: /promo/*welcome*\nDisallow: /promo/yobject/changelog*\nDisallow: /project/eda/partnership_services/$\nDisallow: /q/profile/*/answers\nDisallow: /q/profile/*/questions\nDisallow: /q/profile/*/verifies\nDisallow: /q/profile/*/articles\nDisallow: /q/profile/*/subscribers\nDisallow: /q/profile/*/org_answers\nDisallow: /q/user/*/answers\nDisallow: /q/user/*/questions\nDisallow: /q/user/*/verifies\nDisallow: /q/user/*/articles\nDisallow: /q/user/*/subscribers\nDisallow: /q/user/*/org_answers\nDisallow: /soft/bm/goodbye\nDisallow: /soft/chrome/ext-install/*\nDisallow: /soft/chrome/searchline-install/*\nDisallow: /soft/*goodbye*\nDisallow: /soft/punto/mac/uninstall/*\nDisallow: /soft/punto/win/uninstall/*\nDisallow: /soft/*welcome*\nDisallow: /soft/win/?*\nDisallow: /quotes\nDisallow: /realty\nDisallow: /redir\nDisallow: /region_map\nDisallow: /regions_list.xml\nDisallow: /regions.html?\nDisallow: /s/\nDisallow: /save\nDisallow: /safety/?*\nDisallow: /search\nAllow: /search/entity/touch/\nDisallow: /setup\nDisallow: /showcaptcha\nDisallow: /sitesearch\nDisallow: /sl/*.html\nDisallow: /soft/extensions/goodbye\nDisallow: /sportagent\nDisallow: /storeclick\nDisallow: /storerequest\nDisallow: /subscribe/confirm.pl\nDisallow: /subscribe/view.pl\nAllow: /support/\nDisallow: /support/direct-images\nDisallow: /support/direct-tooltips\nDisallow: /support/distr\nDisallow: /support/dsp\nDisallow: /support/fe4be44a295cc679e19bf0b8f133083d\nDisallow: /support/maps-beta\nDisallow: /support/market-images\nDisallow: /support/security\nDisallow: /support/webmaster-images\nDisallow: /support/*zout_\nDisallow: /support/search-results/\nDisallow: /support/praktikum/flow.html\nDisallow: /support/praktikum-english/\nDisallow: /telsearch\nDisallow: /themes\nDisallow: /toggle-experiment\nDisallow: /touchsearch\nDisallow: /tune*retpath=\nDisallow: /8bit-fest/parents\nAllow: /tutor\nDisallow: /tutor/admin\nDisallow: /turbo\nAllow: /ugcpub/\nDisallow: /ugcpub/cabinet\nDisallow: /v$\nDisallow: /viewconfig$\nDisallow: /images/v$\nDisallow: /images/viewconfig$\nDisallow: /versions\nDisallow: /video/*\nAllow: /video/$\nAllow: /video/?\nAllow: /video/sitemap\nAllow: /video/touch/$\nAllow: /video/touch/?\nAllow: /video/preview\nAllow: /video/touch/preview\nAllow: /video/search?text=video$\nAllow: /video/touch/search?text=video$\nAllow: /video/search?text=%D0%B2%D0%B8%D0%B4%D0%B5%D0%BE$\nAllow: /video/touch/search?text=%D0%B2%D0%B8%D0%B4%D0%B5%D0%BE$\nAllow: /video/video_sitemap_*.xml\nAllow: /video/movies_sitemap_*.xml\nDisallow: /xmlsearch\nDisallow: /yandsearch\nDisallow: /yca/cy\nDisallow: /soft/distribution\nDisallow: /soft/?*\nDisallow: /promo/launcher/?*\nDisallow: /opera/?*\nDisallow: /firefox/?*\nDisallow: /soft/bm/?*\nDisallow: /soft/browsers/?*\nDisallow: /soft/punto/?*\nDisallow: /ie/?*\nDisallow: /soft/*goodbye*\nDisallow: /adult\nDisallow: /addappterless\nDisallow: /archive/api/image.jpg*\nAllow: /archive$\nAllow: /archive/project/sovsport$\nDisallow: /all?\nDisallow: /familysearch\nDisallow: /schoolsearch\nDisallow: /yandpage\nDisallow: *maps/covid19*\nDisallow: /promo/market/mobile/app*?from=\nDisallow: /support2/*zout_\nDisallow: /activity/\nDisallow: /sport\nDisallow: /support2/sport*\n\nDisallow: /medicine\nDisallow: /finance\n\nDisallow: /gundem\n\nUser-agent: Googlebot\nDisallow: /?\nDisallow: /403.html\nDisallow: /404.html\nDisallow: /500.html\nDisallow: /adddata\nDisallow: /adresa-segmentator\nDisallow: /advanced_engl.html\nDisallow: /advertising\nDisallow: /ads/\nDisallow: /adfox/\nDisallow: /an/\nDisallow: /alice/term/\nDisallow: /alice/*/term/\nDisallow: /alice/old-browser\nDisallow: /alice/store/orders/\nDisallow: /alice*?\nDisallow: /alice/*&\nDisallow: /alice/chat/\nDisallow: /alice?share=\nDisallow: /all-supported-params\nDisallow: /articles\nDisallow: /archive/search\nDisallow: /aura/\nAllow: /aura/promo/\nDisallow: /blog/*?text=\nDisallow: /blog/*?tag=\nDisallow: /blog/*/*/*\nDisallow: /blog/punto\nDisallow: /business/widget\nDisallow: /MAPS/\nDisallow: /MaPS/\nDisallow: /MapS/\nDisallow: /Maps/\nDisallow: /mApS/\nDisallow: /maps-api/docs/js-api/reference-guide/instruction-gxs.html\nDisallow: /maps-api/docs/js-api/reference-guide/instruction-rs.html\nDisallow: /catalog/?text=\nDisallow: /cgi-bin/\nDisallow: /cgi/\nDisallow: /clck\nDisallow: /collections/\nDisallow: /company/*.rss\nDisallow: /company/search\nDisallow: /comments/*\nDisallow: /courier\nDisallow: /conflagexp\nDisallow: /cy\nDisallow: /cycounter\nDisallow: /dzen\nDisallow: /experiments.xml\nDisallow: /formfeedback\nDisallow: /fresh\nDisallow: /goto_issue/\nDisallow: /goto_rubric/\nDisallow: /gundem\nDisallow: /i/\nDisallow: /images-data\nDisallow: /images.html\nDisallow: /images/\nDisallow: /images-apphost/\nAllow: /images$\nAllow: /images/$\nAllow: /images/?\nAllow: /images/smart/$\nAllow: /images/touch$\nAllow: /images/touch/$\nAllow: /images/touch/?\nAllow: /images/pad/search?text=*&source=morda*\nAllow: /images/pad/search?rpt=imageview\nAllow: /images/search?\nAllow: /images/touch/search?\n\nAllow: /images/sitemap\nDisallow: /index_m\nDisallow: /iznanka/\nDisallow: /int_search_summary\nDisallow: /delivery_tracking\nDisallow: /jobs/skill-diagnostic/private/*\nDisallow: /keyboard_qwerty.html\nDisallow: /logotypes\nDisallow: /local/\nDisallow: /lab/neurodetector\nDisallow: /lab/habr-detector\nDisallow: /more_samples\nDisallow: /msearch\nDisallow: /map-constructor/loader*\nDisallow: /maps/print/*\nDisallow: /harita/business/\nDisallow: /maps/business/\nAllow: /maps/*?lang=ru$\nAllow: /maps/*?lang=uk$\nAllow: /maps/*?lang=kk$\nAllow: /maps/*?lang=uz$\nDisallow: /nmaps/*?\nDisallow: /mapeditor/*?\nDisallow: /metro/*?from\nDisallow: /masters\nDisallow: /services/executor/\nDisallow: /neuralsearch/api\nDisallow: /n?\nDisallow: /n$\nDisallow: /n/\nDisallow: /norobot\nDisallow: /opensearch.xml\nDisallow: /padsearch\nAllow: /products/sitemaps/new/*\nDisallow: /person\nDisallow: /podpiska/login.pl\nDisallow: /pogoda/404\nDisallow: /polling\nDisallow: /poll/\nDisallow: /portal/set/\nDisallow: /portal/tune/\nDisallow: /prefetch\nDisallow: /products\nDisallow: /promo/*goodbye*\nDisallow: /promo/diskelement\nDisallow: /promo/launcher/mgoodbye/*\nDisallow: /promo/*welcome*\nDisallow: /promo/yobject/changelog*\nDisallow: /project/eda/partnership_services/$\nDisallow: /q/profile/*/answers\nDisallow: /q/profile/*/questions\nDisallow: /q/profile/*/verifies\nDisallow: /q/profile/*/articles\nDisallow: /q/profile/*/subscribers\nDisallow: /q/profile/*/org_answers\nDisallow: /q/user/*/answers\nDisallow: /q/user/*/questions\nDisallow: /q/user/*/verifies\nDisallow: /q/user/*/articles\nDisallow: /q/user/*/subscribers\nDisallow: /q/user/*/org_answers\nDisallow: /soft/bm/goodbye\nDisallow: /soft/chrome/ext-install/*\nDisallow: /soft/chrome/searchline-install/*\nDisallow: /soft/*goodbye*\nDisallow: /soft/punto/mac/uninstall/*\nDisallow: /soft/punto/win/uninstall/*\nDisallow: /soft/*welcome*\nDisallow: /soft/win/?*\nDisallow: /quotes\nDisallow: /realty*?*pinned_offer_id=\nDisallow: /realty*?*pinned_nb_id=\nDisallow: /realty/search\nDisallow: /realty*?*text=\nDisallow: /realty/result\nDisallow: /realty/map_pins\nDisallow: /realty/map_dust\nDisallow: /realty/callme\nDisallow: /redir\nDisallow: /region_map\nDisallow: /regions_list.xml\nDisallow: /regions.html?\nDisallow: /s/\nDisallow: /save\nDisallow: /safety/?\nDisallow: /safety?\nDisallow: /search\nAllow: /search/entity/touch/\nAllow: /search?products_mode=1\nAllow: /search/?products_mode=1\nAllow: /search?*&products_mode=1\nAllow: /search/?*&products_mode=1\nAllow: /search/touch?products_mode=1\nAllow: /search/touch/?products_mode=1\nAllow: /search/touch?*&products_mode=1\nAllow: /search/touch/?*&products_mode=1\nAllow: /search/products\nAllow: /products/$\nAllow: /products$\nDisallow: /setup\nDisallow: /showcaptcha\nDisallow: /sitesearch\nDisallow: /sl/*.html\nDisallow: /soft/extensions/goodbye\nDisallow: /sportagent\nDisallow: /storeclick\nDisallow: /storerequest\nDisallow: /subscribe/confirm.pl\nDisallow: /subscribe/view.pl\nAllow: /support/\nDisallow: /support/direct-images\nDisallow: /support/direct-tooltips\nDisallow: /support/distr\nDisallow: /support/dsp\nDisallow: /support/fe4be44a295cc679e19bf0b8f133083d\nDisallow: /support/maps-beta\nDisallow: /support/market-images\nDisallow: /support/security\nDisallow: /support/webmaster-images\nDisallow: /support/*zout_\nDisallow: /support/search-results/\nDisallow: /support/praktikum/flow.html\nDisallow: /support/praktikum-english/\nDisallow: /telsearch\nDisallow: /themes\nDisallow: /toggle-experiment\nDisallow: /touchsearch\nDisallow: /tune*retpath=\nDisallow: /turbo\nAllow: /ugcpub/\nDisallow: /ugcpub/cabinet\nDisallow: /v$\nDisallow: /viewconfig$\nDisallow: /images/v$\nDisallow: /images/viewconfig$\nAllow: /tutor\nDisallow: /tutor/admin\nDisallow: /tutor/search/problems\nDisallow: /versions\nDisallow: /video/\nAllow: /video/$\nAllow: /video/?\nAllow: /video/sitemap\nAllow: /video/movies/entity\nAllow: /video/touch/movies/entity\nAllow: /video/touch/$\nAllow: /video/touch/?\nAllow: /video/preview\nAllow: /video/touch/preview\nAllow: /video/search?text=video$\nAllow: /video/touch/search?text=video$\nAllow: /video/search?text=%D0%B2%D0%B8%D0%B4%D0%B5%D0%BE$\nAllow: /video/touch/search?text=%D0%B2%D0%B8%D0%B4%D0%B5%D0%BE$\nAllow: /video/movies/catalog\nAllow: /video/touch/movies/catalog\nAllow: /video/video_sitemap_*.xml\nAllow: /video/movies_sitemap_*.xml\nDisallow: /xmlsearch\nDisallow: /yandsearch\nDisallow: /yca/cy\nDisallow: /soft/distribution\nDisallow: /soft/?*\nDisallow: /promo/launcher/?*\nDisallow: /opera/?*\nDisallow: /firefox/?*\nDisallow: /soft/bm/?*\nDisallow: /soft/browsers/?*\nDisallow: /soft/punto/?*\nDisallow: /ie/?*\nDisallow: /soft/*goodbye*\nDisallow: /adult\nDisallow: /addappterless\nDisallow: /archive/api/image.jpg*\nAllow: /archive$\nAllow: /archive/project/sovsport$\nDisallow: /all?\nDisallow: /games/*?*reqId*\nDisallow: /games/*?*pageId*\nDisallow: /games/*?experimental=\nDisallow: /games/*?*app-id=\nDisallow: /games/*?*game-id=\nDisallow: /games/*?*div-rtx-reqid=\nDisallow: /games/*?*header=\nDisallow: /games/*?*from=\nDisallow: /games/*?*clid=\nDisallow: /games/*?*deviceid=\nDisallow: /games/*?*flags=\nDisallow: /games/*?*inapp_id=\nDisallow: /games/*?*k50id=\nDisallow: /games/*?*lang=\nDisallow: /games/*?*payload=\nDisallow: /games/*?*pfid=\nDisallow: /games/*?*promo_id=\nDisallow: /games/*?*promo_intent=\nDisallow: /games/*?*referrer=\nDisallow: /games/*?*utm_\nDisallow: /games/*?*ypr=\nDisallow: /familysearch\nDisallow: /schoolsearch\nDisallow: /yandpage\nDisallow: *maps/covid19*\nDisallow: /promo/market/mobile/app*?from=\nDisallow: /support2/*zout_\nDisallow: /activity/\nDisallow: /sport\nDisallow: /support2/sport*\n\nAllow: /medicine\nDisallow: /medicine/search\nDisallow: /medicine?text=\nDisallow: /finance*?\nDisallow: /finance/assistant\n\nUser-agent: OAI-SearchBot\nDisallow: /?\nDisallow: /403.html\nDisallow: /404.html\nDisallow: /500.html\nDisallow: /adddata\nDisallow: /adresa-segmentator\nDisallow: /advanced_engl.html\nDisallow: /advertising\nDisallow: /alice/term/\nDisallow: /alice/*/term/\nDisallow: /alice/old-browser\nDisallow: /alice/store/orders/\nDisallow: /alice*?\nDisallow: /alice/*&\nDisallow: /alice/chat/\nDisallow: /alice?share=\nDisallow: /all-supported-params\nDisallow: /articles\nDisallow: /aura/\nAllow: /aura/promo/\nDisallow: /blog/*?text=\nDisallow: /blog/*?tag=\nDisallow: /blog/*/*/*\nDisallow: /blog/punto\nDisallow: /MAPS/\nDisallow: /MaPS/\nDisallow: /MapS/\nDisallow: /Maps/\nDisallow: /mApS/\nDisallow: /harita/business/\nDisallow: /maps/business/\nDisallow: /maps-api/docs/js-api/reference-guide/instruction-gxs.html\nDisallow: /maps-api/docs/js-api/reference-guide/instruction-rs.html\nDisallow: /catalog/?text=\nDisallow: /chat\nDisallow: /cgi-bin/\nDisallow: /cgi/\nDisallow: /clck\nDisallow: /collections/\nDisallow: /company/*.rss\nDisallow: /company/search\nDisallow: /comments/*\nDisallow: /courier\nDisallow: /conflagexp\nDisallow: /cy\nDisallow: /cycounter\nDisallow: /dzen\nDisallow: /experiments.xml\nDisallow: /formfeedback\nDisallow: /fresh\nDisallow: /goto_issue/\nDisallow: /goto_rubric/\nDisallow: /gundem\nDisallow: /i/\nDisallow: /images-data\nDisallow: /images.html\nDisallow: /images/*\nDisallow: /images-apphost/*\nAllow: /images/$\nAllow: /images/smart/$\nAllow: /images/touch/$\nAllow: /images/touch/?\nAllow: /images/search?text=*&source=morda*\nAllow: /images/search?rpt=imageview\nAllow: /images/touch/search?text=*&source=morda*\nAllow: /images/touch/search?rpt=imageview\nAllow: /images/pad/search?text=*&source=morda*\nAllow: /images/pad/search?rpt=imageview\nDisallow: /index_m\nDisallow: /iznanka/\nDisallow: /int_search_summary\nDisallow: /delivery_tracking\nDisallow: /jobs/skill-diagnostic/private/*\nDisallow: /keyboard_qwerty.html\nDisallow: /logotypes\nDisallow: /business/widget\nDisallow: /local/\nDisallow: /lab/neurodetector\nDisallow: /lab/habr-detector\nDisallow: /more_samples\nDisallow: /msearch\nDisallow: /map-constructor/loader*\nDisallow: /maps/print/*\nAllow: /maps/*?lang=ru$\nAllow: /maps/*?lang=uk$\nAllow: /maps/*?lang=kk$\nAllow: /maps/*?lang=uz$\nDisallow: /nmaps/*?\nDisallow: /mapeditor/*?\nDisallow: /metro/*?from\nDisallow: /masters\nDisallow: /services/executor/\nDisallow: /n?\nDisallow: /n$\nDisallow: /n/\nDisallow: /neuralsearch/api\nDisallow: /norobot\nDisallow: /opensearch.xml\nDisallow: /padsearch\nDisallow: /person\nDisallow: /podpiska/login.pl\nDisallow: /pogoda/search\nDisallow: /pogoda/404\nDisallow: /pogoda/0\nDisallow: /polling\nDisallow: /poll/\nDisallow: /portal/set/\nDisallow: /portal/tune/\nDisallow: /prefetch\nDisallow: /promo/*goodbye*\nDisallow: /promo/diskelement\nDisallow: /promo/launcher/mgoodbye/*\nDisallow: /promo/*welcome*\nDisallow: /promo/yobject/changelog*\nDisallow: /project/eda/partnership_services/$\nDisallow: /q/profile/*/answers\nDisallow: /q/profile/*/questions\nDisallow: /q/profile/*/verifies\nDisallow: /q/profile/*/articles\nDisallow: /q/profile/*/subscribers\nDisallow: /q/profile/*/org_answers\nDisallow: /q/user/*/answers\nDisallow: /q/user/*/questions\nDisallow: /q/user/*/verifies\nDisallow: /q/user/*/articles\nDisallow: /q/user/*/subscribers\nDisallow: /q/user/*/org_answers\nDisallow: /soft/bm/goodbye\nDisallow: /soft/chrome/ext-install/*\nDisallow: /soft/chrome/searchline-install/*\nDisallow: /soft/*goodbye*\nDisallow: /soft/punto/mac/uninstall/*\nDisallow: /soft/punto/win/uninstall/*\nDisallow: /soft/*welcome*\nDisallow: /soft/win/?*\nDisallow: /quotes\nDisallow: /realty*?*pinned_offer_id=\nDisallow: /realty*?*pinned_nb_id=\nDisallow: /realty/search\nDisallow: /realty*?*text=\nDisallow: /realty/result\nDisallow: /realty/map_pins\nDisallow: /realty/map_dust\nDisallow: /realty/callme\nDisallow: /redir\nDisallow: /region_map\nDisallow: /regions_list.xml\nDisallow: /regions.html?\nDisallow: /s/\nDisallow: /save\nDisallow: /safety/?\nDisallow: /safety?\nDisallow: /search\nAllow: /search/entity/touch/\nDisallow: /setup\nDisallow: /showcaptcha\nDisallow: /sitesearch\nDisallow: /sl/*.html\nDisallow: /soft/extensions/goodbye\nDisallow: /sportagent\nDisallow: /storeclick\nDisallow: /storerequest\nDisallow: /subscribe/confirm.pl\nDisallow: /subscribe/view.pl\nAllow: /support/\nDisallow: /support/direct-images\nDisallow: /support/direct-tooltips\nDisallow: /support/distr\nDisallow: /support/dsp\nDisallow: /support/fe4be44a295cc679e19bf0b8f133083d\nDisallow: /support/maps-beta\nDisallow: /support/market-images\nDisallow: /support/security\nDisallow: /support/webmaster-images\nDisallow: /support/*zout_\nDisallow: /support/search-results/\nDisallow: /support/praktikum/flow.html\nDisallow: /support/praktikum-english/\nDisallow: /telsearch\nDisallow: /themes\nDisallow: /toggle-experiment\nDisallow: /touchsearch\nDisallow: /tune*retpath=\nDisallow: /turbo\nAllow: /ugcpub/\nDisallow: /ugcpub/cabinet\nDisallow: /v$\nDisallow: /viewconfig$\nDisallow: /images/v$\nDisallow: /images/viewconfig$\nAllow: /tutor\nDisallow: /tutor/admin\nDisallow: /tutor/search/problems\nDisallow: /versions\nDisallow: /video/*\nAllow: /video/$\nAllow: /video/?\nAllow: /video/sitemap\nAllow: /video/movies/entity\nAllow: /video/touch/movies/entity\nAllow: /video/touch/$\nAllow: /video/touch/?\nAllow: /video/preview\nAllow: /video/touch/preview\nAllow: /video/search?text=video$\nAllow: /video/touch/search?text=video$\nAllow: /video/search?text=%D0%B2%D0%B8%D0%B4%D0%B5%D0%BE$\nAllow: /video/touch/search?text=%D0%B2%D0%B8%D0%B4%D0%B5%D0%BE$\nAllow: /video/video_sitemap_*.xml\nAllow: /video/movies_sitemap_*.xml\nDisallow: /xmlsearch\nDisallow: /yandsearch\nDisallow: /yca/cy\nDisallow: /soft/distribution\nDisallow: /soft/?*\nDisallow: /promo/launcher/?*\nDisallow: /opera/?*\nDisallow: /firefox/?*\nDisallow: /soft/bm/?*\nDisallow: /soft/browsers/?*\nDisallow: /soft/punto/?*\nDisallow: /ie/?*\nDisallow: /soft/*goodbye*\nDisallow: /adult\nDisallow: /addappterless\nDisallow: /archive/api/image.jpg*\nAllow: /archive$\nAllow: /archive/project/sovsport$\nDisallow: /all?\nDisallow: /games/*?*reqId*\nDisallow: /games/*?*pageId*\nDisallow: /games/*?experimental=\nDisallow: /games/*?*app-id=\nDisallow: /games/*?*game-id=\nDisallow: /games/*?*div-rtx-reqid=\nDisallow: /games/*?*header=\nDisallow: /games/*?*from=\nDisallow: /games/*?*clid=\nDisallow: /games/*?*deviceid=\nDisallow: /games/*?*flags=\nDisallow: /games/*?*inapp_id=\nDisallow: /games/*?*k50id=\nDisallow: /games/*?*lang=\nDisallow: /games/*?*payload=\nDisallow: /games/*?*pfid=\nDisallow: /games/*?*promo_id=\nDisallow: /games/*?*promo_intent=\nDisallow: /games/*?*referrer=\nDisallow: /games/*?*utm_\nDisallow: /games/*?*ypr=\nDisallow: /familysearch\nDisallow: /schoolsearch\nDisallow: /yandpage\nDisallow: *maps/covid19*\nDisallow: /promo/market/mobile/app*?from=\nDisallow: /support2/*zout_\nDisallow: /activity/\nDisallow: /sport\nDisallow: /support2/sport\n\nAllow: /medicine\nDisallow: /medicine/search\nDisallow: /medicine?text=\nDisallow: /finance\nAllow: /finance/reviews\nAllow: /finance/banks/*/reviews\nDisallow: /finance/reviews*?*\nDisallow: /finance/banks/*/reviews*?\n\nSitemap: https://yandex.ru/pogoda/sitemap-index.xml\nSitemap: https://yandex.ru/support/sitemap.xml\nSitemap: https://yandex.ru/blog/sitemap.xml\nSitemap: https://frontend.s3.yandex.net/health/sitemap.xml\nSitemap: https://yandex.ru/uslugi/sitemap/sitemap.xml.gz\nSitemap: https://yandex.ru/turbo/public/sitemap.xml\nSitemap: https://yandex.ru/tutor/sitemap/?sitemap=sitemap.xml\nSitemap: https://yandex.ru/games/sitemaps/sitemap.index.xml\nSitemap: https://yandex.ru/q/sitemap/sitemap-index.xml\nSitemap: https://yandex.ru/images/sitemap.xml\nSitemap: https://yandex.ru/medicine/sitemaps/sitemap.xml\nSitemap: https://yandex.ru/yandexapp/ru/callerid/sitemap.xml\nSitemap: https://yandex.ru/video/sitemap_index.xml\nSitemap: https://yandex.ru/realty/sitemaps/human_friendly/sitemap_index_hfp-00001.xml\nSitemap: https://yandex.ru/images/sitemap_index.xml\nSitemap: https://yandex.ru/services/sitemaps/sitemap.xml\nSitemap: https://yandex.ru/archive/sitemap.xml\n\nClean-Param: ncrnd&redircnt&clid&_&win /\nClean-Param: utm_source&utm_medium&utm_campaign&utm_term&utm_content&utm_nooverride&utm_expid&gclid&_openstat&yclid&gsaid&_gs_rmark&_gs_ref&_gs_cttl&iframe /health/\nClean-Param: print=1 /tutor/\nClean-Param: w&w_question_id /q/question/*\nClean-Param: &appsearch_header=1&comments=1 /news/story/\nClean-Param: app-id&game-id&div-rtx-reqid&lang&platform&header&from&backQueryParams /games/\nClean-Param: ll&z&sll /maps/\nClean-param: oll&rtext&pt&source&via&from&origin&um&utm_medium&utm_source&portal&panorama%5Bspan%5D&panorama%5Bpoint%5D&panorama%5Bfull%5D&panorama%5Bdirection%5D&panorama%5Bair%5D&masstransit%5BstopId%5D&ol&poi%5Bpoint%5D&whatshere%5Bpoint%5D&whatshere%5Bzoom%5D&ouri&ruri&l&mode&tab&rtt&rtm&display-text&text&indoorLevel&trfm&trfst&no-distribution&distance&slices&scale&descy&descx&mapY&mapX&mapID&sctx&lay&where&results&map&id&spn&sspn&rtn&ost&profile-mode /maps/\nClean-Param: appsearch_header /pogoda\nClean-param: from&publication_id&email_form&id&page&test&year&month&lang&onlyWithCases&gcmes&cmnt_login&_aid&currentURL&res&cmnt_from&amp&comments&certId&mindbox-message-key&status&menu&block&version&ref&etext&ctime&nw&commentId&yqrid&erid&specializations&specializations[]&cities&workFormats&product&filters&autm_medium&industries&search_open&tasks&search_text&region&region_name&lastname /adv/\nClean-param: fl_checkpoint&clckid&begin&yprqee&products&yadclid&yadordid&direct&case_products&wvstest&modal&suggest_reqid&only-content&tb&source&p&m-message-key-id&journal&date&months[]&answer_email&url&intermedia&industry&service&country&K1[K2]&lr&budgetOrder&certificate&utm_source&utm_campaign&utm_medium&utm_term&utm_content /adv/\nClean-param: z&ll&mapTypeRoute&slug&uuid&app&is_user_location&app_with_layers&app_soyuz&from&dateRoute&clid&via&city&amp&topic&lat&lon&lang&le_Lightning&from&utm_source&utm_campaign&utm_medium&utm_content&utm_term /pogoda/\nClean-param: z&ll&mapTypeRoute&slug&uuid&app&is_user_location&app_with_layers&app_soyuz&from&dateRoute&clid&via&city&amp&topic&lat&lon&lang&le_Lightning&from&utm_source&utm_campaign&utm_medium&utm_content&utm_term /hava/\nClean-param: z&ll&mapTypeRoute&slug&uuid&app&is_user_location&app_with_layers&app_soyuz&from&dateRoute&clid&via&city&amp&topic&lat&lon&lang&le_Lightning&from&utm_source&utm_campaign&utm_medium&utm_content&utm_term /weather/\nClean-param: nw&from_block&mode&mar&photoType&regid /pogoda/\nClean-param: nw&from_block&mode&mar&photoType&regid /hava/\nClean-param: nw&from_block&mode&mar&photoType&regid /weather/\nClean-param: from_pumpkin&le_WindParticles&nowcast&yw_preset&gen_time&for_date&morda_geolocation&retpath&acc_maps&le_TemperatureBalloons&request&le_Borders&elmt&may&source&ids&june&july&referrer&PAGEN_1&from_render_fallback&preset&to&august&appsearch_ys&region&tm&banner-test-tags /pogoda/\nClean-param: stat-id&format-type&actual-format&april&test-tag&adgroup&campaign_id&c&nowcast_header&detailsType&utm_from&bar_exp&march&november&bounds&game&text&city_id&february&september&january&no_turbo&october&ref&december&weather_url&build&lp_source&tab&geoid&cameras&day&thndr_onbrd /pogoda/\nClean-param: from_pumpkin&le_WindParticles&nowcast&yw_preset&gen_time&for_date&morda_geolocation&retpath&acc_maps&le_TemperatureBalloons&request&le_Borders&elmt&may&source&ids&june&july&referrer&PAGEN_1&from_render_fallback&preset&to&august&appsearch_ys&region&tm&banner-test-tags&stat-id /hava/\nClean-param: format-type&actual-format&april&test-tag&adgroup&campaign_id&c&nowcast_header&detailsType&utm_from&bar_exp&march&november&bounds&game&text&city_id&february&september&january&no_turbo&october&ref&december&weather_url&build&lp_source&tab&geoid&cameras&day&thndr_onbrd /hava/\nClean-param: from_pumpkin&le_WindParticles&nowcast&yw_preset&gen_time&for_date&morda_geolocation&retpath&acc_maps&le_TemperatureBalloons&request&le_Borders&elmt&may&source&ids&june&july&referrer&PAGEN_1&from_render_fallback&preset&to&august&appsearch_ys&region&tm&banner-test-tags&stat-id /weather/\nClean-param: format-type&actual-format&april&test-tag&adgroup&campaign_id&c&nowcast_header&detailsType&utm_from&bar_exp&march&november&bounds&game&text&city_id&february&september&january&no_turbo&october&ref&december&weather_url&build&lp_source&tab&geoid&cameras&day&thndr_onbrd /weather/\nClean-param: utm_source&utm_medium&utm_term&utm_content&utm_campaign&yclid&referrer&ref&from&adj_campaign&win&lp_source&openstat /alice/\n","agents":{"GPTBot":true,"OAI-SearchBot":true,"ChatGPT-User":true,"ClaudeBot":true,"Claude-SearchBot":true,"PerplexityBot":true,"Google-Extended":true,"Applebot-Extended":true,"CCBot":true,"Bytespider":true,"Meta-ExternalAgent":true,"Amazonbot":true}},"sitemap":{"status":200,"content_type":"text/xml","bytes":13477,"sha256":"f3b052a7fadac11365e892fc42211ddd3053fa4edc9e90ea49f24521e2eed41e","final_host":"yandex.ru","final_path":"/support/sitemap.xml","ms":67,"url":"https://yandex.ru/support/sitemap.xml","declared":true,"xml":true},"files":{"llms_txt":{"status":404,"content_type":"text/plain","bytes":9,"sha256":"e3ebaa16dd9d9b9fc107c42183fb6cf9d22927e1af03dbbdfa0ccc38e4e4ac31","final_host":"yandex.ru","final_path":"/llms.txt","ms":58,"path":"/llms.txt","html":false,"json":null,"present":false},"agents_md":{"status":404,"content_type":"text/plain","bytes":9,"sha256":"e3ebaa16dd9d9b9fc107c42183fb6cf9d22927e1af03dbbdfa0ccc38e4e4ac31","final_host":"yandex.ru","final_path":"/agents.md","ms":54,"path":"/agents.md","html":false,"json":null,"present":false},"agent_skills":{"status":404,"content_type":null,"bytes":9,"sha256":"0019dfc4b32d63c1392aa264aed2253c1e0c2fb09216f8e2cc269bbfb8bb49b5","final_host":"yandex.ru","final_path":"/.well-known/agent-skills/index.json","ms":59,"path":"/.well-known/agent-skills/index.json","html":false,"json":null,"present":false},"mcp_card":{"status":404,"content_type":null,"bytes":9,"sha256":"0019dfc4b32d63c1392aa264aed2253c1e0c2fb09216f8e2cc269bbfb8bb49b5","final_host":"yandex.ru","final_path":"/.well-known/mcp/server-card.json","ms":79,"path":"/.well-known/mcp/server-card.json","html":false,"json":null,"present":false},"api_catalog":{"status":404,"content_type":null,"bytes":9,"sha256":"0019dfc4b32d63c1392aa264aed2253c1e0c2fb09216f8e2cc269bbfb8bb49b5","final_host":"yandex.ru","final_path":"/.well-known/api-catalog","ms":68,"path":"/.well-known/api-catalog","html":false,"json":null,"present":false},"a2a_card":{"status":404,"content_type":null,"bytes":9,"sha256":"0019dfc4b32d63c1392aa264aed2253c1e0c2fb09216f8e2cc269bbfb8bb49b5","final_host":"yandex.ru","final_path":"/.well-known/agent-card.json","ms":161,"path":"/.well-known/agent-card.json","html":false,"json":null,"present":false}},"rings":{"v":1,"candidates":[],"mcp":[],"a2a":null,"webmcp":{"model_context":false,"bridge":false,"mcp_json":false,"link_rel":false,"any":false},"handshakes":0,"get_path":{"status":404,"content_type":"application/json","ms":172,"error":null},"get_host":{"status":404,"content_type":"application/json","ms":197,"error":null}},"sf":[4,0,0,0,0,0,0,0,0],"derived":{"reachable":true,"home_class":"answers","robots_served":true,"robots_sitemap":true,"content_signal":false,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":[],"blocks_any_ai":false,"files":{"llms_txt":false,"agents_md":false,"agent_skills":false,"mcp_card":false,"api_catalog":false,"a2a_card":false},"any_capability":false,"ai_refused":[],"ai_thin":[],"aipref":0,"aipref_measured":true,"sf":[4,0,0,0,0,0,0,0,0],"rings":{"mcp_declared":false,"mcp_live":false,"mcp_auth":false,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":3,"fh":"sso.passport.yandex.ru","fp_other_host":[]},"derive_v":5,"frame":"top10k","shard":2},{"v":1,"kind":"crawlcheck-census-record","run":18,"domain":"wordpress.com","rank":100,"measured_at":"2026-11-13T18:51:23.228Z","home":{"browser":{"status":403,"content_type":"text/html","bytes":1762,"sha256":"fc89e2a14f48c43f7c85082a9360f737c7b13e2180a76370e7ae7a4ed6e3355f","final_host":"wordpress.com","final_path":"/","ms":23,"identity":"Mobile browser","html":true},"gptbot":{"status":403,"content_type":"text/html","bytes":1780,"sha256":"9288554aae934d49582257310e647b1da6118a37e253c9944e515c62d0cb8870","final_host":"wordpress.com","final_path":"/","ms":11,"identity":"GPTBot","html":true},"claudebot":{"status":403,"content_type":"text/html","bytes":1780,"sha256":"9288554aae934d49582257310e647b1da6118a37e253c9944e515c62d0cb8870","final_host":"wordpress.com","final_path":"/","ms":6,"identity":"ClaudeBot","html":true},"oaisearch":{"status":403,"content_type":"text/html","bytes":1780,"sha256":"9288554aae934d49582257310e647b1da6118a37e253c9944e515c62d0cb8870","final_host":"wordpress.com","final_path":"/","ms":6,"identity":"OAI-SearchBot","html":true},"perplexity":{"status":403,"content_type":"text/html","bytes":1780,"sha256":"9288554aae934d49582257310e647b1da6118a37e253c9944e515c62d0cb8870","final_host":"wordpress.com","final_path":"/","ms":5,"identity":"PerplexityBot","html":true}},"robots":{"status":200,"content_type":"text/plain","bytes":1387,"sha256":"7114fbf0e10e6344c975106bfe1a50382943f4999906633297578a67a0f8dba7","final_host":"wordpress.com","final_path":"/robots.txt","ms":48,"html":false,"text":"# If you are regularly crawling WordPress.com sites, please use our firehose to receive real-time push updates instead.\n# Please see https://developer.wordpress.com/docs/firehose/ for more details.\n\nSitemap: https://wordpress.com/sitemap.xml\nSitemap: https://wordpress.com/news-sitemap.xml\n\nSitemap: https://wordpress.com/go/sitemap.xml\nSitemap: https://wordpress.com/blog/sitemap.xml\nSitemap: https://wordpress.com/support/sitemap.xml\n\nUser-agent: *\nDisallow: /wp-admin/\nAllow: /wp-admin/admin-ajax.php\nDisallow: /typo/?subdomain=\nDisallow: /read/\nDisallow: /*/read/\nAllow: /forums/topic-tag/reader/\nAllow: /*/forums/topic-tag/reader/\nAllow: /support/reader/\nAllow: /*/support/reader/\nDisallow: /reader/\nDisallow: /*/reader/\nAllow: /log-in/\nAllow: /log-in$\nDisallow: /log-in?\nDisallow: /abuse/?*\nDisallow: /abuse?*\nDisallow: /plugins/?s=\nDisallow: /*/plugins/?s=\nDisallow: /*?aff=\nDisallow: /*&aff=\nDisallow: /*?affiliate=\nDisallow: /*&affiliate=\nDisallow: /*?cid=\nDisallow: /*&cid=\nDisallow: /*?irclickid=\nDisallow: /*&irclickid=\nDisallow: /*/?like_comment=\nDisallow: /*?retry=\nDisallow: /*?sid=\nDisallow: /*?action=\nDisallow: /wp-login.php\nDisallow: /wp-signup.php\nDisallow: /press-this.php\nDisallow: /remote-login.php\nDisallow: /activate/\nDisallow: /cgi-bin/\nDisallow: /mshots/v1/\nDisallow: /next/\nDisallow: /public.api/\n\n# This file was generated on Fri, 13 Nov 2026 13:49:09 +0000\n","agents":{"GPTBot":true,"OAI-SearchBot":true,"ChatGPT-User":true,"ClaudeBot":true,"Claude-SearchBot":true,"PerplexityBot":true,"Google-Extended":true,"Applebot-Extended":true,"CCBot":true,"Bytespider":true,"Meta-ExternalAgent":true,"Amazonbot":true}},"sitemap":null,"files":null,"derived":{"reachable":false,"home_class":"refused_us","robots_served":false,"robots_sitemap":false,"content_signal":false,"cs_ai_train_no":false,"cs_ai_input_no":false,"blocks":[],"blocks_any_ai":false,"files":{"llms_txt":false,"agents_md":false,"agent_skills":false,"mcp_card":false,"api_catalog":false,"a2a_card":false},"any_capability":false,"ai_refused":[],"ai_thin":[],"rings":{"mcp_declared":false,"mcp_live":false,"mcp_auth":false,"a2a_declared":false,"a2a_answers":false,"webmcp":false},"fp_v":3,"fh":"wordpress.com","fp_other_host":[]},"derive_v":5,"frame":"top10k","shard":3}]}