# robots.txt for ahrefs.com — corrected by CrawlCheck, 2026-09-03
# Source: the file the site served at the time. Every change is listed here; nothing else was touched.
# Changes:
#   - Added a Sitemap line: https://ahrefs.com/sitemap.xml answered 200 with XML but the file never named it.
# Verify after publishing: curl -sI https://ahrefs.com/robots.txt must show text/plain, on the bare URL, with no cache-buster.

User-agent: *
Disallow: /article
Disallow: /site-explorer/ajax/
Allow: /site-explorer/$
Disallow: /site-explorer/*
Allow: /link-intersect/$
Disallow: /link-intersect/*
Disallow: /v4*
Disallow: /blog/*?s=*
Disallow: /blog/*?archive*
Disallow: /seo/for/*?*draft
Disallow: /academy/*?*draft
Disallow: /seo-toolbar/welcome
Disallow: /seo-toolbar/uninstall
Disallow: /*/seo-toolbar/welcome
Disallow: /*/seo-toolbar/uninstall
Disallow: /*?input
Disallow: /draft/*
Disallow: /academy/draft/*
Allow: /agencies/*?services[]=*
Allow: /agencies/*&services[]=*
Disallow: /agencies/*?*languages[]=*
Disallow: /agencies/*&*languages[]=*
Disallow: /agencies/*?*industries[]=*
Disallow: /agencies/*&*industries[]=*
Disallow: /agencies/*?*budget=*
Disallow: /agencies/*&*budget=*
Disallow: /agencies/*?*businessSize=*
Disallow: /agencies/*&*businessSize=*
Disallow: /cdn-cgi/
Disallow: /writing-tools/experimental-tools.json

Sitemap: https://ahrefs.com/sitemap.xml