# robots.txt for serpstat.com — corrected by CrawlCheck, 2026-11-04
# Source: the file the site served at the time. Every change is listed here; nothing else was touched.
# Changes:
#   - 4 named groups lacked Disallow lines the * group carries. A crawler obeys only its own most specific group and ignores *, so those rules never applied to it. Copied in.
# Verify after publishing: curl -sI https://serpstat.com/robots.txt must show text/plain, on the bare URL, with no cache-buster.

User-Agent: *
Disallow: /rate/
Disallow: /files/
Disallow: /blocked/
Allow: /files/img/
Allow: /files/*.css
Allow: /files/*.js
Allow: /llms.txt
Disallow: /node/*?origin=
Disallow: */users/isAccountNoVerified/
Sitemap: https://serpstat.com/sitemap.xml
Sitemap: https://serpstat.com/ru/sitemap.xml
Sitemap: https://serpstat.com/de/sitemap.xml
Sitemap: https://serpstat.com/uk/sitemap.xml
Clean-param: ref&origin&search&user&lang
User-agent: AhrefsBot
Disallow: /
Disallow: /rate/
Disallow: /files/
Disallow: /blocked/
Disallow: /node/*?origin=
Disallow: */users/isAccountNoVerified/
User-agent: MJ12bot
Disallow: /
Disallow: /rate/
Disallow: /files/
Disallow: /blocked/
Disallow: /node/*?origin=
Disallow: */users/isAccountNoVerified/
User-agent: rogerbot
Disallow: /
Disallow: /rate/
Disallow: /files/
Disallow: /blocked/
Disallow: /node/*?origin=
Disallow: */users/isAccountNoVerified/
User-agent: MegaIndex.ru/2.0
Disallow: /
Disallow: /rate/
Disallow: /files/
Disallow: /blocked/
Disallow: /node/*?origin=
Disallow: */users/isAccountNoVerified/