# robots.txt for modal.com — corrected by CrawlCheck, 2026-11-03
# Source: the file the site served at the time. Every change is listed here; nothing else was touched.
# Changes:
#   - 5 named groups lacked Disallow lines the * group carries. A crawler obeys only its own most specific group and ignores *, so those rules never applied to it. Copied in.
# Verify after publishing: curl -sI https://modal.com/robots.txt must show text/plain, on the bare URL, with no cache-buster.

# Private surfaces also send `X-Robots-Tag: noindex, nofollow`; the path list
# below is checked against src/lib/core/crawlers.ts by crawlers.test.ts.
#
# A Disallow only stops a fetch — it does not remove a URL a search engine
# already stores, because the noindex directive is never re-read. So the search
# engines whose indexes we want purged keep crawl access, and every other
# crawler is blocked from those paths outright.

# TODO(gleb-modal): delete this group on 2026-09-15, once the stale dashboard
# URLs have dropped out of Google and Bing.
User-agent: Googlebot
User-agent: Bingbot
Allow: /
Allow: /api/user
Disallow: /api/
Disallow: /account-migration
Disallow: /account-pending
Disallow: /apps
Disallow: /authorize
Disallow: /clusters
Disallow: /containers
Disallow: /dashboard
Disallow: /dd
Disallow: /endpoints
Disallow: /evaluations
Disallow: /home
Disallow: /id
Disallow: /images
Disallow: /internal
Disallow: /login
Disallow: /logs
Disallow: /notebooks
Disallow: /oauth
Disallow: /ph
Disallow: /runners
Disallow: /sandboxes
Disallow: /secrets
Disallow: /settings
Disallow: /signup
Disallow: /slack-link-workspace
Disallow: /storage
Disallow: /token-flow
Disallow: /v2

User-agent: *
Allow: /api/user
Disallow: /api/
Disallow: /account-migration
Disallow: /account-pending
Disallow: /apps
Disallow: /authorize
Disallow: /clusters
Disallow: /containers
Disallow: /dashboard
Disallow: /dd
Disallow: /endpoints
Disallow: /evaluations
Disallow: /home
Disallow: /id
Disallow: /images
Disallow: /internal
Disallow: /login
Disallow: /logs
Disallow: /notebooks
Disallow: /oauth
Disallow: /ph
Disallow: /runners
Disallow: /sandboxes
Disallow: /secrets
Disallow: /settings
Disallow: /signup
Disallow: /slack-link-workspace
Disallow: /storage
Disallow: /token-flow
Disallow: /v2

User-agent: PetalBot
Disallow: /
Disallow: /api/
Disallow: /account-migration
Disallow: /account-pending
Disallow: /apps
Disallow: /authorize
Disallow: /clusters
Disallow: /containers
Disallow: /dashboard
Disallow: /dd
Disallow: /endpoints
Disallow: /evaluations
Disallow: /home
Disallow: /id
Disallow: /images
Disallow: /internal
Disallow: /login
Disallow: /logs
Disallow: /notebooks
Disallow: /oauth
Disallow: /ph
Disallow: /runners
Disallow: /sandboxes
Disallow: /secrets
Disallow: /settings
Disallow: /signup
Disallow: /slack-link-workspace
Disallow: /storage
Disallow: /token-flow
Disallow: /v2

User-agent: BLEXBot
Disallow: /
Disallow: /api/
Disallow: /account-migration
Disallow: /account-pending
Disallow: /apps
Disallow: /authorize
Disallow: /clusters
Disallow: /containers
Disallow: /dashboard
Disallow: /dd
Disallow: /endpoints
Disallow: /evaluations
Disallow: /home
Disallow: /id
Disallow: /images
Disallow: /internal
Disallow: /login
Disallow: /logs
Disallow: /notebooks
Disallow: /oauth
Disallow: /ph
Disallow: /runners
Disallow: /sandboxes
Disallow: /secrets
Disallow: /settings
Disallow: /signup
Disallow: /slack-link-workspace
Disallow: /storage
Disallow: /token-flow
Disallow: /v2

User-agent: Bytespider
Disallow: /
Disallow: /api/
Disallow: /account-migration
Disallow: /account-pending
Disallow: /apps
Disallow: /authorize
Disallow: /clusters
Disallow: /containers
Disallow: /dashboard
Disallow: /dd
Disallow: /endpoints
Disallow: /evaluations
Disallow: /home
Disallow: /id
Disallow: /images
Disallow: /internal
Disallow: /login
Disallow: /logs
Disallow: /notebooks
Disallow: /oauth
Disallow: /ph
Disallow: /runners
Disallow: /sandboxes
Disallow: /secrets
Disallow: /settings
Disallow: /signup
Disallow: /slack-link-workspace
Disallow: /storage
Disallow: /token-flow
Disallow: /v2

User-agent: Barkrowler
Disallow: /
Disallow: /api/
Disallow: /account-migration
Disallow: /account-pending
Disallow: /apps
Disallow: /authorize
Disallow: /clusters
Disallow: /containers
Disallow: /dashboard
Disallow: /dd
Disallow: /endpoints
Disallow: /evaluations
Disallow: /home
Disallow: /id
Disallow: /images
Disallow: /internal
Disallow: /login
Disallow: /logs
Disallow: /notebooks
Disallow: /oauth
Disallow: /ph
Disallow: /runners
Disallow: /sandboxes
Disallow: /secrets
Disallow: /settings
Disallow: /signup
Disallow: /slack-link-workspace
Disallow: /storage
Disallow: /token-flow
Disallow: /v2

Sitemap: https://modal.com/sitemap.xml