# Block SEO, advertising and monitoring crawlers — cloudflare # 27 user agents, generated from https://crawlcheck.io/ai-crawlers # A user agent string can be forged; this stops crawlers that name themselves. # Security > WAF > Custom rules: paste as the expression, action Block. # Redirect rules run before custom rules: on a zone that redirects every request this never fires, so put it on the zone that serves the pages. (lower(http.user_agent) contains "klaviyoaibot") or (lower(http.user_agent) contains "mediapartners-google") or (lower(http.user_agent) contains "apis-google") or (lower(http.user_agent) contains "bingpreview") or (lower(http.user_agent) contains "magpie-crawler") or (lower(http.user_agent) contains "awariorssbot") or (lower(http.user_agent) contains "awariosmartbot") or (lower(http.user_agent) contains "ahrefsbot") or (lower(http.user_agent) contains "semrushbot") or (lower(http.user_agent) contains "mj12bot") or (lower(http.user_agent) contains "dotbot") or (lower(http.user_agent) contains "rogerbot") or (lower(http.user_agent) contains "dataforseobot") or (lower(http.user_agent) contains "blexbot") or (lower(http.user_agent) contains "velenpublicwebcrawler") or (lower(http.user_agent) contains "google-cloudvertexbot") or (lower(http.user_agent) contains "cloudflarebrowserrenderingcrawler") or (lower(http.user_agent) contains "cloudflare-autorag") or (lower(http.user_agent) contains "cloudflare-ai-search-external") or (lower(http.user_agent) contains "peer39_crawler") or (lower(http.user_agent) contains "adsbot-google") or (lower(http.user_agent) contains "amazonadbot") or (lower(http.user_agent) contains "adidxbot") or (lower(http.user_agent) contains "scrapy") or (lower(http.user_agent) contains "semrushbot-ocob") or (lower(http.user_agent) contains "awariobot") or (lower(http.user_agent) contains "oai-adsbot")