# Block SEO, advertising and monitoring crawlers — nginx # 27 user agents, generated from https://crawlcheck.io/ai-crawlers # A user agent string can be forged; this stops crawlers that name themselves. # Put the map in the http block and the if in each server block. map $http_user_agent $cc_block { default 0; "~*(KlaviyoAIBot|Mediapartners-Google|APIs-Google|BingPreview|Magpie-crawler|AwarioRssBot|AwarioSmartBot|AhrefsBot|SemrushBot|MJ12bot|DotBot|rogerbot|DataForSeoBot|BLEXBot|VelenPublicWebCrawler|Google-CloudVertexBot|CloudflareBrowserRenderingCrawler|Cloudflare-AutoRAG|Cloudflare-AI-Search-External|Peer39_Crawler|AdsBot-Google|AmazonAdBot|AdIdxBot|Scrapy|SemrushBot-OCOB|AwarioBot|OAI-AdsBot)" 1; } # server { # if ($cc_block) { return 403; } # }