user_agent禁止某些用户访问，干掉一些不知名的爬虫

通过分析日志来得出频繁的user_agent
tail -n 1000 /www/wdlinux/nginx/logs/access.log | awk -F\” ‘{A[$(NF-1)]++}END{for(k in A)print A[k],k}’ | sort -n | tail
#禁止Scrapy等工具的抓取
if ($http_user_agent ~* (Scrapy|Curl|HttpClient|Googlebot)) { return 403;}
#禁止指定UA及UA为空的访问
if ($http_user_agent ~ “FeedDemon|JikeSpider|Indy Library|Alexa Toolbar|AskTbFXTV|AhrefsBot|CrawlDaddy|CoolpadWebkit|Java|Feedly|UniversalFeedParser|ApacheBench|Microsoft URL Control|Swiftbot|ZmEu|oBot|jaunty|Python-urllib|lightDeckReports Bot|YYSpider|DigExt|YisouSpider|HttpClient|MJ12bot|heritrix|EasouSpider|LinkpadBot|Ezooms|^$” ){ return 403;}
#禁止非GET|HEAD|POST方式的抓取
if ($request_method !~ ^(GET|HEAD|POST)$) { return 403;}
还有加一些针对特殊的user_agent的访问
if ($http_user_agent ~ “Mozilla/4.0/ /(compatible;/ MSIE/ 6.0;/ Windows/ NT/ 5.1;/ SV1;/ .NET/ CLR/ 1.1.4322;/ .NET/ CLR/ 2.0.50727/)”) { return 404; }
nginx禁止空agent的浏览器访问
if ($http_user_agent ~ ^$) {
return 403;
rewrite ^  google.com  permanent;
}