Man kann natürlich auch mittel USER_AGENT checken um lästige Dinge loszuwerden, hier mal so als Auszug. Natürlich muss jeder selber entscheiden, was er zulassen möchte und was nicht:
CODE # spambots
RewriteCond %{HTTP_USER_AGENT} compatible\;\ MSIE\ 6.0 [OR]
RewriteCond %{HTTP_USER_AGENT} DTS\ Agent [OR]
RewriteCond %{HTTP_REFERER} iaea\.org [OR]
#RewriteCond %{HTTP_USER_AGENT} ^IE\ \d\.\d\ Compatible.*Browser$ [OR]
RewriteCond %{HTTP_USER_AGENT} Microsoft\ URL\ Control [OR]
RewriteCond %{HTTP_USER_AGENT} NG/2.0 [OR]
RewriteCond %{HTTP_USER_AGENT} RPT-HTTPClient [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*\ Charlotte.*$ [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^.*\ Daumoa.*$ [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^.*\ Exabot.*$ [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^.*ACONTBOT.*$ [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^.*adressendeutschland.*$ [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^.*askpeter\_bot.*$ [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^.*askpeter\_jeanie.*$ [OR,NC]
# image grabber
RewriteCond %{HTTP_USER_AGENT} ^.*Exabot\-Thumbnail.*$ [OR,NC]
# website grabber
RewriteCond %{HTTP_USER_AGENT} ^.*HTTrack.*$ [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^.*IDBot.*$ [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^.*iCcrawler\ \-\ iCjobs.*$ [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^.*jobs\.de\-Robot.*$ [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^.*MJ12bot.*$ [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^.*MooseBot.*$ [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^.*NaverBot.*$ [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^.*OpenX\ Spider.*$ [OR,NC]
# image grabber
RewriteCond %{HTTP_USER_AGENT} ^.*PicSpider.*$ [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^.*proximic.*$ [OR,NC]
# image grabber
RewriteCond %{HTTP_USER_AGENT} ^.*sexsearcher.*$ [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^.*TopBlogsInfo.*$ [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^.*VoilaBot.*$ [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^Alexibot [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^ArtfaceBot [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^asterias [OR,NC]
# exploit searcher
RewriteCond %{HTTP_USER_AGENT} ^BackDoorBot [OR,NC]
# exploit searcher
RewriteCond %{HTTP_USER_AGENT} ^Black.Hole [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^BlackWidow [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^Baiduspider [OR,NC]
# website grabber
RewriteCond %{HTTP_USER_AGENT} ^BlowFish [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^boitho\.com\-dc [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^BotALot [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^BotOnParade [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^Bot\ mailto:craftbot@yahoo\.com [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^BuiltBotTough [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^Bullseye [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^BunnySlippers [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^Cegbfeieh [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^CheeseBot [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^CherryPicker [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^ChinaClaw [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^Cityreview [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^CMUImageBot/spider\.pl [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^Convera [OR,NC]
# Copyright checker
RewriteCond %{HTTP_USER_AGENT} ^CopyRightCheck [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^cosmos [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^Crescent [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^Custo [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^DISCo [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^DittoSpyder [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^Download\ Demon [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^eCatch [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^EirGrabber [OR,NC]
# email crawler
RewriteCond %{HTTP_USER_AGENT} ^EmailCollector.*$ [OR,NC]
# email crawler
RewriteCond %{HTTP_USER_AGENT} ^EmailSearch.*$ [OR,NC]
# email crawler
RewriteCond %{HTTP_USER_AGENT} ^EmailSiphon.*$ [OR,NC]
# email crawler
RewriteCond %{HTTP_USER_AGENT} ^EmailWolf.*$ [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^EmeraldShield [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^envolk [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^EroCrawler [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^Eurobot [OR,NC]
# image grabber
RewriteCond %{HTTP_USER_AGENT} ^Express\ WebPictures [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^ExtractorPro [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^EyeNetIE [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^Faxobot [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^FlashGet [OR,NC]
# link grabber
RewriteCond %{HTTP_USER_AGENT} ^findlinks.*wortschatz\.uni-leipzig\.de [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^Foobot [OR,NC]
# website grabber
RewriteCond %{HTTP_USER_AGENT} ^GetRight [OR,NC]
# website grabber
RewriteCond %{HTTP_USER_AGENT} ^GetWeb! [OR,NC]
# website grabber
RewriteCond %{HTTP_USER_AGENT} ^Go!Zilla [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^Go-Ahead-Got-It [OR,NC]
# RewriteCond %{HTTP_USER_AGENT} ^Googlebot-Image [OR]
RewriteCond %{HTTP_USER_AGENT} ^GornKer [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^GrabNet [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^Grafula [OR,NC]
# Perl Module
RewriteCond %{HTTP_USER_AGENT} ^GT\:\:WWW [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^Harvest [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^hloader [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^HMView [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^holmes [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^httplib [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^humanlinks [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^iearthworm [OR,NC]
# image grabber
RewriteCond %{HTTP_USER_AGENT} ^Image\ Stripper [OR,NC]
# image grabber
RewriteCond %{HTTP_USER_AGENT} ^Image\ Sucker [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^Indy\ Library [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^InfoNaviRobot [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^Infosearch [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^InnerpriseBot [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^InterGET [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^Internet\ Ninja [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^IRLbot [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^Irvine [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^Isidorus [OR,NC]
# Apache Jakarta std bot
RewriteCond %{HTTP_USER_AGENT} ^Jakarta\ Commons\-HttpClient [OR,NC]
# Java std bot
RewriteCond %{HTTP_USER_AGENT} ^Java/1\.6 [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^JennyBot [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^JetCar [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^JOC\ Web\ Spider [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^Kenjin.Spider [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^Keyword.Density [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^Knowledge.com [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^larbin [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^LeechFTP [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^LexiBot [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^libWeb/clsHTTP [OR,NC]
# link grabber
RewriteCond %{HTTP_USER_AGENT} ^LinkextractorPro [OR,NC]
# link grabber
RewriteCond %{HTTP_USER_AGENT} ^LinkScan/8.1a.Unix [OR,NC]
# link grabber
RewriteCond %{HTTP_USER_AGENT} ^LinkWalker [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^lwp-trivial [OR,NC]
# vermutlich Spambot (Forum/Blogs/Co)
RewriteCond %{HTTP_USER_AGENT} ^Mail\.Ru [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^Mata.Hari [OR,NC]
# oesterreichische softwarefirma
RewriteCond %{HTTP_USER_AGENT} ^McBot [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^Microsoft [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^Microsoft.URL [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^MIDown\ tool [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^MIIxpc [OR,NC]
# image grabber
RewriteCond %{HTTP_USER_AGENT} ^Mister.PiX [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^Mnogosearch [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^moget [OR,NC]
# spybot
RewriteCond %{HTTP_USER_AGENT} ^Mozilla.*\ obot [OR]
RewriteCond %{HTTP_USER_AGENT} ^NaverBot [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^Navroad [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^NearSite [OR,NC]
# website grabber
RewriteCond %{HTTP_USER_AGENT} ^Net\ Vampire [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^NetAnts [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^NetMechanic [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^NetSpider [OR,NC]
# website grabber
RewriteCond %{HTTP_USER_AGENT} ^NetZIP [OR,NC]
# ein Arcor-Bot
RewriteCond %{HTTP_USER_AGENT} ^NG-Search [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^NICErsPRO [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^NPBot [OR,NC]
# spybot
RewriteCond %{HTTP_USER_AGENT} ^oBot [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^Ocelli [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^Octopus [OR,NC]
# website grabber
RewriteCond %{HTTP_USER_AGENT} ^Offline\ Explorer [OR,NC]
# website grabber
RewriteCond %{HTTP_USER_AGENT} ^Offline\ Navigator [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^OmniExplorer [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^Openfind [OR,NC]
# website grabber
RewriteCond %{HTTP_USER_AGENT} ^PageGrabber [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^panscient\.com [OR,NC]
# image grabber
RewriteCond %{HTTP_USER_AGENT} ^Papa\ Foto [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^pavuk [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^pcBrowser [OR,NC]
# favicon-browser
RewriteCond %{HTTP_USER_AGENT} ^PEERbot [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^picscout [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^PingALink [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^PrivacyFinder [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^ProPowerBot [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^ProWebWalker [OR,NC]
# image grabber
RewriteCond %{HTTP_USER_AGENT} ^psbot [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^QueryN.Metasearch [OR,NC]
# keine robotstxt
RewriteCond %{HTTP_USER_AGENT} ^RedKernel [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^ReGet [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^RepoMonkey [OR,NC]
# spiders
RewriteCond %{HTTP_USER_AGENT} ^RexyoBot [OR,NC]
# spambot
RewriteCond %{HTTP_USER_AGENT} ^RIN\.\ Web\ crawler [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^RMA [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^Seekbot [OR,NC]
# favicon-browser
RewriteCond %{HTTP_USER_AGENT} ^Shit [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^ShopWiki [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^Siphon [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^SiteSnagger [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^SlySearch [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^SmartDownload [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^Speedy\ Spider [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^SpankBot [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^spanner [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^Sosospider [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^SSM [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^SuperBot [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^SuperHTTP [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^Surfbot [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^SurveyBot [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^suzuran [OR,NC]
# russische Suchmaschine
RewriteCond %{HTTP_USER_AGENT} ^StackRambler [OR,NC]
# statcrawler@gmail.com
RewriteCond %{HTTP_USER_AGENT} ^stat [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^tAkeOut [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^Tasapspider [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^Teleport [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^Telesoft [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^The.Intraformant [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^TheNomad [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^TightTwatBot [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^Titan [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^toCrawl/UrlDispatcher [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^Touche [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^True_Robot [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^turingos [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^URLy.Warning [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^VCI [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^VoidEYE [OR,NC]
# image grabber
RewriteCond %{HTTP_USER_AGENT} ^Web\ Image\ Collector [OR,NC]
# image grabber
RewriteCond %{HTTP_USER_AGENT} ^Web.Image.Collector [OR,NC]
# image grabber
RewriteCond %{HTTP_USER_AGENT} ^Web\ Sucker [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^WebAuto [OR,NC]
# website grabber
RewriteCond %{HTTP_USER_AGENT} ^WebBandit [OR,NC]
# website grabber
RewriteCond %{HTTP_USER_AGENT} ^WebCopier [OR,NC]
# email crawler
RewriteCond %{HTTP_USER_AGENT} ^WebEMailExtrac.*$ [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^WebEnhancer [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^WebFetch [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^WebGo\ IS [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^WebLeacher [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^WebmasterWorldForumBot [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^WebReaper [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^WebSauger [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^Website\ eXtractor [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^Website.Quester [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^Webster.Pro [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^WebStripper [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^WebWhacker [OR,NC]
# website grabber
RewriteCond %{HTTP_USER_AGENT} ^WebZip [OR,NC]
# image grabber
RewriteCond %{HTTP_USER_AGENT} ^Web\ Image\ Collector [OR,NC]
# image grabber
RewriteCond %{HTTP_USER_AGENT} ^Web.Image.Collector [OR,NC]
# website grabber
RewriteCond %{HTTP_USER_AGENT} ^Wget [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^Widow [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^WikioFeedBot [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^WikioPxyFeedBot [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^WiseWire.*$ [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^WorQmada [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^WWW-Collector-E [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^WWWOFFLE [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^Xaldon\ WebSpider [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^Yandex [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^Yanga\ WorldSearch\ Bot [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^Yeti [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^Xenu's [OR,NC]
# Car Search Engine
RewriteCond %{HTTP_USER_AGENT} ^Yotta [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^YoudaoBot [OR,NC]
RewriteCond %{HTTP_USER_AGENT} ^Zeus
RewriteRule .* - [F,L]
Wie einigen Betreibern bekannt sein dürfte, ist der Anbieter "Semalt" ziemlich hartnäckig. Mir wird immer wieder die Frage gestellt, was da passiert, warum das passiert und wie man das unterbinden kann. Da dachte ich mir: Mach ich mal nen Thread dazu und poste einen Schnipsel...
