User-agent: NetcraftSurveyAgent
Disallow: /
User-agent: SputnikBot
Disallow: /
User-agent: cis455crawler
Disallow: /
User-agent: ia_archiver
Disallow: /
User-agent: Scrapy
Disallow: /
User-agent: MojeekBot
Disallow: /
User-agent: CCBot
Disallow: /
User-agent: Riddler
Disallow: /
User-agent: CrazyWebCrawler-Spider
Disallow: /
User-agent: spbot
Disallow: /
User-agent: dotbot
Disallow: /
User-agent: BUbiNG
Disallow: /
User-agent: Uptimebot
Disallow: /
User-agent: GnowitNewsbot
Disallow: /
User-agent: linkdexbot
Disallow: /
User-agent: YamanaLab-Robot
Disallow: /
User-agent: yacybot
Disallow: /
User-agent: MJ12bot
Disallow: /
User-agent: SemrushBot
Disallow: /
User-agent: AhrefsBot
Disallow: /
User-agent: MegaIndex.ru
Disallow: /
User-agent: megaindex.com
Disallow: /
User-agent: dcrawl
Disallow: /
User-agent: SBIder
Disallow: /
User-agent: OmniExplorer_Bot
Disallow: /
User-agent: psbot
Disallow: /
User-agent: Yahoo-MMCrawler
Disallow: /
User-agent: lmspider
Disallow: /
User-agent: lmspider [email protected]
Disallow: /
User-agent: lmspider ([email protected])
Disallow: /
User-agent: IRLbot
Disallow: /
User-agent: NPBot (http://www.nameprotect.com/botinfo.html)
Disallow: /
User-agent: NPBot
Disallow: /
User-agent: npbot
Disallow: /
User-agent: TurnitinBot
Disallow: /
User-agent: DeadLinkCheck/0.4.0
Disallow: /
User-agent: DeadLinkCheck
Disallow: /
User-agent: LinkWalker
Disallow: /
User-agent: SafeDNSBot
Disallow: /
User-agent: Barkrowler
Disallow: /
User-agent: *
Disallow: /profpic
User-agent: *
Disallow: /mysearches
User-agent: *
Disallow: /users/addreview
User-agent: *
Disallow: /imageman
User-agent: *
Disallow: /cgi-bin/
Disallow: /images/
User-agent: *
Disallow: /addurl
User-agent: *
Disallow: /users/addreview
User-agent: *
Disallow: /feedback
User-agent: *
Disallow: /queuelookup
User-agent: *
Disallow: /queuestatus
User-agent: becomebot
Disallow: /
User-agent: Wget
Disallow: /
User-agent: wget
Disallow: /
User-agent: Wget/1.5.3
Disallow: /
User-agent: Wget/1.6
Disallow: /
User-agent: http://www.almaden.ibm.com/cs/crawler
Disallow: /
User-agent: *
Disallow: /searchperson
User-agent: *
Disallow: /resetpassword
User-agent: *
Disallow: /login