# This is for every user agent.
# block Chinese Search Engine Baidu, raj 06/09/2012
User-agent: SemrushBot
Disallow: /
User-agent: BLEXBot
Disallow: /
User-agent:Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.1; .NET CLR 1.1.4322; .NET CLR 2.0.50727; .NET CLR 3.0.04506.30; Media Center PC 6.0; MS-RTC LM 8)
Disallow: /
User-agent:Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.1; SLCC2; .NET CLR 1.1.4322; .NET CLR 2.0.50727; .NET CLR 3.0.04506.648; Media Center PC 6.0)
Disallow: /
User-agent: Baiduspider
Disallow: /
User-agent: Baiduspider/2.0
Disallow: /
User-agent: magpie-crawler
Disallow: /
User-agent: MJ12bot
Disallow: /
User-Agent: panscient.com
Disallow: /
User-agent: Ezooms
Disallow: /
User-agent: Ezooms/1.0
Disallow: /
User-agent: Ezooms/1.0; [email protected]
Disallow: /
User-agent: Hailoobot
Disallow: /
User-agent: Sogou web spider
Disallow: /
User-agent: sogou spider
Disallow: /
User-agent: Sogou web spider/4.0
Disallow: /
User-agent: Sogou web spider/4.0(+http://www.sogou.com/docs/help/webmasters.htm#07)
Disallow: /
User-agent: YandexImages/3.0
Disallow: /
User-agent: YandexBot/3.0
Disallow: /
User-agent: Yeti
Disallow: /
User-agent: ichiro
Disallow: /
user-agent: AhrefsBot
disallow: /
User-agent: *
# Disallow stuff we don't want to have coming up in search engines.
Disallow: /templates
Disallow: /wwwthreads
Disallow: /upb
Disallow: /upbpnp
Disallow: /mail-old
Disallow: /staff
Disallow: /who
Disallow: /phpMyAdmin
Disallow: /my
Disallow: /what
Disallow: /us
Disallow: /raj
Disallow: /pvl
Disallow: /stats
Disallow: /drg/upb
Disallow: /sales/admin
Disallow: /venues/admin
Disallow: /venues/sales
Disallow: /training
Disallow: /lifestyle/admin
Disallow: /basma
Disallow: /mek
Disallow: /btshow/admin
Disallow: /edesk/admin
Disallow: /dyk/admin
Disallow: /lifestyle/admin
Disallow: /sale
Disallow: /virtual_assistants/admin
Disallow: /us
Disallow: /nl
Disallow: /jobs
Disallow: /ddb
Disallow: /duk
Disallow: /fkb
Disallow: /iqps
Disallow: /shefsec
Disallow:/jane
Disallow: /susan
Disallow: /download
Disallow: /adserv
Disallow: /adserv2
Disallow: /clickscounter
Disallow: /crm
Disallow: /poll
Disallow: /ck
Disallow: /dfp
# Kev.
Disallow /pages/uk/edesk/search
Disallow: /pages/uk/edesk/details
Disallow: /pages/uk/templates/search
Disallow: /pages/uk/venues/search
Disallow: /pages/uk/jobdesk/result
Disallow: /pages/us/venues/search
Disallow: /pages/us/venues/details
Disallow: /pages/uk/venues/details
Disallow: /dnet/twitter_timeline_public.php
Disallow: /dnet///twitter_timeline_public.php
Disallow: /dnet////twitter_timeline_public.php
Disallow: /dnet/////twitter_timeline_public.php
Disallow: /dnet//////twitter_timeline_public.php
Disallow: /pages/uk/jobdesk/jobseekers
Disallow: /pages/btu/index
Disallow: /community/forums/showflat.php
Disallow: /dnet//browse_listings.php
Disallow: /dnet///browse_listings.php
Disallow: /dnet////browse_listings.php
Disallow: /dnet/////browse_listings.php
Disallow: /dnet//////browse_listings.php
Disallow: /dnet/profile.php
Disallow: /dnet//profile.php
Disallow: /dnet///profile.php
Disallow: /dnet////profile.php
Disallow: /dnet/////profile.php
Disallow: /dnet//////profile.php
Disallow: /dnet///////profile.php
Disallow: /dnet//link.php
Disallow: /dnet///link.php
Disallow: /dnet////link.php
Disallow: /dnet/////link.php
Disallow: /dnet//////link.php
Disallow: /dnet//browse_links.php
Disallow: /dnet///browse_links.php
Disallow: /dnet////browse_links.php
Disallow: /dnet/////browse_links.php
Disallow: /dnet//////browse_links.php
Disallow: /dnet//browse_vids.php
Disallow: /dnet///browse_vids.php
Disallow: /dnet////browse_vids.php
Disallow: /dnet/////browse_vids.php
Disallow: /dnet//////browse_vids.php
Disallow: /dnet//vid.php
Disallow: /dnet///vid.php
Disallow: /dnet////vid.php
Disallow: /dnet/////vid.php
Disallow: /dnet//////vid.php
Disallow: /dnet//browse_businesses.php
Disallow: /dnet///browse_businesses.php
Disallow: /dnet////browse_businesses.php
Disallow: /dnet/////browse_businesses.php
Disallow: /dnet//////browse_businesses.php
Disallow: /dnet//userpage.php
Disallow: /dnet///userpage.php
Disallow: /dnet////userpage.php
Disallow: /dnet/////userpage.php
Disallow: /dnet//////userpage.php
Disallow: /dnet//browse_pages.php
Disallow: /dnet///browse_pages.php
Disallow: /dnet////browse_pages.php
Disallow: /dnet/////browse_pages.php
Disallow: /dnet//////browse_pages.php
# Somehow this wrong link crept in :-/ - Kev, 2011/12/27
Disallow: /dnet//twitter_timeline_public.php
# Added 2011/06/08 by Kev after crawlers seemed to mess us up :-/
# Increased (3>8) 2011/12/27 by Kev after crawlers continued to mess us up :-/
# User-Agent line added 2019/02/21 because it looks like Bingbot doesn't respect
# it otherwise.
User-Agent: *
Crawl-delay: 8
# Disallow pages with session ID's in them for /dnet/ that gets search bots into
# a loop, Kev, 2019/11/16.
Disallow: /dnet/index.php?Deskdemon=*&type=rss;action=.xml