totaljobs.com

# tj.com - last edited 28 01 2020 AC 14:00
User-agent: Twitterbot
User-agent: AdsBot-Google
User-agent: AdsBot-Google-Mobile
User-agent: Mediapartners-Google
User-agent: Botify

Allow: /$
Allow: /*?WT.mc_id= # allow aggregated jobs
Allow: /*&WT.mc_id= # allow aggregated jobs - added 28 01 2020 AC
Allow: /*DCMP=
Allow: /*campaign-id=
Allow: /job/
Disallow: /job/*?
Allow: /jobs/
Allow: /jobs/*?q=
Disallow: /jobs/*?q=*&
Disallow: /jobs/*?
Allow: /jobs-at/
Disallow: /jobs-at/*?
Allow: /advice/
Disallow: /advice/*?
Allow: /salary-checker/
Disallow: /salary-checker/*?
Allow: /insidejob/
Disallow: /insidejob/*?
Allow: /recruiters/ # RD 10/3/2020
#Disallow: /recruiters/*? # added 06/05/2021 RD
Allow: /recruiters
Disallow: /

User-agent: Applebot 
User-agent: Bingbot
User-agent: BingPreview
User-agent: MSNBot
User-agent: Slurp

Allow: /$
Allow: /JobSearch/
Allow: /job/
Disallow: /job/*?
Allow: /jobs/
Allow: /jobs/*?q=
Disallow: /jobs/*?q=*&
Disallow: /jobs/*?
Allow: /jobs-at/
Disallow: /jobs-at/*?
Allow: /advice/
Disallow: /advice/*?
Allow: /salary-checker/
Disallow: /salary-checker/*?
Allow: /recruiters/
Disallow: /recruiters/*?
Allow: /insidejob/
Disallow: /insidejob/*?
Allow: /courses/
Disallow: /courses/*?
Disallow: /

User-agent: *
Allow: /*?*&crawl=true$
Allow: /*WT.mc_id=A_RE_GOOORG* 
Allow: /*WT.mc_id=A_SE_Go* # added 28/01/2020
Allow: /*WT.mc_id=R_AE_* # added 05/05/2020
Allow: /*WT.mc_id=B2B-* # added 06/05/2021 RD

Disallow: /*WT.mc_id= 
Disallow: /courses/search-results?
Disallow: /CompanySearch/CompanyLogoLink.asp
Disallow: /CompanySearch/CompanyVacancies.aspx
Disallow: /LearnDirect/
Disallow: /*w=*&as=
Disallow: /*DCMP=
Disallow: /*campaign-id=
Disallow: /*utm_medium=
Disallow: /browse-jobs?redir=*&CompanyTerm=
Disallow: /browse-jobs?redir=*&Keywords=&
Disallow: /*/main.asp
Disallow: /entry
Disallow: /*Visitor-Source=
Allow: /*?q*&page=  # 8/9/2017
Disallow: /*==$  #  2/11/2016
Disallow: /*.ashx$
Disallow: /*&page=  # this is where there is a page parameter on a facited link - its a catch all
Disallow: /*similarjobsmodal=
Disallow: /*entryUrl= # 5/10/18 RD
Disallow: /*entryurl= # 5/10/18 RD
Disallow: /job/*Visitor-Source= # 5/10/18 RD
Disallow: /jsd/
Disallow: /insidejob/wp-admin/
Disallow: /*/apply?
Disallow: /*/apply$
Disallow: /Login/
Disallow: /savedjobs
Disallow: /personalisation/
Disallow: /Accessible/
Disallow: /Authenticated/
Disallow: /authenticated/
Disallow: /cannedsearch/
Disallow: /CannedSearch/
Disallow: /CompanySearch/CompanyProfileFrameset.aspx
Disallow: /*/JobSearch/JobContactDetails.aspx
Disallow: /*/JobSearch/JobPrinterFriendlyDetails.aspx
Disallow: /*/JobSearch/Results.aspx
Disallow: /*/JobSearch/SendToAFriend.aspx
Disallow: /*/JobSearch/UnAuthApplyOnline.aspx
Disallow: /*Broaden=
Disallow: /*Rate=
Disallow: /*RateType=
Disallow: /*ValidFromDay=
Disallow: /*CompanyType=
Disallow: /*JobTitle=
Disallow: /*Discipline=
Disallow: /*JobType1=
Disallow: /*JobType=
Disallow: /*Radius=
Disallow: /*companytype=  # 5/10/18 RD
Disallow: /*&radius=
Disallow: /*?radius=
Disallow: /*industry=  # 2/12/20 RD
Disallow: /*Industry=
Disallow: /*salary=
Disallow: /*Salary=
Disallow: /*jobtype=
Disallow: /*postedwithin=
Disallow: /*PostedWithin=
Disallow: /*duration=  # 9/4/19 RD
#Disallow: /jobs/*&r= # 5/10/18 RD
#Disallow: /jobs/*?r= # 5/10/18 RD
Disallow: /JobSearch/JobBasket.aspx
Disallow: /JobSearch/JobContactDetails.aspx
Disallow: /JobSearch/JobPrinterFriendlyDetails.aspx
Disallow: /JobSearch/jobs-on-a-map.aspx
Disallow: /JobSearch/PromotedClick.aspx
Disallow: /JobSearch/SendToAFriend.aspx
Disallow: /JobSearch/UnAuthApplyOnline.aspx
Disallow: /JobSearch/PreExternalApplyOnline.aspx
Disallow: /JobSearch/JobsByEmailSetup.aspx
Disallow: /Jobsearch/Results.aspx
Disallow: /JobSearch/*/api/
Disallow: /jobsearch/*/api/
Disallow: /help/
Disallow: */articledetail.asp
Disallow: */archivedetails.asp
Disallow: /LocationHub/
Disallow: /jobs-at/jobs/in-
Disallow: /jobs-at/*/jobs/part-time/
Disallow: /jobs-at/*/jobs/permanent/
Disallow: /jobs-at/*/jobs/contract/
Disallow: /jobs-at/*/jobs/temporary/
Disallow: /mail_rd.asp
Disallow: /newwave_rd.asp
Disallow: /sitecore/*/www.
Disallow: /careers-advice/*/www.
Disallow: /JobSearch/LocationSelect.aspx
Disallow: /JobSearch/EmailLink.aspx
Disallow: /JobSearch/JobMailRecruiter.aspx
Disallow: /*.swf$
Disallow: /*/jbe/
Disallow: /JobsByEmail/
Disallow: /JobSearch/AdvancedJobSearch.aspx
Disallow: /account/ # updated 01/02/15 RD
Disallow: /Account/ # updated 17/07/16 RD
Disallow: /*/similarjobs/*
Disallow: /Jobsearch/ApplyViaDocuments.aspx
Disallow: /~/
Disallow: /login/
Disallow: /Help/
Disallow: /SalaryChecker/SalaryCheckerResults.aspx
Disallow: /jobseekers/*_new.asp
Disallow: /tag/
Disallow: /jobwidget/
Disallow: /advertise-
Disallow: /JobSearch/RSS.aspx
Disallow: /MSN/
Disallow: /pgl/
Disallow: /pjb_ui/
Disallow: /WebServices/
Disallow: /Webservices/
Disallow: /Maintenance.aspx
Disallow: /service/nolayout.aspx
# 2/12/20 RD
Disallow: /jobs-at/*/jobs/work-from-home/
Disallow: /harm/
Disallow: /*?action=
Disallow: /*&action=
Disallow: /jobs/work-from-home/*/in-*
Disallow: /recruiters/campaign-*


User-Agent: OmniExplorer_Bot
User-Agent: Verticrawlbot
User-agent: AddSearchBot
User-agent: admantx
User-agent: AhrefsBot
User-agent: Baiduspider
User-agent: Baiduspider-video
User-agent: Baiduspider-image
User-agent: ChangeDetection
User-agent: dotbot
User-agent: Exabot
User-agent: GozaikBot
User-agent: grapeshot
User-Agent: Ichiro
User-Agent: LinkedInBot/1.0  # added backin 10/04/19 RD
User-agent: proximic
User-agent: SeznamBot
User-agent: sogou spider
User-agent: trovitBot
User-agent: WeSEE
User-agent: WijuBot
User-agent: XoviBot
User-agent: YakazBot/1.0
User-agent: Yandexbot
User-agent: pingdom
User-agent: Mail.RU_Bot/2.0 # Mail.RU added 24/09/2014 RD
User-agent: Mail.RU_Bot
User-agent: mail.ru_bot/robots/2.0
User-agent: Mail.RU
User-agent: Mail.Ru
User-agent: Mozilla/5.0 (compatible; Linux x86_64; Mail.RU_Bot/2.0; +http://go.mail.ru/help/robots)
User-agent: Cliqzbot
User-agent: PiplBot
User-agent: CCBot
Disallow: /