#Mozilla/5.0+(compatible;+Googlebot/2.1;++http://www.google.com/bot.html)
#rbs-crawler+(Enterprise;+T4-G4A8F9VEL6JLP;[email protected])
#Mozilla/5.0+(compatible;+heritrix/3.0.0++http://www.mindbreeze.com)
User-agent: T4-G4A8F9VEL6JLP
Disallow: /
User-agent: MJ12Bot
Disallow: /
User-agent: heritrix
Disallow: /
User-agent: *
Crawl-delay: 30
#don't crawl style data (css/images etc)
#Disallow: /EasySiteWeb/EasySite/StyleData/*
#Disallow: /easysiteweb/easysite/styledata/*
Disallow: /*espagelisterespager_p
Disallow: /*/?char=
Disallow: /*/?entryid4=
Disallow: /*/?cord=
Disallow: /*/?pxxx=
Disallow: /*?pxxx=
#Disallow: /contact/?
#Disallow: /contact?
#Disallow: /search
#Disallow: /search.*
Disallow: /membership/our-members/?cid=.*
Disallow: /membership/our-industrial-members/?cid=.*
Disallow: /*&cord=*C
Disallow: /?action=logout
#Disallow: /services/research-and-consultancy/public-funded-projects/*cid=.*
Disallow: /EasySiteWeb/EasySite/Handlers/EventsICalHandler.ashx?sskey=
Disallow: /news-events/events/*EventId2=1357544&sDate=*&sdate*
Disallow: /*/*spagelister*
Disallow: /j32k/unprotected/band_1/*eaey*.html
Disallow: /j32k/unprotected/pdfs/*eaey*.pdf
Disallow: /j32k/Forms/RegFormEAEY+H.xtp
Disallow: *.flv
Disallow: /EasysiteWeb/contact_34.swf
Disallow: /technologies/welding-surface-engineering-and-material-processing/plastics-welding/plastic-pipe-weldi...
Disallow: /EasysiteWeb/get$
Disallow: /dev/*
Disallow: /ca/*
Disallow: /aep/*
Disallow: /wsp/*
#Disallow: /login/*