austlii.edu.au

# Google adsbot ignores robots.txt unless specifically named!
User-agent: adsbot-google
Disallow: /
#
# 2 July 2010 - unrestricted access to everything except the below
User-agent: *
Crawl-delay: 120
Disallow: /au/cases/
Disallow: /au/legis/cth/digest/
Disallow: /au/other/
Disallow: /austlii/stats/
Disallow: /nz/cases/
Disallow: /au/special/
Disallow: /austlii/metstats/

#### 5 Dec 2018

# Added cases as somehow all cases indexed via this link that does not exist! TR 17 June 2011
Disallow: /cases/
Disallow: /cgi-bin/
Disallow: /cgi-bin/sinodisp/
Disallow: /cgi-dev/
Disallow: /do/
Disallow: /do2/
Disallow: /LawCite/
Disallow: /lawcite/
Disallow: /incite/
Disallow: /inCite/
Disallow: /form/
Disallow: /forms/
Disallow: /fcgi-bin/
Disallow: /rsjlibrary/rciadic/
Disallow: /austlii/survey/
# Disallow indexing of dynamically generated pages eg search results
# by Google
Disallow: /*?
# PTHC-20080107
# Disallow access to personal directories
Disallow: /~andrew/
Disallow: /~armin/
Disallow: /~jones/
Disallow: /~joseph/
Disallow: /~philip/
Disallow: /~trev/
Disallow: /~chris/

User-agent: Gromit
Disallow: /cgi-bin/
Disallow: /fcgi-bin/
Disallow: /do/
Disallow: /do2/
Disallow: /au/cases/
Disallow: /au/legis/
Disallow: /au/other/
Disallow: /au/journals/
Disallow: /au/special/
Disallow: /nz/cases/

#JO - 20160826
User-agent: Baiduspider
User-agent: Baiduspider-video
User-agent: Baiduspider-image
Crawl-delay: 120
Disallow: /au/cases/
Disallow: /au/legis/cth/digest/
Disallow: /au/other/
Disallow: /austlii/stats/
Disallow: /nz/cases/
Disallow: /au/special/
Disallow: /austlii/metstats/

# Australian Business On-Line
# Created: PTHC-20000919
#
User-agent: Australian Business On-Line
Disallow: /au/legis/
Disallow: /au/other/hca/
Disallow: /au/other/iponline/
Disallow: /au/other/ipaus/
Disallow: /au/journals/
Disallow: /rsjlibrary/rciadic
Disallow: /cgi-bin/
Disallow: /do/
Disallow: /do2/
Disallow: /lists/
Disallow: /ombud/
Disallow: /austlii/editors/
Disallow: /austlii/FAQ
Disallow: /form/
Disallow: /fcgi-bin/

# Wang -- you are flooding the server please e-mail us
# User-agent: Wang New Zealand
# Disallow: /

User-agent: Wang New Zealand
Disallow: /au/legis/
Disallow: /au/cases/
Disallow: /au/other/hca/
Disallow: /au/other/iponline/
Disallow: /au/other/ipaus/
Disallow: /au/journals/
Disallow: /rsjlibrary/rciadic
Disallow: /cgi-bin/
Disallow: /fcgi-bin/
Disallow: /do/
Disallow: /do2/
Disallow: /lists/
Disallow: /ombud/
Disallow: /austlii/editors/
Disallow: /austlii/FAQ
Disallow: /nz/
Disallow: /form/

# Googlebot --You flooding the server with request please e-mail us
# 6 August 2002
# Googlebot -- Allow google to index everything except cases
# 14 August 2003
# User-agent: Googlebot
# Disallow: /au/cases
# Disallow: /nz/cases


# Slurp --You are flooding the server with request please e-mail us
# 08 Oct 2002
User-agent: Slurp
Disallow: /au/other/
Disallow: /au/cases/
Disallow: /au/legis/
Disallow: /au/special/
Disallow: /au/journals
Disallow: /cgi-bin/
Disallow: /fcgi-bin/
Disallow: /form/

#User-agent: FAST Enterprise Crawler 6 used by LexisNexisAU
#Fast crawler banned from crawling cases
#Disallow: /au/cases/
#Disallow: /au/other/
#Disallow: /nz/cases/
#Disallow: /cgi-bin/
#Disallow: /cgi-dev/
#Disallow: /do/
#Disallow: /do2/
#Disallow: /LawCite/
#Disallow: /lawcite/
#Disallow: /incite/
#Disallow: /inCite/
#Disallow: /form/
#Disallow: /forms/
#Disallow: /fcgi-bin/
#Disallow: /rsjlibrary/rciadic

# Disallow forbidden directories
Disallow: *.OLD/
Disallow: *.NEW/
Disallow: *.BUILD/

# Disallow forbidden files
Disallow: /au/journals/FedLawRw/2001/23.html