# Check this file using the following tool or similar
# http://tool.motoricerca.info/robots-checker.phtml
# User-agent: blocks must be separated by blank lines
User-agent: *
# scripts
Disallow: /cgi-bin/
Disallow: /cgi-bin-secure/
# directories
Disallow: /msuinfo/dx/
Disallow: /itc/
# voting system
Disallow: /vote/
Disallow: /wwwsy/
Disallow: /oldhome/
# wwwcat old or unused catalogs
Disallow: /wwwcat/programs1998-00/
Disallow: /wwwcat/programs1996-98/
Disallow: /wwwcat/programs2000-02/
Disallow: /wwwcat/programs2002-04/
Disallow: /wwwcat/2002-04_public_html/
Disallow: /wwwcat/2004-06_public_html/
Disallow: /wwwcat/2006-08_public_html/
Disallow: /wwwcat/2008-10_public_html/
#
Disallow: /wwwchem/
Disallow: /wwwdt/mta/
Disallow: /craigs/quizzes/buy/
Disallow: /bobcatmotorsports/modules/
Disallow: /bobcatmotorsports/modules/pharmacy-shop/
Disallow: /wwwhi/files/
Disallow: /arexpeditions/discussion/
Disallow: /arexpeditions/docs/
Disallow: /arexpeditions/networks/
Disallow: /cpa/graphics/workaround/
Disallow: /registrar/Videos/
# wiping from search engine cache
Disallow: /wwwfa/dlloans.htm
Disallow: /wwwpn/New-TerminatingEmployee/BZ_New_Hire_%20report%2008-16-06.xls
Disallow: /opa/salary_report/salary_report_docs/
Disallow: /emergency/documents/security-reports/2013-annual-security-report.pdf
Google 
Alexa 
Bing 
Aol 
Search.com