bay12forums.com

# Google - Most Important bot
#	Unfortunately a robots.txt will only stop it crawling certain urls, and NOT adding any
#	urls which it comes across into its index. So we're relying on a meta noindex tag.
User-agent: Googlebot
Disallow:

# Yahoo - Too aggressive
#	So limit it as much as possible.
User-agent: Slurp
# Disallow Everything
Disallow: /
# Now allow bits and then disallow bits
Allow: /robots.txt$
Allow: /smf/index.php$
Allow: /smf/index.php?topic=*.0$
Allow: /smf/index.php?topic=*.*0$
Allow: /smf/index.php?topic=*.*5$
Allow: /smf/index.php?board=*.0$
Allow: /smf/index.php?board=*.*0$
Allow: /smf/index.php?board=*.*5$
# But don't allow these
Disallow: /smf/index.php?*.msg
Disallow: /smf/index.php?topic=*.msg*0$
Disallow: /smf/index.php?topic=*.msg*5$
Disallow: /smf/index.php?*.new
# Anything with a ; disallow
Disallow: /smf/index.php?*;*

# Bad bot - Often ignores robots.txt - Waste of bandwidth
#	Despite claiming on their website to be a search engine in development
#	I'm suspicious as to whether they are a harvester pretending to be SE
User-agent: Twiceler
Disallow: /

# Archive.org's "wayback machine" ... nice concept, but not when it sucks down 1.3 GB of bandwidth within a week
User-agent: ia_archiver
Disallow: /

User-Agent: W3C-checklink
Disallow: /

# Stop following PHPSESSID's
User-Agent: MJ12bot
Disallow: /smf/index.php?PHPSESSID

# Catch all (remainder)
#	Will be followed by any bots other than ones identified above
#	Uses BASIC robots.txt directives without wildcards, end-anchors etc
#	So Spiders should understand these (including MSNBOT)
User-agent: *
# Default SMF Folders
Disallow: /smf/attachments/
Disallow: /smf/avatars/
Disallow: /smf/cache/
Disallow: /smf/Packages/
Disallow: /smf/Smileys/
Disallow: /smf/Sources/
Disallow: /smf/Themes/
# Default SMF Actions
Disallow: /smf/index.php?action=activate
Disallow: /smf/index.php?action=admin
Disallow: /smf/index.php?action=calendar
Disallow: /smf/index.php?action=emailuser
Disallow: /smf/index.php?action=findmember
Disallow: /smf/index.php?action=help
Disallow: /smf/index.php?action=helpadmin
Disallow: /smf/index.php?action=login
Disallow: /smf/index.php?action=logout
Disallow: /smf/index.php?action=mlist
Disallow: /smf/index.php?action=modifykarma
Disallow: /smf/index.php?action=pm
Disallow: /smf/index.php?action=post
Disallow: /smf/index.php?action=printpage
Disallow: /smf/index.php?action=profile
Disallow: /smf/index.php?action=recent
Disallow: /smf/index.php?action=register
Disallow: /smf/index.php?action=reminder
Disallow: /smf/index.php?action=search
Disallow: /smf/index.php?action=theme
Disallow: /smf/index.php?action=unread
Disallow: /smf/index.php?action=unreadreplies
Disallow: /smf/index.php?action=verificationcode
Disallow: /smf/index.php?action=who
Disallow: /smf/index.php?theme