# See http://www.robotstxt.org/wc/norobots.html for documentation on how to use the robots.txt file
# MSN bot is prone to over-crawling
User-agent: msnbot
Crawl-delay: 120
# YandexBot bot is prone to over-crawling
User-agent: YandexBot
Crawl-delay: 120
# 80legs is often abused
User-agent: 008
Disallow: /
# Disabled due to abuse
User-agent: TurnitinBot
Disallow: /
# Disabled due to abuse
User-agent: WBSearchBot
Disallow: /
# Disabled due to abuse
User-agent: SearchmetricsBot
Disallow: /
# Please don't crawl the API
User-agent: *
Disallow: /api/
Disallow: /admin/
Disallow: /dashboard/
Disallow: /assets/