# Allow only major search spiders
User-agent: Mediapartners-Google
Disallow: /config/
Disallow: /handlers/
Disallow: /includes/
Disallow: /interceptors/
Disallow: /layouts/
Disallow: /logs/
Disallow: /model/
Disallow: /plugins/
Disallow: /views/
Allow: /
Crawl-delay: 4
User-agent: Googlebot
Disallow: /config/
Disallow: /handlers/
Disallow: /includes/
Disallow: /interceptors/
Disallow: /layouts/
Disallow: /logs/
Disallow: /model/
Disallow: /plugins/
Disallow: /views/
Allow: /
Crawl-delay: 4
User-agent: Adsbot-Google
Disallow: /config/
Disallow: /handlers/
Disallow: /includes/
Disallow: /interceptors/
Disallow: /layouts/
Disallow: /logs/
Disallow: /model/
Disallow: /plugins/
Disallow: /views/
Allow: /
Crawl-delay: 4
User-agent: Googlebot-Image
Disallow: /config/
Disallow: /handlers/
Disallow: /includes/
Disallow: /interceptors/
Disallow: /layouts/
Disallow: /logs/
Disallow: /model/
Disallow: /plugins/
Disallow: /views/
Allow: /
Crawl-delay: 4
User-agent: Googlebot-Mobile
Disallow: /config/
Disallow: /handlers/
Disallow: /includes/
Disallow: /interceptors/
Disallow: /layouts/
Disallow: /logs/
Disallow: /model/
Disallow: /plugins/
Disallow: /views/
Allow: /
Crawl-delay: 4
User-agent: MSNBot
#Disallow: /config/
#Disallow: /handlers/
#Disallow: /includes/
#Disallow: /interceptors/
#Disallow: /layouts/
#Disallow: /logs/
#Disallow: /model/
#Disallow: /plugins/
#Disallow: /views/
#Allow: /
Disallow: /
#Crawl-delay: 4
User-agent: bingbot
#Disallow: /config/
#Disallow: /handlers/
#Disallow: /includes/
#Disallow: /interceptors/
#Disallow: /layouts/
#Disallow: /logs/
#Disallow: /model/
#Disallow: /plugins/
#Disallow: /views/
#Allow: /
Disallow: /
#Crawl-delay: 4
User-agent: Slurp
#Disallow: /config/
#Disallow: /handlers/
#Disallow: /includes/
#Disallow: /interceptors/
#Disallow: /layouts/
#Disallow: /logs/
#Disallow: /model/
#Disallow: /plugins/
#Disallow: /views/
#Allow: /
Disallow: /
#Crawl-delay: 4
User-agent: Yahoo! Slurp
#Disallow: /config/
#Disallow: /handlers/
#Disallow: /includes/
#Disallow: /interceptors/
#Disallow: /layouts/
#Disallow: /logs/
#Disallow: /model/
#Disallow: /plugins/
#Disallow: /views/
#Allow: /
Disallow: /
#Crawl-delay: 4
# added 10/19/2016, bbale - Browsershots is a multi-platform browser emulator which respects robots.txt
User-agent: Browsershots
Disallow: /config/
Disallow: /handlers/
Disallow: /includes/
Disallow: /interceptors/
Disallow: /layouts/
Disallow: /logs/
Disallow: /model/
Disallow: /plugins/
Disallow: /views/
Allow: /
#Crawl-delay: 4
# Block all other spiders
User-agent: *
Disallow: /
Disallow: /config/
Disallow: /forms/
Disallow: /handlers/
Disallow: /includes/
Disallow: /interceptors/
Disallow: /layouts/
Disallow: /logs/
Disallow: /model/
Disallow: /plugins/
Disallow: /views/