kribi-info.com

# disallow all 
User-agent: *
Disallow: /
Crawl-delay: 600


# Disallow crawler
User-agent: Yandex
User-agent: moget
User-agent: ichiro
User-agent: NaverBot
User-agent: Yeti
User-agent: Baiduspider
User-agent: Baiduspider-video
User-agent: Baiduspider-image
User-agent: sogou spider
User-agent: YoudaoBot
User-agent: URLMetriken
Disallow: /

# but allow only important bots
User-agent: Googlebot
User-agent: Googlebot-Image
User-agent: Mediapartners-Google
User-agent: AdsBot-Google-Mobile
User-agent: Googlebot-News
User-agent: Google-Video
User-agent: Bingbot
User-agent: MSNBot-Media
User-agent: AdIdxBot
User-agent: BingPreview
Sitemap: http://www.kribi-info.com/sitemap.xml
Disallow: 

#User-agent: msnbot
#User-agent: msnbot-media
#User-agent: Slurp
#User-agent: Yahoo-Blogs
#User-agent: Yahoo-MMCrawler
# Directories
User-agent: *
Disallow: /admin/
Disallow: /adminfr/
Disallow: /bo1/
Disallow: /bo2/
Disallow: /languages/
Disallow: /block/
Disallow: /tmp/
# Files
#Disallow: /CHANGELOG.txt
#Disallow: /cron.php
#Disallow: /INSTALL.mysql.txt
#Disallow: /update.php
# Paths (clean URLs)
Disallow: /login/
Disallow: /logout/
Disallow: /user/
Disallow: /contact/
Disallow: /register/
Disallow: /login2/
Disallow: /pdf/
Disallow: /multiout/
Disallow: /comout/
# Paths (no clean URLs)
#Disallow: /?q=admin/
#Disallow: /?q=comment/reply/
Allow: /ads.txt