sitemap: http://mx.mercadojobs.com/sitemaps.php?sitemap=mjobs
User-agent: Twitterbot
Disallow:
#Baiduspider
User-agent: Baiduspider
Disallow: /
#Yandex
User-agent: Yandex
Disallow: /
User-agent: AdsBot-Google
Disallow:
User-agent: Googlebot
Disallow:
User-agent: Googlebot-image
Disallow:
User-agent: Googlebot-Video
Disallow:
User-agent: Googlebot-News
Disallow:
User-agent: Googlebot-Image
Disallow:
User-agent: msnbot
Disallow:
User-agent: msnbot-news
Disallow:
User-agent: msnbot-media
Disallow:
User-agent: msnbot-products
Disallow: /
User-agent: SLURP
Disallow:
crawl-delay: 120
User-agent: TEOMA
Disallow:
crawl-delay: 120
User-agent: Baiduspider
Disallow:
User-agent: Baiduspider-video
Disallow: /
User-agent: Baiduspider-image
Disallow: /
User-agent: moget
Disallow: /
User-agent: ichiro
Disallow: /
User-agent: NaverBot
Disallow: /
User-agent: Yeti
Disallow: /
User-agent: YandexBot
Disallow:
crawl-delay: 120
User-agent: Yandex
Disallow:
crawl-delay: 120
User-agent: sogou spider
Disallow: /
User-agent: YoudaoBot
Disallow: /
User-Agent: MJ12bot
Disallow: /
User-agent: grapeshot
Disallow: /
User-agent: *
Disallow:
crawl-delay: 10
User-agent: BomboraBot
Disallow: /
User-Agent: bingbot
Disallow: /