# metmuseum.org/
# robots.txt created for http://metmuseum.org
user-agent: googlebot
allow: /
user-agent: googlebot-image
allow: /
user-agent: googlebot-mobile
allow: /
User-agent: Pinterest
Crawl-delay: 10
User-agent: Flipboard
disallow:
User-agent: FlipboardProxy
disallow:
User-agent: Twitterbot
Allow: /
User-agent: Mozilla/5.0 (compatible; parse.ly scraper/0.16; +http://parsely.com)
allow: /
user-agent: msnbot
disallow:
user-agent: slurp
disallow:
user-agent: teoma
disallow:
user-agent: twiceler
disallow:
user-agent: gigabot
disallow:
user-agent: scrubby
disallow:
user-agent: robozilla
disallow:
user-agent: nutch
disallow:
user-agent: ia_archiver
disallow:
user-agent: baiduspider-ads
disallow:
user-agent: naverbot
disallow:
user-agent: yeti
disallow:
user-agent: yahoo-mmcrawler
disallow:
user-agent: psbot
disallow:
user-agent: asterias
disallow:
user-agent: yahoo-blogs/v3.9
disallow:
#baiduspider
user-agent: baiduspider
disallow:
user-agent: mozilla/5.0(compatible; baiduspider/2.0; +http://www.baidu.com/search/spider.html)
disallow:
#yandex
user-agent: spider
disallow:
user-agent: yandex
disallow:
user-agent: yandex.com
disallow:
user-agent: yandexbot/3.0
disallow:
#sosospider
user-agent: sosospider+
disallow:
#ezooms
user-agent: ezooms
disallow:
user-agent: ezooms/1.0
disallow:
#sogou
user-agent: sogou spider
disallow:
user-agent: *
disallow: /api/
disallow: /sitecore/
disallow: /sitecore/
disallow: /sitecore_files/
disallow: /sitecore modules/
disallow: /app_browsers/
disallow: /app_config/
disallow: /app_data/
disallow: /temp/
disallow: /upload/
disallow: /xsl/
disallow: /ghidorah/
disallow: /kramar-notebook/
disallow: /art/libraries-and-research-centers/leonard-lauder-research-center/research/digital-archives/kramar-project
sitemap: http://www.metmuseum.org/sitemaps/sitemap_index