www.unesco.org

User-agent: *
Disallow: /iftc/ 
Disallow: /rss/scripts/
Disallow: /error/
Disallow: /science/forum/
Disallow: /education/forum/
Disallow: /ulis/cgi-bin/
Disallow: /moscow-new/ 
Disallow: /ulis/cgi-bin/
Disallow: /new/fr/tip-a-friend/
Disallow: /new/en/tip-a-friend/
Disallow: /new/es/tip-a-friend/
Disallow: /pv_obj_cache/
Disallow: /tools/filedepot/
Disallow: /new/index.php?eID=tx_cms_showpic*
Disallow: /archives/multimedia/extra/

Disallow: /fileadmin/unesco/
Disallow: /uploads/

Disallow: /cms/ # Nothing to see here
Disallow: /t3lib/ # Nothing to see here
Disallow: /typo3/ # Nothing to see here
Disallow: /typo3conf/ # Nothing to see here
Disallow: /typo3temp/ # Nothing to see here
Disallow: /index.php?id=* # Disable non-realurl


Disallow: /new/fileadmin/TEMPLATES/  # Exclude only folders with no link from frontend, like templates, css, js.
Disallow: /new/fileadmin/unesco/
Disallow: /new/fileadmin/NICE/
Disallow: /new/uploads/

Disallow: /new/cms/ # Nothing to see here
Disallow: /new/t3lib/ # Nothing to see here
Disallow: /new/typo3/ # Nothing to see here
Disallow: /new/typo3conf/ # Nothing to see here
Disallow: /new/typo3temp/ # Nothing to see here
Disallow: /new/index.php?id=* # Disable non-realurl
Disallow: /new/en/media-services/for-the-press/all-news/

Disallow: /mediabank/cgi/
Disallow: /mediabank/views/

User-agent: bingbot
Crawl-delay: 10

User-agent: BLEXBot
Crawl-delay: 10

User-agent: SemrushBot
Crawl-delay: 10

User-agent: PetalBot
Disallow: /