User-Agent: googlebot
User-Agent: bingbot
User-Agent: msnbot
User-Agent: Twitterbot
User-Agent: Slurp
User-Agent: DuckDuckBot
Crawl-delay: 30
sitemap: https://www.cuny.edu/sitemap.xml
sitemap: https://policy.cuny.edu/sitemap.xml
Disallow: /global-components/
Disallow: /d-i/
Disallow: /cgi-bin/
Disallow: /alumni
Disallow: /wp-admin
Disallow: /alumni-test/
Disallow: /role-template/
Disallow: /home-preview/
Disallow: /about/administration/offices/sa/
User-agent: SemanticScholarBot
Disallow: /
User-agent: *
Disallow: /