Sitemap: https://computertotaal.nl/sitemap.xml
# Disallow crawling the 404 and 500 pages
User-agent: *
Disallow: /404/
Disallow: /500.html
Disallow: /password_reset/
Disallow: /cms/
Crawl-delay: 20
User-agent: MJ12bot
Disallow: /
User-agent: Googlebot
Disallow: /404/
Disallow: /500.html
Disallow: /password_reset/
Disallow: /cms/
Crawl-delay: 15
User-agent: bingbot
Disallow: /404/
Disallow: /500.html
Disallow: /password_reset/
Disallow: /cms/
Crawl-delay: 15
User-agent: msnbot
Disallow: /404/
Disallow: /500.html
Disallow: /password_reset/
Disallow: /cms/
Crawl-delay: 15