# robots.txt for https://ablio.eu/ and friends
# Friendly, low-speed bots are welcome viewing pages, but not
# dynamically-generated pages.
# If you're irresponsible, your access to the site may be blocked.
# DISALLOWED CRAWLERS
User-agent: Ahrefs*
#User-agent: DOC
User-agent: Download Ninja
User-agent: Fetch
User-agent: HTTrack
User-agent: MJ12bot
#User-agent: NPBot
User-agent: Offline Explorer
User-agent: SemrushBot
User-agent: SiteSnagger
User-agent: Telepor*
User-agent: Testomatobot
#User-agent: UbiCrawler
User-agent: WebCopier
#User-agent: WebReaper
User-agent: WebStripper
User-agent: WebZIP
User-agent: Xenu
#User-agent: Zao
User-agent: Zealbot
User-agent: ZyBORG
User-agent: duggmirror
#User-agent: grub-client
User-agent: ia_archive*
User-agent: k2spider
User-agent: larbin
#User-agent: libwww
User-agent: linko
User-agent: sitecheck.internetseer.com
#User-agent: wget
Disallow: /
# APPLE
#User-agent: Applebot
#Disallow: /
# BAIDU
#User-agent: Baiduspider
#Disallow: /
# FACEBOOK
#User-agent: Facebot/1.0
#Crawl-delay: 10
#Disallow: /
# GOOGLE
#User-agent: Adsbot-Google
#User-agent: Googlebot
#User-agent: Googlebot-Image
#User-agent: Googlebot-Mobile
#User-agent: Mediapartners-Googl*
#Disallow: /
# MICROSOFT
#User-agent: MSIECrawler
#User-agent: Microsoft.URL.Control
#User-agent: bingbot
#User-agent: msnbot
#Crawl-delay: 30
#Disallow: /
# CRAWL DELAY (in seconds)
# There are some misbehaved spiders out there that go _way_ too fast.
#User-agent: Facebot/1.0
#User-agent: Slurp
#User-agent: msnbot
#User-agent: noxtrumbot
#Crawl-delay: 10
# ALL CRAWLERS
User-agent: *
Crawl-delay: 2
# ALLOWED URLs (see 'urllist.txt')
Allow: /$
Allow: /about-us/$
Allow: /careers/$
Allow: /feed/
Allow: /it/$
Allow: /it/about-us/$
Allow: /it/careers/$
Allow: /it/language-interpreting-platforms/$
Allow: /it/opi-services/$
Allow: /it/privacy-policy/$
Allow: /it/privacy-policy/about-privacy-rules-and-regulations/$
Allow: /it/privacy-policy/archives/$
Allow: /it/privacy-policy/cookie-policy/$
Allow: /it/privacy-policy/security-policy/$
Allow: /it/privacy-policy/third-party/$
Allow: /it/references/$
Allow: /it/rsi-features/$
Allow: /it/servizi-di-interpretariato/$
Allow: /it/simultaneous-interpreting-services/$
Allow: /it/subtitles/$
Allow: /it/terms-and-conditions/$
Allow: /it/translations/$
Allow: /language-interpreting-platforms/$
Allow: /opi-services/$
Allow: /privacy-policy/$
Allow: /privacy-policy/about-privacy-rules-and-regulations/$
Allow: /privacy-policy/archives/$
Allow: /privacy-policy/cookie-policy/$
Allow: /privacy-policy/security-policy/$
Allow: /privacy-policy/third-party/$
Allow: /references/$
#Allow: /robots.txt$
Allow: /ror.xml$
Allow: /rsi-features/$
Allow: /simultaneous-interpreting-services/$
Allow: /sitemap.xml$
Allow: /sitemap.xml.gz$
Allow: /subtitles/$
Allow: /terms-and-conditions/$
Allow: /translations/$
Allow: /urllist.txt$
Allow: /wp-content/uploads
Allow: /wp-sitemap.xml$
# ALL OTHER URLs ARE DISALLOWED!
#Disallow: /
# SITEMAPs
Sitemap: https://ablio.eu/ror.xml
Sitemap: https://ablio.eu/sitemap.xml
Sitemap: https://ablio.eu/sitemap.xml.gz
Sitemap: https://ablio.eu/urllist.txt
Sitemap: https://ablio.eu/wp-sitemap.xml