# --------------------------------------------------------------
# robots.txt for http(s)://(www.)xin.at, last refresh 2018/06/10
# --------------------------------------------------------------
# not all bots below may obey robots.txt in general
# or specific rules, respectively
# --------------------------------------------------------------
# http://crawler.007ac9.net/
User-agent: 007ac9
Disallow: /
# http://www.aboundex.com/crawler/
User-agent: Aboundexbot
Disallow: /
# https://www.adbeat.com/operation_policy
User-agent: adbeat_bot
Disallow: /
User-agent: AddThis
Disallow: /
# http://advbot.org/bot.html
User-agent: AdvBot
Disallow: /
# https://ahrefs.com/robot/
User-agent: AhrefsBot
Disallow: /
User-agent: aiHitBot
Disallow: /
# http://alphaseobot.com/bot.html
User-agent: AlphaSeoBot
Disallow: /
# http://alphaseobot.com/bot.html
User-agent: AlphaSeoBot-SA
Disallow: /
# https://audisto.com/help/crawler/bot
User-agent: audisto
Disallow: /
# https://audisto.com/help/crawler/bot
User-agent: audisto-essential
Disallow: /
# https://awario.com/bots.html
User-agent: AwarioRssBot
Disallow: /
# https://awario.com/bots.html
User-agent: AwarioSmartBot
Disallow: /
# http://www.backlinktest.com/crawler.html
User-agent: BacklinkCrawler
Disallow: /
# (http://begun.ru/advertiser/technologies/indexer.php)
User-agent: BegunAdvertising
Disallow: /
User-agent: betaBot
Disallow: /
# (https://bitly.com/)
User-agent: bitlybot
Disallow: /
# http://webmeup-crawler.com/
User-agent: BLEXBot
Disallow: /
User-agent: BoogleBot
Disallow: /
# http://law.di.unimi.it/BUbiNG.html
User-agent: BUbiNG
Disallow: /
# https://hypefactors.com/webcrawler
User-agent: BuckyOHare
Disallow: /
# no more active!?
User-agent: careerbot
Disallow: /
User-agent: ca-crawler
Disallow: /
User-agent: calculon
Disallow: /
# http://catchbot.com/ (via archive.org)
User-agent: CatchBot
Disallow: /
# http://commoncrawl.org/big-picture/frequently-asked-questions/
User-agent: CCBot
Disallow: /
User-agent: CheckMarkNetwork/1.0 (+http://www.checkmarknetwork.com/spider.html)
Disallow: /
# http://www.cmscrawler.com/ (missachtet robots.txt)
User-agent: CMS Crawler
Disallow: /
User-agent: CompSpyBot
Disallow: /
User-agent: crawler4j
Disallow: /
# http://www.crazywebcrawler.com/
User-agent: CrazyWebCrawler-Spider
Disallow: /
# https://www.companiesintheuk.co.uk/bot.html
User-agent: CukBot
Disallow: /
# https://www.dataprovider.com/spider/
User-agent: Dataprovider
Disallow: /
# https://www.deepcrawl.com/bot/
User-agent: deepcrawl
Disallow: /
# https://benbernardblog.com/der-bot/
User-agent: DER-bot
Disallow: /
User-agent: Diffbot
Disallow: /
# https://support.dlvrit.com/hc/en-us/articles/200402934-How-do-I-block-dlvr-it-from-retrieving-the-feeds-on-my-site-
User-agent: dlvr.it
Disallow: /
# (http://www.profound.net/domainappender)
User-agent: DomainAppender
Disallow: /
# http://domainstats.io/our-bot
User-agent: DomainStatsBot
Disallow: /
# https://moz.com/researchtools/ose/dotbot (previously Ezooms bot!?)
User-agent: dotbot
Disallow: /
# http://adressendeutschland.de/ (obeys specific rule!?)
User-agent: dubaiindex
Disallow: /
# https://www.duedil.com/our-crawler
User-agent: ElectricMonk
Disallow: /
# http://www.envolk.com/envolkspiderinfo.html
User-agent: envolk
Disallow: /
User-agent: evc-batch/2.0
Disallow: /
# http://www.expertsearch.nl/spider (redirects to JobdiggerSpider)
User-agent: ExpertSearchSpider
Disallow: /
# https://extlinks.com/Bot.html
User-agent: ExtLinksBot
Disallow: /
User-agent: Fatbot
Disallow: /
# http://www.findxbot.com/
User-agent: Findxbot
Disallow: /
User-agent: fr-crawler
Disallow: /
# (http://www.garlik.com)
User-agent: GarlikCrawler
Disallow: /
# http://www.genieo.com/webfilter.html
User-agent: Genieo
Disallow: /
# https://www.gloomar.com/bot
User-agent: gloomarbot
Disallow: /
# http://www.grapeshot.com/crawler/
User-agent: grapeshot
Disallow: /
# http://www.huaweisymantec.com/en/IRL/spider/ (via archive.org)
User-agent: HuaweiSymantecSpider
Disallow: /
# https://knowledge.hubspot.com/articles/kcs_article/reports/why-do-i-get-an-error-in-page-performance-that-the-crawler-is-blocked-by-robots-txt-for-my-hubspot-staging-domain
User-agent: HubSpot Crawler 1.0 http://www.hubspot.com/
Disallow: /
# https://knowledge.hubspot.com/articles/kcs_article/reports/why-do-i-get-an-error-in-page-performance-that-the-crawler-is-blocked-by-robots-txt-for-my-hubspot-staging-domain
User-agent: HubSpot Links Crawler 1.0 http://www.hubspot.com/
Disallow: /
# http://www.seograph.net/bot.html
User-agent: HyperCrawl
Disallow: /
# http://www.hypestat.com/bot
User-agent: HypeStat
Disallow: /
# https://integralads.com/site-indexing-policy/
User-agent: ias_crawler
Disallow: /
# https://www.icjobs.de/bot.htm
User-agent: ICCrawler - iCjobs
Disallow: /
# http://www.idmarch.org/bot.html
User-agent: idmarch
Disallow: /
# (http://implisense.com/)
User-agent: ImplisenseBot
Disallow: /
User-agent: ips-agent
Disallow: /
# http://irl.cs.tamu.edu/crawler/
User-agent: IRLbot
Disallow: /
# https://cognitiveseo.com/bot.html
User-agent: JamesBOT
Disallow: /
# https://www.jobb�rse.com/bot.htm
User-agent: JobboerseBot
Disallow: /
# http://www.jobdigger.nl/spider/
User-agent: JobdiggerSpider
Disallow: /
# http://static.kalooga.com/legal/crawler.html
User-agent: kalooga
Disallow: /
# (http://linkfluence.net/)
User-agent: Kraken
Disallow: /
# (http://larbin.sourceforge.net/index-eng.html)
User-agent: larbin
Disallow: /
# https://www.linkdex.com/en-us/about/bots/ (doesn't obey specific rule)
User-agent: linkdexbot
Disallow: /
# http://www.linkpad.ru
User-agent: LinkpadBot
Disallow: /
# https://www.lipperhey.com/en/about/website-spider/ (doesn't obey specific rule)
User-agent: Lipperhey
Disallow: /
User-agent: LSSRocketCrawler
Disallow: /
# http://ltx71.com/
User-agent: ltx71
Disallow: /
# https://www.brandwatch.com/magpie-crawler/
User-agent: magpie-crawler
Disallow: /
# http://mappydata.net/#eng
User-agent: Mappy
Disallow: /
User-agent: MauiBot
Disallow: /
# http://www.meanpath.com/meanpathbot.html
User-agent: meanpathbot
Disallow: /
User-agent: MegaIndex.ru 2.0
Disallow: /
# http://www.metajob.de/the/crawler
User-agent: metajobbot
Disallow: /
# http://www.majestic12.co.uk/projects/dsearch/mj12bot.php
User-agent: MJ12bot
Disallow: /
User-agent: MSIECrawler
Disallow: /
User-agent: nbot
Disallow: /
# https://www.neofonie.de/spider/
User-agent: neofonie
Disallow: /
# http://nerdybot.com/
User-agent: NerdyBot
Disallow: /
User-agent: NetcraftSurveyAgent
Disallow: /
User-agent: NetpeakSpiderBot
Disallow: /
# http://www.netseer.com/crawler/
User-agent: netseer
Disallow: /
# http://www.zoominfo.com/About/misc/NextGenSearchBot.aspx (via arcvhive.org)
User-agent: NextGenSearchBot
Disallow: /
# http://nutch.apache.org/bot.html
User-agent: Nutch
Disallow: /
# http://filterdb.iss.net/crawler/
User-agent: oBot
Disallow: /
# https://www.openindex.io/saas/about-our-spider/
User-agent: OpenindexSpider
Disallow: /
# http://www.panscient.com/faq.htm
User-agent: panscient.com
Disallow: /
# https://pipl.com/bot/
User-agent: PiplBot
Disallow: /
# http://www.botje.com/plukkie.htm
User-agent: plukkie
Disallow: /
User-agent: Pockey-GetHTML
Disallow: /
# https://www.comscore.com/proximic-spider
User-agent: proximic
Disallow: /
# https://www.semanticjuice.com/web-crawler.php
User-agent: Pu_iN
Disallow: /
# (bixocrawler)
User-agent: pub-crawler
Disallow: /
# R6_FeedFetcher + R6_CommentReader (www.radian6.com/crawler)
User-agent: Radian6
Disallow: /
# https://ranksonic.com/ranksonic_sab.html
User-agent: RankSonicSiteAuditor
Disallow: /
# https://raven.zendesk.com/hc/en-us/articles/203221440-How-do-I-slow-down-Site-Auditor-s-crawl-of-my-website-
# (https://raventools.com/)
User-agent: RavenCrawler
Disallow: /
# https://redbot.org/about/
User-agent: RED
Disallow: /
# (http://g2pi.tsc.uc3m.es/)
User-agent: RedesScrapy
Disallow: /
# http://riddler.io/about
User-agent: Riddler
Disallow: /
# http://moz.com/help/pro/what-is-rogerbot- (via archive.org)
User-agent: rogerbot
Disallow: /
# https://www.safedns.com/searchbot
User-agent: SafeDNSBot
Disallow: /
# http://www.secretsearchenginelabs.com/secret-web-crawler.php
User-agent: SBSearch
Disallow: /
# http://scoutjet.com/
# (Blekkobot)
User-agent: ScoutJet
Disallow: /
# https://www.screamingfrog.co.uk/seo-spider/user-guide/general/
User-agent: Screaming Frog SEO Spider
Disallow: /
# http://de.semrush.com/bot/
User-agent: SemrushBot
Disallow: /
# http://sentibot.eu/
User-agent: sentibot
Disallow: /
# https://seocharger.com/robot
User-agent: seocharger-robot
Disallow: /
# http://www.seokicks.de/robot.html
User-agent: SEOkicks-Robot
Disallow: /
# http://crawler.seolytics.net/
User-agent: SEOlyticsCrawler
Disallow: /
# http://crawler.seolytics.net/
User-agent: seoscanners.net/1
Disallow: /
# http://www.searchgears.com/ueber-uns/crawling-faq.html
User-agent: sg-Orbiter
Disallow: /
# http://crawler.sistrix.net/
User-agent: sistrix
Disallow: /
# http://siteexplorer.info/about.html
User-agent: SiteExplorer
Disallow: /
# http://www.sitesucker.us/mac/limitations.html
User-agent: SiteSucker
Disallow: /
# https://smabbler.com/en/Home/About
User-agent: SmabblerBot
Disallow: /
# https://www.similartech.com/smtbot
User-agent: SMTBot
Disallow: /
# http://www.openlinkprofiler.org/bot
User-agent: spbot
Disallow: /
# http://nlp.fi.muni.cz/projects/biwec/
User-agent: SpiderLing
Disallow: /
User-agent: Spiderlytics
Disallow: /
User-agent: ssearch_bot
Disallow: /
User-agent: sSearch Crawler
Disallow: /
# http://www.tkl.iis.u-tokyo.ac.jp/~crawler/
User-agent: Steeler
Disallow: /
User-agent: stq_bot
Disallow: /
# http://sur.ly/bot.html
User-agent: SurdotlyBot
Disallow: /
User-agent: SurveyBot
Disallow: /
# https://swiftype.com/swiftbot
User-agent: Swiftbot
Disallow: /
User-agent: The Knowledge AI
Disallow: /
# http://www.thumbshots.de/content-39-seite_auszuschliessen.html
User-agent: thumbshots-de-bot
Disallow: /
# https://thumbsniper.com/cat/news/ (via Google cache)
User-agent: ThumbSniper (http://thumbsniper.com)
Disallow: /
# http://search.thunderstone.com/texis/websearch/about.html
User-agent: thunderstone
Disallow: /
# (https://www.tineye.com/crawler.html)
User-agent: TinEye
Disallow: /
# http://www.toshiba.co.jp/rdc/about/crawl_info_en.htm
User-agent: TosCrawler
Disallow: /
# http://www.trendiction.com/de/publisher/bot
User-agent: trendictionbot
Disallow: /
# https://turnitin.com/robot/crawlerinfo.html
User-agent: TurnitinBot
Disallow: /
# https://blog.ucoz.ru/upolicy/
User-agent: uCrawler
Disallow: /
# https://uptime.com/uptimebot
User-agent: Uptimebot
Disallow: /
# http://www.wise-guys.nl/webcrawler.php
User-agent: Vagabondo
Disallow: /
# (https://blog.vebidoo.de/vebidoobot/)
User-agent: vebidoobot
Disallow: /
# https://velen.io/
User-agent: VelenPublicWebCrawler
Disallow: /
# http://80legs.com/the-80legs-web-crawler/
User-agent: voltron
Disallow: /
# http://warebay.com/bot.html (expired)
User-agent: WBSearchBot
Disallow: /
# http://rvs.informatik.uni-leipzig.de/bot.php
User-agent: webmeasurement-bot
Disallow: /
# http://www.wesee.com/bot (via archive.org)
User-agent: WeSEE
Disallow: /
User-agent: WeViKaBot
Disallow: /
# http://www.wikido.com/wikido.php
User-agent: WikiDo
Disallow: /
User-agent: wonderbot
Disallow: /
# http://www.wotbox.com/bot/
User-agent: wotbox
Disallow: /
# http://x28.ch/bot.html
User-agent: x28-job-bot
Disallow: /
# http://www.xovibot.net/
User-agent: XoviBot
Disallow: /
# ------------
# allowed bots
# ------------
User-agent: *
Crawl-delay: 5