User-agent: 360Spider
Crawl-delay: 100
User-agent: Baiduspider
Allow:
User-agent: Baiduspider-image
Allow:
User-agent: Bingbot
Crawl-delay: 100
User-agent: Googlebot
Allow:
User-agent: HaoSouSpider
Allow:
User-agent: Sogou web spider
Allow:
User-agent: Sogou spider2
Allow:
User-agent: Slurp
Crawl-delay: 100
User-agent: ToutiaoSpider
Allow:
User-agent: YandexBot
Allow:
User-agent: ia_archiver
Allow:
User-agent: Robozilla
Allow:
User-agent: AddSugarSpiderBot
Disallow: /
User-agent: AdsBot-Google
Disallow: /
User-agent: AhrefsBot
Disallow: /
User-agent: archive.org_bot
Disallow: /
User-agent: AugustBot
Disallow: /
User-agent: Barkrowler
Disallow: /
User-agent: BLEXBot
Disallow: /
User-agent: CatchBot
Disallow: /
User-agent: CCBot
Disallow: /
User-agent: ChinasoSpider
Disallow: /
User-agent: DoCoMo
Disallow: /
User-agent: DotBot
Disallow: /
User-agent: EasouSpider
Disallow: /
User-agent: EtaoSpider
Disallow: /
User-agent: Ezooms
Disallow: /
User-agent: FoxSpider
Disallow: /
User-agent: Gigabot
Disallow: /
User-agent: istellabot
Disallow: /
User-agent: JikeSpider
Disallow: /
User-agent: LinkpadBot
Disallow: /
User-agent: Lycos
Disallow: /
User-agent: ltx71
Disallow: /
User-agent: MJ12bot
Disallow: /
User-agent: msnbot
Disallow: /
User-agent: msnbot-media
Disallow: /
User agent: MTRobot
Disallow: /
User-agent: psbot
Disallow: /
User-agent: SafeSearch microdata crawler
Disallow: /
User-agent: SemrushBot
Disallow: /
User-agent: SeznamBot
Disallow: /
User-agent: SinaWeiboBot
Disallow: /
User-agent: SiteExplorer
Disallow: /
User-agent: Slackbot-LinkExpanding
Disallow: /
User-agent: Sogou blog
Disallow: /
User-agent: Sogou inst spider
Disallow: /
User-agent: Sogou News Spider
Disallow: /
User-agent: Sogou Orion spider
Disallow: /
User-agent: Sosospider
Disallow: /
User-agent: spbot
Disallow: /
User-agent: SurveyBot
Disallow: /
User-agent: TweetmemeBot
Disallow: /
User-agent: uMBot-LN
Disallow: /
User-agent: WangIDSpider
Disallow: /
User-agent: Y!J-ASR
Disallow: /
User-agent: yacybot
Disallow: /
User-agent: Yeti
Disallow: /
User-agent: yisouspider
Disallow: /
User-agent: YodaoBot
Disallow: /
User-agent: yoozBot
Disallow: /
User-agent: YoudaoBot
Disallow: /
User-agent: YRSpider
Disallow: /
User-agent: ZumBot
Disallow: /
#Updated on April. 13, 2021