User-Agent: *
Disallow: # empty Disallow instruction SHOULD be there
Crawl-delay: 10
Disallow: /flash/
Disallow: /bitrix/
Disallow: /about-bitrix/
Disallow: /auth/
Disallow: /communication/
Disallow: /english/
Disallow: /forms/
Disallow: /gosserv/
Disallow: /includes/
Disallow: /informers/
Disallow: /personal/
Disallow: /vote/
Disallow: /upload/
Disallow: /forum/
Disallow: /images/
Disallow: /data/
Disallow: /desktop_app/
Disallow: /documents/
Disallow: /photo/
Disallow: /js/
Disallow: /search/
Disallow: /video/
Disallow: /zip/
Disallow: /municipal-inspection/photofact-violations.php
Disallow: /municipal-list/department-of-culture/city-contest-christmas-in-your-window.php
Sitemap: http://shakhty-gorod.ru/sitemap.xml
User-Agent: Yandex
Disallow: # empty Disallow instruction SHOULD be there
Crawl-delay: 10
Disallow: /bitrix
Disallow: /about-bitrix
Disallow: /DAiG
Disallow: /desktop_app
Disallow: /forum
Disallow: /js
Disallow: /photo
Disallow: /photovideogallery
Disallow: /upload
Disallow: /popular control
Disallow: Clean-param: forgot_password&login /auth/index.php
User-Agent: Googlebot
Disallow: # empty Disallow instruction SHOULD be there
Crawl-delay: 10
Disallow: /bitrix
Disallow: /about-bitrix
Disallow: /DAiG
Disallow: /desktop_app
Disallow: /forum
Disallow: /js
Disallow: /photo
Disallow: /photovideogallery
Disallow: /upload
Disallow: /popular control
User-Agent: Applebot
Disallow: # empty Disallow instruction SHOULD be there
Disallow: /
User-Agent: BLEXBot
Disallow: # empty Disallow instruction SHOULD be there
Disallow: /
User-Agent: MJ12bot
Disallow: # empty Disallow instruction SHOULD be there
Disallow: /
User-Agent: MegaIndex.ru
Disallow: # empty Disallow instruction SHOULD be there
Disallow: /
User-Agent: SputnikBot
Disallow: # empty Disallow instruction SHOULD be there
Disallow: /
User-Agent: SemrushBot
Disallow: # empty Disallow instruction SHOULD be there
Disallow: /
User-Agent: bingbot
Disallow: # empty Disallow instruction SHOULD be there
Disallow: /
User-Agent: DotBot
Disallow: # empty Disallow instruction SHOULD be there
Disallow: /
User-Agent: AhrefsBot
Disallow: # empty Disallow instruction SHOULD be there
disallow: /
User-Agent: Slurp
Disallow: # empty Disallow instruction SHOULD be there
Disallow: /