mds-partner.com

## GENERAL SETTINGS
###
## Enable robots.txt rules for all crawlers
User-agent: *

## Crawl-delay parameter: number of seconds to wait between successive requests to the same server.
## Set a custom crawl rate if you're experiencing traffic problems with your server.
#Crawl-delay: 5

## DEVELOPMENT RELATED SETTINGS

## Do not crawl development files and folders: CVS, svn directories and dump files
Disallow: /CVS
Disallow: /*.svn$
Disallow: /*.idea$
Disallow: /*.sql$
Disallow: /*.tgz$
Disallow: /*.git$


## GENERAL MAGENTO SETTINGS

## Do not crawl common Magento technical folders
Disallow: /shop/_aitoc_filelists/
Disallow: /shop/404/
Disallow: /shop/app/
Disallow: /shop/coupons_import_csv/
Disallow: /shop/db/
Disallow: /shop/default_js/
Disallow: /shop/downloader/
Disallow: /shop/errors/
Disallow: /shop/import/
Disallow: /shop/includes/
Disallow: /shop/js/
Disallow: /shop/lib/
Disallow: /shop/media/
Disallow: /shop/shell/
Disallow: /shop/skin/
Disallow: /shop/var/
Disallow: /shop/junixx/


Allow: /shop/media/catalog/product/

## Do not crawl common Magento files
Disallow: /shop/api.php
Disallow: /shop/cron.php
Disallow: /shop/cron.sh
Disallow: /shop/error_log
Disallow: /shop/get.php
Disallow: /shop/install.php
Disallow: /shop/LICENSE.html
Disallow: /shop/LICENSE.txt
Disallow: /shop/LICENSE_AFL.txt
Disallow: /shop/README.txt
Disallow: /shop/RELEASE_NOTES.txt

## MAGENTO SEO IMPROVEMENTS

## Do not crawl sub category pages that are sorted or filtered.
Disallow: */*?
Disallow: */*?dir*
Disallow: */*?dir=desc
Disallow: */*?dir=asc
Disallow: */*?limit=all
Disallow: */*?mode*
Disallow: */*?p=*&

# Paths (no clean URLs)
Disallow: /*.gif$
Disallow: /*.js$
Disallow: /*.css$
Disallow: /*.pdf$
Disallow: /*.php$

## Do not crawl links with session IDs
Disallow: */*?SID=

Disallow: /shop/index.php/

## Do not crawl checkout and user account pages
Disallow: /shop/checkout/
Disallow: /shop/onestepcheckout/
Disallow: /shop/customer/
Disallow: /shop/customer/account/
Disallow: /shop/customer/account/login/

## Do not crawl search pages and not-SEO optimized catalog links
Disallow: /shop/catalog/
Disallow: /shop/catalog/product_compare/
Disallow: /shop/catalog/category/view/
Disallow: /shop/catalog/product/view/
Disallow: /shop/catalogsearch/
Disallow: /shop/control/
Disallow: /shop/contacts/
Disallow: /shop/customer/
Disallow: /shop/customize/
Disallow: /shop/newsletter/
Disallow: /shop/poll/
Disallow: /shop/review/
Disallow: /shop/sendfriend/
Disallow: /shop/tag/
Disallow: /shop/wishlist/
Disallow: /shop/catalog/product/gallery/


# ===================================
# Schliesse folgende Spider komplett aus:
# ===================================

User-agent: yandex
Disallow: /

User-agent: exabot
Disallow: /

User-agent: gigabot
Disallow: /

User-agent: ia_archiver
Disallow: /

User-agent: baiduspider
Disallow: /

User-agent: nutch
Disallow: /

User-agent: cityreview
Disallow: /

User-agent: WebReaper
Disallow: /

User-agent: WebCopier
Disallow: /

User-agent: Offline Explorer
Disallow: /

User-agent: HTTrack
Disallow: /

User-agent: Microsoft.URL.Control
Disallow: /

User-agent: EmailCollector
Disallow: /

User-agent: penthesilea
Disallow: /