vax.co.nz

# $Id: robots.txt,v magento-specific 2010/28/01 18:24:19 goba Exp $

#

# robots.txt

#

# This file is to prevent the crawling and indexing of certain parts

# of your site by web crawlers and spiders run by sites like Yahoo!

# and Google. By telling these “robots” where not to go on your site,

# you save bandwidth and server resources.

#

# This file will be ignored unless it is at the root of your host:

# Used:  http://example.com/robots.txt

# Ignored: http://example.com/site/robots.txt

#

# For more information about the robots.txt standard, see:

# http://www.robotstxt.org/wc/robots.html

#

# For syntax checking, see:

# http://www.sxw.org.uk/computing/robots/check.html

Sitemap: http://www.vax.co.uk/sitemaps/sitemap.xml

Sitemap: http://www.vax.com.au/sitemaps/au/sitemap.xml



# Crawlers Setup

User-agent: *

Crawl-delay: 30

# Allowable Index

Allow: /*?p=

Allow: /index.php/blog/

Allow: /catalog/seo_sitemap/category/

# Directories

Disallow: /404/

Disallow: /app/

Disallow: /cgi-bin/

Disallow: /downloader/

Disallow: /errors/

Disallow: /includes/

Disallow: /lib/

Disallow: /magento/

Disallow: /pkginfo/

Disallow: /report/

Disallow: /scripts/

Disallow: /shell/

#Disallow: /skin/

Disallow: /stats/

Disallow: /var/

# Paths (clean URLs)

Disallow: /index.php/

Disallow: /catalog/product_compare/

Disallow: /catalog/category/view/

Disallow: /catalog/product/view/

Disallow: /catalogsearch/

Disallow: /checkout/

Disallow: /control/

Disallow: /contacts/

Disallow: /customer/

Disallow: /customize/

Disallow: /poll/

Disallow: /review/

Disallow: /search/

Disallow: /sendfriend/

Disallow: /tag/

Disallow: /wishlist/

# Files

Disallow: /cron.php

Disallow: /cron.sh

Disallow: /error_log

Disallow: /install.php

Disallow: /LICENSE.html

Disallow: /LICENSE.txt

Disallow: /LICENSE_AFL.txt

Disallow: /STATUS.txt

# Paths (no clean URLs)

Disallow: /*.php$

Disallow: /*?p=*&

Disallow: /*?SID=

Disallow: /*?limit=all

# Turpentine

Disallow: /turpentine/

Disallow: /air-cordless-solo-upright-vacuum-cleaner



# Uncomment if you do not wish for Google to index your images

#User-agent: Googlebot-Image

#Disallow: /