thevideobeat.com

## robots.txt for Magento Community and Enterprise
 
## GENERAL SETTINGS
 
## Enable robots.txt rules for all crawlers

User-agent: *
 
## Crawl-delay parameter: number of seconds to wait between successive requests to the same server.
## Set a custom crawl rate if you're experiencing traffic problems with your server.
# Crawl-delay: 30
 

## Magento sitemap
Sitemap: http://www.thevideobeat.com/sm1.html
# Sitemap: http://www.thevideobeat.com/sm-phrases1.html
# Sitemap: http://www.thevideobeat.com/sitemap.xml

 

## DEVELOPMENT RELATED SETTINGS
 
## Do not crawl development files and folders: CVS, svn directories and dump files
Disallow: /CVS
Disallow: /*.svn$
Disallow: /*.idea$
Disallow: /*.sql$
Disallow: /*.tgz$
 

## GENERAL MAGENTO SETTINGS
 
## Do not crawl Magento admin page
Disallow: /admin/
 

## Do not crawl common Magento technical folders
# Disallow: /app/
Disallow: /downloader/
Disallow: /errors/
# Disallow: /includes/
# Disallow: /lib/
Disallow: /pkginfo/
### Disallow: /shell/
# Disallow: /var/


## Do not crawl common Magento files
Disallow: /api.php
Disallow: /cron.php
Disallow: /cron.sh
Disallow: /error_log
Disallow: /get.php
Disallow: /install.php
Disallow: /LICENSE.html
Disallow: /LICENSE.txt
Disallow: /LICENSE_AFL.txt
Disallow: /README.txt
Disallow: /RELEASE_NOTES.txt
 

## MAGENTO SEO IMPROVEMENTS
 
## Do not crawl sub category pages that are sorted or filtered.
Disallow: /*?dir*
Disallow: /*?dir=desc
Disallow: /*?dir=asc
Disallow: /*?limit=all
Disallow: /*?mode*
 

## Do not crawl 2-nd home page copy (example.com/index.php/). Uncomment it only if you activated Magento SEO URLs. Not sure what's happening here
# Disallow: /index.php/
 

## Do not crawl links with session IDs
Disallow: /*?SID=
 

## Do not crawl checkout and user account pages
Disallow: /checkout/
Disallow: /onestepcheckout/
Disallow: /customer/
Disallow: /customer/account/
Disallow: /customer/account/login/
 

## Do not crawl search pages and not-SEO optimized catalog links
Disallow: /catalogsearch/
Disallow: /catalogsearch/advanced/
Disallow: /catalog/product_compare/
# Disallow: /catalog/category/view/
# Disallow: /catalog/product/view/

 
## SERVER SETTINGS
 
## Do not crawl common server technical folders and files
# Disallow: /cgi-bin/
Disallow: /cleanup.php
Disallow: /apc.php
# Disallow: /memcache.php
# Disallow: /phpinfo.php
 


## Below is what I had been using before I found above on web.  Q: How does this play with GWT URL Parameters?  Should I remove parms from GWT?
# Many of these came from WebmasterTools reports
# Duplicate Content

Disallow: /store/
# Disallow: /contactus/
# Disallow: /cm-unsubscribe.html


# Disallow: /catalog/
# Disallow: /ordersendemail/
# Disallow: /wishlist.htm
# Disallow: /products-new.html
# Disallow: /review/
Disallow: /enable-cookies/
# Disallow: /sendfriend/
# Disallow: /wishlist/
Disallow: /checkout/
Disallow: /tag/list
Disallow: /tag/product/list/tagId/*
Disallow: /sales/
Disallow: /newsletter/
Disallow: /newsletters/
# Disallow: /landingpage/
# Disallow: /category/
Disallow: /_vti_bin/
# Disallow: /*product_info.php?
# Disallow: /*product
# Disallow: /*account_wishlist.php?
Disallow: /monster-teenage-movies/teenage-frankenstein-1957.html
Disallow: /monster-teenage-movies/teenage-werewolf-1957.html
Disallow: /music-documentaries/red-dog-saloon.html
# Disallow: /pp-payments.htm

###########################################

# Disallow: /rock-roll-tv/happening-68-volume1.html
# Disallow: /rock-roll-tv/happening-68-volume2.html
# Disallow: /rock-roll-tv/happening-68-volume3.html
# Disallow: /rock-roll-tv/american-bandstand-1957.html
# Disallow: /rock-roll-tv/ab-64.html
# Disallow: /rock-roll-tv/ab-64-vol2.html
# Disallow: /rock-roll-tv/ab-66-vol1.html
# Disallow: /rock-roll-tv/ab-66-vol2.html
# Disallow: /rock-roll-tv/ab-67-volume-1.html
# Disallow: /rock-roll-tv/ab-67-volume-2.html
# Disallow: /rock-roll-tv/ab-67-volume-3.html
# Disallow: /rock-roll-tv/ab-68-vol1.html
# Disallow: /rock-roll-tv/beech-nut-show-1959-vol-1.html
# Disallow: /rock-roll-tv/birth-of-beatles-1979.html
# Disallow: /rock-roll-tv/tami-show-1964.html
# Disallow: /rock-roll-tv/wtai-pilot-episode-1965.html
# Disallow: /rock-roll-tv/where-action-is-1966-vol-1.html
# Disallow: /rock-roll-tv/action-vol-2.html
# Disallow: /rock-roll-tv/wtai-vol-3-1966.html
# Disallow: /rock-roll-tv/wtai-volume-4-1966.html
# Disallow: /rock-roll-tv/wtai-1966-volume-5.html
# Disallow: /rock-roll-tv/wtai-1966-volume-6.html
# Disallow: /rock-roll-tv/wtai-1966-vol-7.html
# Disallow: /rock-roll-tv/wtai-1966-vol-8.html
# Disallow: /rock-roll-tv/wtai-1966-vol-9.html

###########################################




Disallow: /jd-biker-hot-rod-movies/george-barris-kustom-cars-1950s-1060s.html	
Disallow: /mod-movies/record-city-1977.html
Disallow: /mod-movies/its-all-over-town-1963.html
Disallow: /mod-movies/playgirl-killer-1966.html
Disallow: /rock-roll-tv/big-tnt-show-1966.html
Disallow: /mod-movies/lineup-1958.html
Disallow: /music-documentaries/shakin-stevens-vol-1-2-3.html
Disallow: /mod-movies/early-rural-music.html
Disallow: /movies/tami_show_t-a-m-i_show_dvd.htm
Disallow: /movies/james_brown_boston_garden_1968.htm
Disallow: /movies/hot_rods_to_hell_dvd.htm
Disallow: /rock-roll-tv/mulqueens-kiddie-a-go-go.html
Disallow: /rock-roll-tv/kiddie-a-go-go.html

###### Bad Neighborhood

Disallow: /images/old/fgzx/uiyb.php?wild-teen-movies