ppbhmd.com

#  robots.txt Updated by Vickie 7/3/2020
#    - Added enabled code so that only allows crawling from site domain name

# This file is to prevent the crawling and indexing of certain parts
# of your site by web crawlers and spiders run by sites like Yahoo!
# and Google. By telling these "robots" where not to go on your site,
# you save bandwidth and server resources.
#
# This file will be ignored unless it is at the root of your host:
# Used:    http://example.com/robots.txt
# Ignored: http://example.com/site/robots.txt
#
# For more information about the robots.txt standard, see:
# http://www.robotstxt.org/wc/robots.html
#
# For syntax checking, see:
# http://www.sxw.org.uk/computing/robots/check.html

# disallow all 
user-agent:*
Sitemap: http://www.premierpedsbh.com/sitemap.xml
Disallow: /
Crawl-delay: 600
# but allow only important bots
User-agent: Googlebot
User-agent: Googlebot-Image
User-agent: Mediapartners-Google
User-agent: msnbot
User-agent: msnbot-media
User-agent: Slurp
User-agent: Yahoo-Blogs
User-agent: Yahoo-MMCrawler
User-agent: BrokenLinkCheck
User-agent: Twitterbot
# Directories
Disallow: /app_code/
Disallow: /css/
Disallow: /js/
Disallow: /CMS_Modules/
Disallow: /app_data/
Disallow: /bin/
Disallow: /img/
Disallow: /wp/
# Files
Disallow: /CHANGELOG.txt
Disallow: /cron.php
Disallow: /INSTALL.mysql.txt
Disallow: /INSTALL.pgsql.txt
Disallow: /install.php
Disallow: /INSTALL.txt
Disallow: /LICENSE.txt
Disallow: /MAINTAINERS.txt
Disallow: /update.php
Disallow: /UPGRADE.txt
Disallow: /xmlrpc.php
Disallow: /hitechavs/concrete/controllers/profile/utf.php
Disallow: /CMSHelp/devguide/syndication_transformations.htm
Disallow: /wp-login.php
# Paths (clean URLs)
Disallow: /review/
Disallow: /review
Disallow: /admin/
Disallow: /comment/reply/
Disallow: /contact/
Disallow: /logout/
Disallow: /node/add/
Disallow: /search/
Disallow: /opensearch/
Disallow: /user/register/
Disallow: /user/password/
Disallow: /user/login/
Disallow: /wp-admin
Disallow: /Resources/Kid-Site-Videos/
Disallow: /Medical-Comprehensive/Kid-Site-Videos/
Disallow: /Kid-Site-Videos/
# Paths (no clean URLs)
Disallow: /?q=admin/
Disallow: /?q=comment/reply/
Disallow: /?q=contact/
Disallow: /?q=logout/
Disallow: /?q=node/add/
Disallow: /?q=search/
Disallow: /?q=user/password/
Disallow: /?q=user/register/
Disallow: /?q=user/login/

Crawl-delay: 600


# Should be Restricting Development or secondary domain names
# These 2 must match
# www.premierpedsbh.com
# http://www.premierpedsbh.com/robotstxt
# 
# True 
# and
# True
# -1 cannot include .remedyconnect.com
# true