User-agent: *
Disallow: /comments/ # Blocca la directory /private_directory/
Disallow: /backyard/
Disallow: /forum/
Disallow: /member/
Disallow: /servizi/
Disallow: /blogs/servizi/print.aspx
Disallow: /app.aspx
Disallow: /maps
Disallow: /ScriptResource.axd
Disallow: /WebResource.axd
Disallow: /post/2135572.html
Crawl-delay: 10
Request-rate: 1/5 # Visita al massimo una pagina ogni 5 secondi
Visit-time: 0100-0600 # Vista soltanto tra le 1:00 AM e le 5:00 AM UT (GMT)
User-agent: googlebot
Disallow: /comments/
Disallow: /backyard/
Request-rate: 1/5 # Visita al massimo una pagina ogni 5 secondi
Visit-time: 0100-0600 # Vista soltanto tra le 1:00 AM e le 5:00 AM UT (GMT)
User-agent: psbot
Disallow: /
User-agent: Exabot
Disallow: /
User-agent: exabot
Disallow: /
User-agent: seekbot
Disallow: /
User-agent: Echo
Disallow: /
User-agent: Slurp
Crawl-delay: 5
User-agent: twiceler
Disallow: /