# as created on 13 Sept 2003 the current online robots.txt 
# of http://www.human-highway.org/robots.txt
# is valid
# see http://www.searchengineworld.com/cgi-bin/robotcheck.cgi
#  or http://tool.motoricerca.info/robots-checker.phtml
#  or http://www.frobee.com/robots-txt-check

# to allow only Google to spider everything, we would write:
# User-agent: GoogleBot
# Disallow:
# or now this:
# User-agent: GoogleBot
# Allow: /
# but we only allow Google for certain directories
# but ban the rest.
# 22 August 2011, hope, his works. 

# 17 Jan 2014: allow Google, DuckDuckBot IArchiver

# 23 August 2011 - turned the sequence around, now Googlebot first again; 
# allow Googlebot
User-agent: Googlebot
Disallow: /css/
Disallow: /bh/
Disallow: /cgi-bin/
Disallow: /bin/
Disallow: /java/
Disallow: /sounds/
Disallow: /videos/
Disallow: /private/
Disallow: /error_docs/
Disallow: /logs/
Disallow: /stats/
Disallow: /contact/
Disallow: /lyrics/
Disallow: /images/
Disallow: /thingie/
Disallow: /wp-admin/
Disallow: /wp-includes/
Disallow: /wp-content/plugins
Disallow: /wp-content/cache
Disallow: /wp-content/themes

# allow DuckDuckBot
User-agent: DuckDuckBot
Disallow: /css/
Disallow: /bh/
Disallow: /cgi-bin/
Disallow: /bin/
Disallow: /java/
Disallow: /sounds/
Disallow: /videos/
Disallow: /private/
Disallow: /error_docs/
Disallow: /logs/
Disallow: /stats/
Disallow: /contact/
Disallow: /lyrics/
Disallow: /images/
Disallow: /thingie/
Disallow: /wp-admin/
Disallow: /wp-includes/
Disallow: /wp-content/plugins
Disallow: /wp-content/cache
Disallow: /wp-content/themes

# allow Wayback machine IArchiver
User-agent: ia_archiver
Disallow: /css/
Disallow: /bh/
Disallow: /cgi-bin/
Disallow: /bin/
Disallow: /java/
Disallow: /sounds/
Disallow: /videos/
Disallow: /private/
Disallow: /error_docs/
Disallow: /logs/
Disallow: /stats/
Disallow: /contact/
Disallow: /lyrics/
Disallow: /images/
Disallow: /thingie/
Disallow: /wp-admin/
Disallow: /wp-includes/
Disallow: /wp-content/plugins
Disallow: /wp-content/cache
Disallow: /wp-content/themes

# ban all the rest
User-agent: *
Disallow: /