#
# See http://www.robotstxt.org/wc/norobots.html for documentation on how to use the robots.txt file
# See http://www.searchtools.com/robots/robots-txt.html for search engine extensions
#
# See:
# http://googlewebmastercentral.blogspot.com/2008/06/improving-on-robots-exclusion-protocol.html
# for a nice summary of the REP (Robots Exclusion Protocol) and relevant META tags
#
# To ban all spiders from the entire site uncomment the next two lines:
# User-Agent: *
# Disallow: /
#
# User-agent: *
# Disallow: /*page=
# Disallow: /users/*/messages/new

# ticket #162 : prevent "Googlebot" from spidering paginated pages on the site
# ticket #170 : prevent "Googlebot" from spidering meaningless pages on the site
# Google-specific wildcard extensions documented on the following pages:
# http://www.google.com/support/webmasters/bin/answer.py?answer=76401&topic=8843
User-Agent: Googlebot
Disallow: /*page=
Disallow: /users/*/messages/new

# ticket #162 : prevent "Yahoo! Slurp" from spidering paginated pages on the site
# ticket #170 : prevent "Yahoo! Slurp" from spidering meaningless pages on the site
# Google-specific wildcard extensions documented on the following pages:
# http://help.yahoo.com/l/us/yahoo/search/webcrawler/slurp-02.html
User-Agent: Slurp
Disallow: /*page=
Disallow: /users/*/messages/new

# ticket #162 : prevent "M$ msnbot" from spidering paginated pages on the site
# ticket #170 : prevent "M$ msnbot" from spidering meaningless pages on the site
# Google-specific wildcard extensions documented on the following pages:
# http://help.live.com/help.aspx?project=wl_webmasters&querytype=keyword&query=txtstobor&mkt=en-US
User-Agent: msnbot
Disallow: /*page=
Disallow: /users/*/messages/new

# ticket #162 : prevent "MJ12bot" from spidering paginated pages on the site
# ticket #170 : prevent "MJ12bot" from spidering meaningless pages on the site
# http://www.majestic12.co.uk/projects/dsearch/mj12bot.php#BlockBot
# NOTE the wildcard extensions may or may not be implemented by MJ12bot, so
# the following may or may not prevent the MJ12bot from spidering the site!
User-Agent: MJ12bot
Disallow: /*page=
Disallow: /users/*/messages/new

# ticket #162 : prevent "ia_archiver" from spidering paginated pages on the site
# ticket #170 : prevent "ia_archiver" from spidering meaningless pages on the site
# http://www.alexa.com/help/webmasters
# NOTE the wildcard extensions may or may not be implemented by ia_archiver, so
# the following may or may not prevent the ia_archiver from spidering the site!
User-Agent: ia_archiver
Disallow: /*page=
Disallow: /users/*/messages/new