# This file set up by andersja according to proposed Standard of Robot
# Exclusion at http://web.nexor.co.uk/mak/doc/robots/norobots.html
#
# created 1997-03-01 16:00
# updated 2003-12-22 10:50
# updated 2006-04-11 added hosts
# updated 2006-04-19 added priv
#
# Currently: allow all well-behaved robots.
#
# (An empty 'Disallow' line, looking like this:)
# User-agent: *              # Means: All robots.
# Disallow:                  # Means: Disallow nothing.
#
#

# http://www.webmasterworld.com/robots.txt has a long list of active
# robots you might want to block. 
# Some of these (and many others) ignore robots.txt, and are forcibly 
# blocked in .htaccess.
# (see also 
# http://diveintomark.org/archives/2003/02/26/how_to_block_spambots_ban_spybots_and_tell_unwanted_robots_to_go_to_hell )

User-agent: MSRBOT
User-agent: Haste
User-agent: InfoNaviRobot
User-agent: MarcoPolo
User-agent: Nutch
User-agent: Zao
User-agent: semanticdiscovery
User-agent: PubCrawl
User-agent: TurnitinBot
User-agent: NPbot
User-agent: psbot
User-agent: baiduspider
User-agent: larbin
User-agent: ia_archiver
User-agent: NationalDirectory
User-agent: LNSpiderguy
User-agent: Teleport
User-agent: MIIxpc
User-agent: asterias
User-agent: lwp-trivial
User-agent: LinkWalker
User-agent: cosmos
User-agent: MSIECrawler
User-agent: sitecheck.internetseer.com
User-agent: pompos
User-agent: Generic
User-agent: WebSearchBench
User-agent: almaden
User-agent: k2spider
User-agent: curl
User-agent: Wget
User-agent: UbiCrawler
Disallow: /

# Rover is a bad dog
# Jfr. http://www.roverbot.com/user/baddog.html
User-agent: Roverbot
Disallow: /

User-agent: *
Disallow: /src/
Disallow: /anders/src/
Disallow: /weather/src/
Disallow: /stats/
Disallow: /hosts/
Disallow: /priv/