# $Id: robots.txt,v 1.1 1997/10/27 09:00:00 fielding Exp $
# robots.txt for http://www.colby.edu/

#--- this section is for the Colby search engine:
User-agent: ColbySpider
Disallow: /cgi-bin/
Disallow: /info.tech/stats/

#--- this section is for all other search engines:
User-agent: *                   # All other spiders should avoid 
Disallow: *			# No indexing of the author server allowed
#Disallow: /cgi-bin/             #     Script files
#Disallow: /focus/             #     nonextant club
#Disallow: /cpa/             #     unfinished club
#Disallow: /security/             #     unfinished dept.
#Disallow: /college/             #     internal
#Disallow: /i/
#Disallow: /info.tech/stats/
#Disallow: /classof/2001.summer/
#Disallow: /faculty/
#Disallow: /colby.mag/issues/2004/spring/alumni/		#	Spring 2004 Magazine | Alumni Section