# $Id: robots.txt,v 1.4 2002/05/14 18:34:09 fiorello Exp $
# robots.txt for http://www.ics.uci.edu/
# see <http://web.nexor.co.uk/mak/doc/robots/norobots.html> for an explanation.

User-agent: MOMspider           # The Multi-Owner Maintenance Spider
Disallow: /cgi-bin/             #     Script files
Disallow: /Admin/MOM/           #     Local MOMspider output
Disallow: /~fielding/MOM/       #     Local MOMspider output
Disallow: /TR/                  #     Dienst Technical Report Server
Disallow: /Server/              #     Dienst Technical Report Server
Disallow: /Document/            #     Dienst Technical Report Server
Disallow: /MetaServer/          #     Dienst Technical Report Server
Disallow: /~eppstein/pubs/cites/          #     Eppstein Database
Disallow: /~fiorello/pvt/       #     Private pages

User-agent: *                   # All other spiders should avoid 
Disallow: /cgi-bin/             #     Script files
Disallow: /Test/                #     The test area for web experimentation
Disallow: /Admin/               #     Huge server statistic logs
Disallow: /TR/                  #     Dienst Technical Report Server
Disallow: /Server/              #     Dienst Technical Report Server
Disallow: /Document/            #     Dienst Technical Report Server
Disallow: /MetaServer/          #     Dienst Technical Report Server
Disallow: /~fielding/MOM/       #     Local MOMspider output
Disallow: /~kanderso/hidden     #     Ken Anderson's stuff
Disallow: /~eppstein/pubs/cites/          #     Eppstein Database
Disallow: /~fiorello/pvt/       #     Private pages
Disallow: /~dean/
Disallow: /~wwwoffic/
Disallow: /~ucounsel/
Disallow: /~sao/
Disallow: /~support/
Disallow: /~icsdb/
Disallow: /bin/