# Disallow: The value of this field specifies a partial URL that is not to be visited. This can be a full path, # or a partial path; any URL that starts with this value will not be retrieved. For example, # Disallow: /help # disallows both /help.html and /help/index.html, whereas # Disallow: /help/ # would disallow /help/index.html but allow /help.html. # Disallow ALL crawlers, bots, robots, spiders, etc from archiving the Pics directory and sub-directories # and all pages starting with pics User-agent: * Disallow: /pics Disallow: /guest Disallow: /form # Disallow ALL crawlers from the entire site # User-agent: * # Disallow: / # See http://www.almaden.ibm.com/cs/crawler/ for some good information on encoding this file