#User-agent: *
#Disallow: /     #disallow indexing of all pages
#
# Revised  robots.text file for whole of blackandwhite.org nov23/98
# This tells scanning robots where they are and are not welcome
# User-agent: can also specify by name; "*" is for all robots
# Disallow: if this matches first part of the requested path, forget
# it. 
# tardis.ed.ac.uk/home/sxw/robots/check/
# this site checks syntax errors on this file
# Unlock this exclusion before launch, exclude files by directory
# name before rollout
#zdnet.com/devhead/resources/tools/htmlcheck/
#this checks links,html etc.
#If you want to exclude all robots from the entire
#server, use the following syntax: 
#
#User-agent: *
#Disallow: /
#
User-agent: *
Disallow: /Shellx/
Disallow: /incoming/
Disallow: /listat/
Disallow: /inbox/
Disallow: /pico.save/
Disallow: /tmp
# directories below pertain to Savvy unused
Disallow: /allopad
Disallow: /doctors
Disallow: /holopad
Disallow: /members
Disallow: /newstand
Disallow: /patipad
Disallow: /pred
Disallow: /outlaw
#
# first robots.text file for whole of blackandwhite.org 10/31/98
# This tells scanning robots where they are and are not welcome
# User-agent: can also specify by name; "*" is for all robots
# Disallow: if this matches first part of the requested path, forget
# it. 
# tardis.ed.ac.uk/home/sxw/robots/check/
# this site checks syntax errors on this file
# Unlock this excluison before launch, exclude files by directory
# name before rollout
#
#User-agent: *   #applies to all robots
#Disallow: /	#disallow indexing of all pages
# Tell the Internet Archive not to keep a copy
User-agent: ia_archiver
Disallow: /
#
User-agent: ia_archiver/1.6
Disallow: /
#
User-agent: EmailSiphon
Disallow: /
#
User-agent: Gulliver/1.2
Disallow: /
#
# Keep out Tovek Tools
User-agent: TovekTools Web Indexer/1.0beta [Win32]
Disallow: /
#
User-agent: InfoNaviRobot
Disallow: /
#
# robots.txt --- index my site for search but don't archive it
#User-agent: *
#Disallow: 
#
User-agent: HURIDOCS  
Disallow: /
#
User-agent: Openfind 
Disallow: /
#
User-agent: EmailWolf 
Disallow: /
#
User-agent: ExtractorPro
Disallow: /
#
User-agent: Harvest/1.5
Disallow: /
#
#
#
#If you want to exclude all robots from specific
#areas or folders on the server, you'll need to
#specify the excluded folders: 
#
#User-agent: *
#Disallow: /cgi-bin/
#Disallow: /dev/
#
#The "*" in the User-agent directive has the
#special value of "any or all robots". 
#To allow a single robot complete access and
#exclude all others, use a syntax like the
#following: 
#
#User-agent: Lycos
#Disallow:
#User-agent: *
#Disallow: /
#
#User-agent: *
#Disallow: /cgi-bin/
#Disallow: /dev/
#Disallow: /membersonly.shtml/
#
#server are excluded: cgi-bin, a live development folder
#called "dev" and the html page, "membersonly.shtml".
#You'll need to establish a separate "Disallow" directive
#for each path you want to exclude. Be careful not to
#include any empty lines in the text file, as the bots will
#intepret this as though you were referring to multiple
#sites. And, you can't use the asterix anywhere else
#within this text file besides within the user-agent field. 
#
#You'll need to know the exact name of the robot agents
#in order to exclude certain bots by name.