#  www.dreamscapemultimedia.com
#  Dreamscape Multimedia

#  FULL access (Acoon)
User-agent: Acoon Robot
Disallow: 

#  FULL access (AllThatNet)
User-agent: ATN_Worldwide
Disallow: 

#  FULL access (Alta Vista)
User-agent: Scooter
Disallow: 

#  FULL access (Anzwers)
User-agent: AnzwersCrawl
Disallow: 

#  FULL access (AustLII)
User-agent: Gromit
Disallow: 

#  FULL access (BizBot04 )
User-agent: kirk.overleaf.com
Disallow: 

#  FULL access (Bizbot003)
User-agent: 
Disallow: 

#  FULL access (CMC/0.01)
User-agent: CMC/0.01
Disallow: 

#  FULL access (CaliforniaBrownSpider)
User-agent: 
Disallow: 

#  FULL access (CyberSpyder )
User-agent: (amant@www.cyberspyder.com)
Disallow: 

#  FULL access (Direct Hit Grabber)
User-agent: grabber
Disallow: 

#  FULL access (EI*Net/0.1  )
User-agent: libwww/0.1
Disallow: 

#  FULL access (EnigmaBot)
User-agent: 
Disallow: 

#  FULL access (Entire Web)
User-agent: speedy
Disallow: 

#  FULL access (EuroSeek)
User-agent: Freecrawl
Disallow: 

#  FULL access (Excite)
User-agent: ArchitextSpider
Disallow: 

#  FULL access (FAST/AllTheWeb)
User-agent: FAST-WebCrawler
Disallow: 

#  FULL access (Fireball)
User-agent: KIT-Fireball
Disallow: 

#  FULL access (Goo)
User-agent: moget
Disallow: 

#  FULL access (Google)
User-agent: Googlebot
Disallow: 

#  FULL access (Google Image)
User-agent: Googlebot-Image
Disallow: 

#  FULL access (Griffon)
User-agent: griffon
Disallow: 

#  FULL access (HappyBot )
User-agent: (gserver.kw.net)
Disallow: 

#  FULL access (Hämähäkki)
User-agent: Hämähäkki
Disallow: 

#  FULL access (ITI Spider)
User-agent: 
Disallow: 

#  FULL access (Ibot/1.0  )
User-agent: libwww-perl/0.40
Disallow: 

#  FULL access (Industry Central)
User-agent: Open Text Site Crawler
Disallow: 

#  FULL access (InfoSeek.de)
User-agent: marvin
Disallow: 

#  FULL access (Ingrid)
User-agent: INGRID/0.1
Disallow: 

#  FULL access (Inktomi)
User-agent: Slurp
Disallow: 

#  FULL access (Internet Cruiser)
User-agent: Internet Cruiser Robot
Disallow: 

#  FULL access (Kvasir)
User-agent: solbot
Disallow: 

#  FULL access (Legs)
User-agent: legs
Disallow: 

#  FULL access (Lets Find It Now!)
User-agent: elfinbot
Disallow: 

#  FULL access (Libertech-Rover)
User-agent: www.libertech.com?
Disallow: 

#  FULL access (Lisa)
User-agent: Voyager/0.0
Disallow: 

#  FULL access (Lycos)
User-agent: Lycos_Spider_(T-Rex)
Disallow: 

#  FULL access (MPRM Group Limited)
User-agent: spider_monkey
Disallow: 

#  FULL access (Merritt/1.0)
User-agent: 
Disallow: 

#  FULL access (Mirago)
User-agent: mirago
Disallow: 

#  FULL access (MyCNNSpider)
User-agent: 
Disallow: 

#  FULL access (NetScoop)
User-agent: NetScoop
Disallow: 

#  FULL access (Northern Light)
User-agent: Gulliver
Disallow: 

#  FULL access (ODiN)
User-agent: Valkyrie libwww-perl
Disallow: 

#  FULL access (OGspider)
User-agent: 
Disallow: 

#  FULL access (Openfind)
User-agent: openbot
Disallow: 

#  FULL access (PlanetSearch)
User-agent: fido
Disallow: 

#  FULL access (Portal Juice)
User-agent: pjspider
Disallow: 

#  FULL access (SandBot)
User-agent: 
Disallow: 

#  FULL access (SearchUK)
User-agent: MegaSheep
Disallow: 

#  FULL access (SlowBot)
User-agent: 
Disallow: 

#  FULL access (speedy)
User-agent: Speedy
Crawl-Delay: 7
Disallow: /secret
Disallow: /search?

#  FULL access (StatFetcher/1.0)
User-agent: 
Disallow: 

#  FULL access (Suchmaschine21)
User-agent: CoolBot
Disallow: 

#  FULL access (SummyCrawler)
User-agent: 
Disallow: 

#  FULL access (Surfbot)
User-agent: 
Disallow: 

#  FULL access (TeacherSoft/1.0 )
User-agent: libwww/2.17
Disallow: 

#  FULL access (Thunderstone)
User-agent: T-H-U-N-D-E-R-S-T-O-N-E
Disallow: 

#  FULL access (TopicLink)
User-agent: topiclink
Disallow: 

#  FULL access (VietGATE)
User-agent: jcrawler
Disallow: 

#  FULL access (WWW Collector)
User-agent: 
Disallow: 

#  FULL access (WebWalker)
User-agent: 
Disallow: 

#  FULL access (WhoWhere Robot)
User-agent: 
Disallow: 

#  FULL access (WhoWhere?)
User-agent: whowhere
Disallow: 

#  FULL access (e-collector)
User-agent: ecollector
Disallow: 

#  FULL access (heraSpider)
User-agent: 
Disallow: 

#  FULL access (iaNett.com)
User-agent: ParaSite
Disallow: 

#  FULL access (kensaku.jp)
User-agent: suke
Disallow: 

#  FULL access (linklooker)
User-agent: 
Disallow: 

#  FULL access (mopilot.com)
User-agent: wapspider
Disallow: 

#  FULL access (nathan)
User-agent: tarantula
Disallow: 

#  FULL access (newscan)
User-agent: newscan-online
Disallow: 

#  FULL access (processor/0.0ALPHA )
User-agent: libwww-perl/0.20
Disallow: 

#  FULL access (spyder3.microsys.com)
User-agent: 
Disallow: 

#  FULL access (w3index)
User-agent: 
Disallow: 

#  FULL access (whatUseek)
User-agent: winona
Disallow: 

#  FULL access (wobot/1.0 )
User-agent: from 206.214.202.45
Disallow: 

#  FULL access (www.freeloader.com.)
User-agent: 
Disallow: 

#  FULL access (All Spiders)
User-agent: *
Disallow: 

BizBot04 kirk.overleaf.com
HappyBot (gserver.kw.net)
CaliforniaBrownSpider
EI*Net/0.1  libwww/0.1
Ibot/1.0 libwww-perl/0.40    
Merritt/1.0
StatFetcher/1.0
TeacherSoft/1.0  libwww/2.17
WWW Collector
processor/0.0ALPHA libwww-perl/0.20
wobot/1.0 from 206.214.202.45
Libertech-Rover         www.libertech.com?
WhoWhere Robot
ITI Spider
w3index
MyCNNSpider
SummyCrawler
OGspider
linklooker
CyberSpyder (amant@www.cyberspyder.com)
SlowBot
heraSpider
Surfbot
Bizbot003
WebWalker
SandBot
EnigmaBot
spyder3.microsys.com
www.freeloader.com.

205.252.60.71
194.20.32.131
198.5.209.201
acke.dc.luth.se
dallas.mt.cs.cmu.edu
darkwing.cadvision.com
waldec.com
www2000.ogsm.vanderbilt.edu
unet.ca
murph.cais.net (rapid fire... sigh)
spyder3.microsys.com
www.freeloader.com.

# ABCdatos BotLink
# Acme.Spider
# Ahoy! The Homepage Finder
# Alkaline
# Anthill
# Walhello appie
# Arachnophilia
# Arale
# Araneo
# AraybOt
# ArchitextSpider
# Aretha
# ARIADNE
# arks
# ASpider (Associative Spider)
# ATN Worldwide
# Atomz.com Search Robot
# AURESYS
# BackRub
# unnamed
# BBot
# Big Brother
# Bjaaland
# BlackWidow
# Die Blinde Kuh
# Bloodhound
# Borg-Bot
# BoxSeaBot
# bright.net caching robot
# BSpider
# CACTVS Chemistry Spider
# Calif
# Cassandra
# Digimarc Marcspider/CGI
# Checkbot
# ChristCrawler.com
# churl
# cIeNcIaFiCcIoN.nEt
# CMC/0.01
# Collective
# Combine System
# ConfuzzledBot
# CoolBot
# Web Core / Roots
# XYLEME Robot
# Internet Cruiser Robot
# Cusco
# CyberSpyder Link Test
# CydralSpider
# Desert Realm Spider
# DeWeb(c) Katalog/Index
# DienstSpider
# Digger
# Digital Integrity Robot
# Direct Hit Grabber
# DNAbot
# DownLoad Express
# DragonBot
# DWCP (Dridus' Web Cataloging Project)
# e-collector
# EbiNess
# EIT Link Verifier Robot
# ELFINBOT
# Emacs-w3 Search Engine
# ananzi
# esculapio
# Esther
# Evliya Celebi
# nzexplorer
# FastCrawler
# Fluid Dynamics Search Engine robot
# Felix IDE
# Wild Ferret Web Hopper #1, #2, #3
# FetchRover
# fido
# Hämähäkki
# KIT-Fireball
# Fish search
# Fouineur
# Robot Francoroute
# Freecrawl
# FunnelWeb
# gammaSpider, FocusedCrawler
# gazz
# GCreep
# GetBot
# GetURL
# Golem
# Googlebot
# Grapnel/0.01 Experiment
# Griffon
# Gromit
# Northern Light Gulliver
# Gulper Bot
# HamBot
# Harvest
# havIndex
# HI (HTML Index) Search
# Hometown Spider Pro
# Wired Digital
# ht://Dig
# HTMLgobble
# Hyper-Decontextualizer
# iajaBot
# IBM_Planetwide
# Popular Iconoclast
# Ingrid
# Imagelock
# IncyWincy
# Informant
# InfoSeek Robot 1.0
# Infoseek Sidewinder
# InfoSpiders
# Inspector Web
# IntelliAgent
# I, Robot
# Iron33
# Israeli-search
# JavaBee
# JBot Java Web Robot
# JCrawler
# AskJeeves
# JoBo Java Web Robot
# Jobot
# JoeBot
# The Jubii Indexing Robot
# JumpStation
# image.kapsi.net
# Katipo
# KDD-Explorer
# Kilroy
# KO_Yappo_Robot
# LabelGrabber
# larbin
# legs
# Link Validator
# LinkScan
# LinkWalker
# Lockon
# logo.gif Crawler
# Lycos
# Mac WWWWorm
# Magpie
# marvin/infoseek
# Mattie
# MediaFox
# MerzScope
# NEC-MeshExplorer
# MindCrawler
# mnoGoSearch search engine software
# moget
# MOMspider
# Monster
# Motor
# MSNBot
# Muncher
# Muninn
# Muscat Ferret
# Mwd.Search
# Internet Shinchakubin
# NDSpider
# NetCarta WebMap Engine
# NetMechanic
# NetScoop
# newscan-online
# NHSE Web Forager
# Nomad
# The NorthStar Robot
# ObjectsSearch
# Occam
# HKU WWW Octopus
# OntoSpider
# Openfind data gatherer
# Orb Search
# Pack Rat
# PageBoy
# ParaSite
# Patric
# pegasus
# The Peregrinator
# PerlCrawler 1.0
# Phantom
# PhpDig
# PiltdownMan
# Pimptrain.com's robot
# Pioneer
# html_analyzer
# Portal Juice Spider
# PGP Key Agent
# PlumtreeWebAccessor
# Poppi
# PortalB Spider
# psbot
# GetterroboPlus Puu
# The Python Robot
# Raven Search
# RBSE Spider
# Resume Robot
# RoadHouse Crawling System
# RixBot
# Road Runner: The ImageScape Robot
# Robbie the Robot
# ComputingSite Robi/1.0
# RoboCrawl Spider
# RoboFox
# Robozilla
# Roverbot
# RuLeS
# SafetyNet Robot
# Scooter
# Search.Aus-AU.COM
# Sleek
# SearchProcess
# Senrigan
# SG-Scout
# ShagSeeker
# Shai'Hulud
# Sift
# Simmany Robot Ver1.0
# Site Valet
# SiteTech-Rover
# Skymob.com
# SLCrawler
# Inktomi Slurp
# Smart Spider
# Snooper
# Solbot
# Speedy Spider
# spider_monkey
# SpiderBot
# Spiderline Crawler
# SpiderMan
# SpiderView(tm)
# Spry Wizard Robot
# Site Searcher
# Suke
# suntek search engine
# Sven
# Sygol
# TACH Black Widow
# Tarantula
# tarspider
# Tcl W3 Robot
# TechBOT
# Templeton
# TitIn
# TITAN
# The TkWWW Robot
# TLSpider
# UCSD Crawl
# UdmSearch
# UptimeBot
# URL Check
# URL Spider Pro
# Valkyrie
# Verticrawl
# Victoria
# vision-search
# void-bot
# Voyager
# VWbot
# The NWI Robot
# W3M2
# WallPaper (alias crawlpaper)
# the World Wide Web Wanderer
# w@pSpider by wap4.com
# WebBandit Web Spider
# WebCatcher
# WebCopy
# webfetcher
# The Webfoot Robot
# Webinator
# weblayers
# WebLinker
# WebMirror
# The Web Moose
# WebQuest
# Digimarc MarcSpider
# WebReaper
# webs
# Websnarf
# WebSpider
# WebVac
# webwalk
# WebWalker
# WebWatch
# Wget
# whatUseek Winona
# WhoWhere Robot
# Weblog Monitor
# w3mir
# WebStolperer
# The Web Wombat
# The World Wide Web Worm
# WWWC Ver 0.2.5
# WebZinger
# XGET
# Nederland.zoek


Name 	Details
ABCdatos BotLink 	
Agent: 	ABCdatos BotLink/1.0.2 (test links)
Host: 	217.126.39.167
Email: 	botlink+AEA-abcdatos.com
Acme.Spider 	
Agent: 	Due to a deficiency in Java it's not currently possible to set the User-Agent.
Host: 	*
Email: 	jef@acme.com
Ahoy! The Homepage Finder 	
Agent: 	'Ahoy! The Homepage Finder'
Host: 	cs.washington.edu
Email: 	marclang@cs.washington.edu
Alkaline 	
Agent: 	AlkalineBOT
Host: 	*
Email: 	dblock@vestris.com
Anthill 	
Agent: 	AnthillV1.1
Host: 	anywhere
Email: 	info@anthill.org
Walhello appie 	
Agent: 	appie/1.1
Host: 	213.10.10.116, 213.10.10.117, 213.10.10.118
Email: 	aimo@walhello.com
Arachnophilia 	
Agent: 	Arachnophilia
Host: 	halsoft.com
Email: 	taluskie@utpapa.ph.utexas.edu
Arale 	
Agent: 	no
Host: 	*
Email: 	flaviotordini@tiscali.it
Araneo 	
Agent: 	Araneo/0.7 (araneo@esperantisto.net; http://esperantisto.net)
Host: 	*.esperantisto.net
Email: 	araneo@esperantisto.net
AraybOt 	
Agent: 	AraybOt/1.0 (+http://www.araykoo.com/araybot.html)
Host: 	*
Email: 	robot@araykoo.com
ArchitextSpider 	
Agent: 	ArchitextSpider
Host: 	*.atext.com
Email: 	spider@atext.com
Aretha 	
Agent: 	
Host: 	
Email: 	davew@well.com
ARIADNE 	
Agent: 	Due to a deficiency in Java it's not currently possible to set the User-Agent.
Host: 	dbs.informatik.uni-muenchen.de
Email: 	Gross@dbs.informatik.uni-muenchen.de
arks 	
Agent: 	arks/1.0
Host: 	dpsindia.com
Email: 	aniruddha.c@usa.net
ASpider (Associative Spider) 	
Agent: 	ASpider/0.09
Host: 	nova.pvv.unit.no
Email: 	fredj@pvv.ntnu.no
ATN Worldwide 	
Agent: 	ATN_Worldwide
Host: 	www.allthatnet.com
Email: 	info@allthatnet.com
Atomz.com Search Robot 	
Agent: 	Atomz/1.0
Host: 	www.atomz.com
Email: 	mike@atomz.com
AURESYS 	
Agent: 	AURESYS/1.0
Host: 	crrm.univ-mrs.fr, 192.134.99.192
Email: 	mannina@crrm.univ-mrs.fr
BackRub 	
Agent: 	BackRub/*.*
Host: 	*.stanford.edu
Email: 	page@leland.stanford.edu
unnamed 	
Agent: 	BaySpider
Host: 	
Email: 	marki@baytsp.com
BBot 	
Agent: 	bbot/0.100
Host: 	*.netcologne.de
Email: 	poluf1 at yahoo dot co dot uk
Big Brother 	
Agent: 	Big Brother
Host: 	*
Email: 	Francois.Pottier@inria.fr
Bjaaland 	
Agent: 	Bjaaland/0.5
Host: 	barry.bitmovers.net
Email: 	tbray@textuality.com
BlackWidow 	
Agent: 	BlackWidow
Host: 	140.190.65.*
Email: 	khooghee@marys.smumn.edu
Die Blinde Kuh 	
Agent: 	Die Blinde Kuh
Host: 	minerva.sozialwiss.uni-hamburg.de
Email: 	maschinist@blinde-kuh.de
Bloodhound 	
Agent: 	None
Host: 	*
Email: 	genius@ukonline.co.uk
Borg-Bot 	
Agent: 	borg-bot/0.9
Host: 	24.11.13.173
Email: 	botdev@skunkfarm.com
BoxSeaBot 	
Agent: 	BoxSeaBot/0.5 (http://boxsea.com/crawler)
Host: 	
Email: 	boxseasearch@yahoo.com
bright.net caching robot 	
Agent: 	Mozilla/3.01 (compatible;)
Host: 	209.143.1.46
Email: 	
BSpider 	
Agent: 	BSpider/1.0 libwww-perl/0.40
Host: 	210.159.73.34, 210.159.73.35
Email: 	okumura@rsl.crl.fujixerox.co.jp
CACTVS Chemistry Spider 	
Agent: 	CACTVS Chemistry Spider
Host: 	utamaro.organik.uni-erlangen.de
Email: 	wdi@eros.ccc.uni-erlangen.de
Calif 	
Agent: 	Calif/0.6 (kosarev@tnps.net; http://www.tnps.dp.ua)
Host: 	cobra.tnps.dp.ua
Email: 	kosarev@tnps.net
Cassandra 	
Agent: 	
Host: 	www.aha.ru
Email: 	billy168@aha.ru
Digimarc Marcspider/CGI 	
Agent: 	Digimarc CGIReader/1.0
Host: 	206.102.3.*
Email: 	wmreader@digimarc.com
Checkbot 	
Agent: 	Checkbot/x.xx LWP/5.x
Host: 	*
Email: 	graaff@xs4all.nl
ChristCrawler.com 	
Agent: 	Mozilla/4.0 (compatible; ChristCrawler.com, ChristCrawler@ChristCENTRAL.com)
Host: 	64.51.218.*, 64.51.219.*, 12.107.236.*, 12.107.237.*
Email: 	jeremy.deyoung@christcentral.com
churl 	
Agent: 	
Host: 	
Email: 	yunke@umich.edu
cIeNcIaFiCcIoN.nEt 	
Agent: 	cIeNcIaFiCcIoN.nEt Spider (http://www.cienciaficcion.net)
Host: 	epervier.cqhost.net
Email: 	root@cyberdark.net
CMC/0.01 	
Agent: 	CMC/0.01
Host: 	haruna.next.ne.jp, 203.183.218.4
Email: 	shinobu@po.next.ne.jp
Collective 	
Agent: 	LWP
Host: 	*
Email: 	genius@ukonline.co.uk
Combine System 	
Agent: 	combine/0.0
Host: 	*.ub2.lu.se
Email: 	tsao@munin.ub2.lu.se
ConfuzzledBot 	
Agent: 	Confuzzledbot/X.X (+http://www.confuzzled.lu/bot/)
Host: 	*.ion.lu
Email: 	bot@confuzzled.lu
CoolBot 	
Agent: 	CoolBot
Host: 	www.suchmaschine21.de
Email: 	info@suchmaschine21.de
Web Core / Roots 	
Agent: 	root/0.1
Host: 	shiva.di.uminho.pt, from www.di.uminho.pt
Email: 	wc@di.uminho.pt
XYLEME Robot 	
Agent: 	cosmos/0.3
Host: 	
Email: 	preda@xyleme.com
Internet Cruiser Robot 	
Agent: 	Internet Cruiser Robot/2.1
Host: 	*.krstarica.com
Email: 	robot@krstarica.com
Cusco 	
Agent: 	Cusco/3.2
Host: 	*.cusco.pt, *.viatecla.pt
Email: 	clerigo@viatecla.pt
CyberSpyder Link Test 	
Agent: 	CyberSpyder/2.1
Host: 	*
Email: 	amant@cyberspyder.com
CydralSpider 	
Agent: 	CydralSpider/X.X (Cydral Web Image Search; http://www.cydral.com/)
Host: 	*.cydral.com
Email: 	cydral@cydral.com
Desert Realm Spider 	
Agent: 	DesertRealm.com; 0.2; [J];
Host: 	*
Email: 	spider@desertrealm.com
DeWeb(c) Katalog/Index 	
Agent: 	Deweb/1.01
Host: 	deweb.orbit.de
Email: 	dewebmaster@orbit.de
DienstSpider 	
Agent: 	dienstspider/1.0
Host: 	sappho.csi.forth.gr
Email: 	asidirop@csi.forth.gr
Digger 	
Agent: 	Digger/1.0 JDK/1.3.0
Host: 	
Email: 	admin@bulldozersoftware.com
Digital Integrity Robot 	
Agent: 	DIIbot
Host: 	digital-integrity.com
Email: 	robot@digital-integrity.com
Direct Hit Grabber 	
Agent: 	grabber
Host: 	*.directhit.com
Email: 	DirectHitGrabber@directhit.com
DNAbot 	
Agent: 	DNAbot/1.0
Host: 	xx.dnainc.co.jp
Email: 	tomatell@xx.dnainc.co.jp
DownLoad Express 	
Agent: 	
Host: 	*
Email: 	dlxpress@mediaone.net
DragonBot 	
Agent: 	DragonBot/1.0 libwww/5.0
Host: 	*.paczone.com
Email: 	admin@paczone.com
DWCP (Dridus' Web Cataloging Project) 	
Agent: 	DWCP/2.0
Host: 	*.dridus.com
Email: 	rmm@dridus.com
e-collector 	
Agent: 	LWP::
Host: 	*
Email: 	smarty@thatrobotsite.com
EbiNess 	
Agent: 	EbiNess/0.01a
Host: 	
Email: 	mdavis@kieser.net
EIT Link Verifier Robot 	
Agent: 	EIT-Link-Verifier-Robot/0.2
Host: 	*
Email: 	mcguire@eit.COM
ELFINBOT 	
Agent: 	elfinbot
Host: 	*.letsfinditnow.com
Email: 	admin@letsfinditnow.com
Emacs-w3 Search Engine 	
Agent: 	Emacs-w3/v[0-9\.]+
Host: 	*
Email: 	wmperry@spry.com
ananzi 	
Agent: 	EMC Spider
Host: 	bilbo.internal.empirical.com
Email: 	hpayne@u-media.com
esculapio 	
Agent: 	esculapio/1.1
Host: 	80.34.92.45
Email: 	imasd@cype.com
Esther 	
Agent: 	esther
Host: 	*.falconsoft.com
Email: 	tim@falconsoft.com
Evliya Celebi 	
Agent: 	Evliya Celebi v0.151 - http://ilker.ulak.net.tr
Host: 	193.140.83.*
Email: 	ilker@ulak.net.tr
nzexplorer 	
Agent: 	explorersearch
Host: 	bitz.co.nz
Email: 	paul@bourke.gen.nz
FastCrawler 	
Agent: 	FastCrawler 3.0.X (crawler@1klik.dk) - http://www.1klik.dk
Host: 	1klik.dk
Email: 	crawler@1klik.dk
Fluid Dynamics Search Engine robot 	
Agent: 	Mozilla/4.0 (compatible: FDSE robot)
Host: 	yes
Email: 	zoltanm@nickname.net
Felix IDE 	
Agent: 	FelixIDE/1.0
Host: 	*
Email: 	felix@pentone.com
Wild Ferret Web Hopper #1, #2, #3 	
Agent: 	Hazel's Ferret Web hopper,
Host: 	
Email: 	ghbos@postoffice.worldnet.att.net
FetchRover 	
Agent: 	ESIRover v1.0
Host: 	*
Email: 	ken@engsoftware.com
fido 	
Agent: 	fido/0.9 Harvest/1.4.pl2
Host: 	fido.planetsearch.com, *.planetsearch.com, 206.64.113.*
Email: 	fido@planetsearch.com
Hämähäkki 	
Agent: 	Hämähäkki/0.2
Host: 	*.www.fi
Email: 	Timo.Metsala@www.fi
KIT-Fireball 	
Agent: 	KIT-Fireball/2.0 libwww/5.0a
Host: 	*.fireball.de
Email: 	info@fireball.de
Fish search 	
Agent: 	Fish-Search-Robot
Host: 	www.win.tue.nl
Email: 	debra@win.tue.nl
Fouineur 	
Agent: 	Mozilla/2.0 (compatible fouineur v2.0; fouineur.9bit.qc.ca)
Host: 	*
Email: 	jvandal@9bit.qc.ca
Robot Francoroute 	
Agent: 	Robot du CRIM 1.0a
Host: 	zorro.crim.ca
Email: 	maparent@crim.ca
Freecrawl 	
Agent: 	Freecrawl
Host: 	*.freeside.net
Email: 	ekhall@freeside.net
FunnelWeb 	
Agent: 	FunnelWeb-1.0
Host: 	earth.planets.com.au
Email: 	eaglesd@pc.com.au
gammaSpider, FocusedCrawler 	
Agent: 	gammaSpider xxxxxxx ()/
Host: 	*
Email: 	support@gammasite.com
gazz 	
Agent: 	gazz/1.0
Host: 	*.nttrd.com, *.infobee.ne.jp
Email: 	gazz@nttrd.com
GCreep 	
Agent: 	gcreep/1.0
Host: 	mbx.instrumentpolen.se
Email: 	anders@instrumentpolen.se
GetBot 	
Agent: 	???
Host: 	
Email: 	zav@macromedia.com
GetURL 	
Agent: 	GetURL.rexx v1.05
Host: 	*
Email: 	James@Snark.apana.org.au
Golem 	
Agent: 	Golem/1.1
Host: 	*.quibble.com
Email: 	geoff@quibble.com
Googlebot 	
Agent: 	Googlebot/2.X (+http://www.googlebot.com/bot.html)
Host: 	googlebot.com
Email: 	googlebot@google.com
Grapnel/0.01 Experiment 	
Agent: 	
Host: 	varies
Email: 	v93_kat@ce.kth.se
Griffon 	
Agent: 	griffon/1.0
Host: 	*.navi.ocn.ne.jp
Email: 	griffon@super.navi.ocn.ne.jp
Gromit 	
Agent: 	Gromit/1.0
Host: 	*.austlii.edu.au
Email: 	dan@austlii.edu.au
Northern Light Gulliver 	
Agent: 	Gulliver/1.1
Host: 	scooby.northernlight.com, taz.northernlight.com, gulliver.northernlight.com
Email: 	crawler@northernlight.com
Gulper Bot 	
Agent: 	Gulper Web Bot 0.2.4 (www.ecsl.cs.sunysb.edu/~maxim/cgi-bin/Link/GulperBot)
Host: 	yuntis*.ecsl.cs.sunysb.edu
Email: 	gulperbot@ecsl.cs.sunysb.edu
HamBot 	
Agent: 	
Host: 	*.hamrad.com
Email: 	john@futureone.com
Harvest 	
Agent: 	yes
Host: 	bruno.cs.colorado.edu
Email: 	
havIndex 	
Agent: 	havIndex/X.xx[bxx]
Host: 	*
Email: 	havIndex@hav.com
HI (HTML Index) Search 	
Agent: 	AITCSRobot/1.1
Host: 	
Email: 	a94385@cs.ait.ac.th
Hometown Spider Pro 	
Agent: 	Hometown Spider Pro
Host: 	63.195.193.17
Email: 	admin@hometownsingles.com
Wired Digital 	
Agent: 	wired-digital-newsbot/1.5
Host: 	gossip.hotwired.com
Email: 	bowen@hotwired.com
ht://Dig 	
Agent: 	htdig/3.1.0b2
Host: 	*
Email: 	andrew@contigo.com
HTMLgobble 	
Agent: 	HTMLgobble v2.2
Host: 	tp70.rz.uni-karlsruhe.de
Email: 	ley@rz.uni-karlsruhe.de
Hyper-Decontextualizer 	
Agent: 	no
Host: 	
Email: 	cliff@tricon.net
iajaBot 	
Agent: 	iajaBot/0.1
Host: 	*.scs.carleton.ca
Email: 	morin@scs.carleton.ca
IBM_Planetwide 	
Agent: 	IBM_Planetwide,
Host: 	www.ibm.com www2.ibm.com
Email: 	epc@www.ibm.com"
Popular Iconoclast 	
Agent: 	gestaltIconoclast/1.0 libwww-FM/2.17
Host: 	gestalt.sewanee.edu
Email: 	chris@gestalt.sewanee.edu
Ingrid 	
Agent: 	INGRID/0.1
Host: 	bart.ilse.nl
Email: 	ilse@ilse.nl
Imagelock 	
Agent: 	Mozilla 3.01 PBWF (Win95)
Host: 	209.111.133.*
Email: 	belanger@imagelock.com
IncyWincy 	
Agent: 	IncyWincy/1.0b1
Host: 	osiris.sunderland.ac.uk
Email: 	simon.stobart@sunderland.ac.uk
Informant 	
Agent: 	Informant
Host: 	informant.dartmouth.edu
Email: 	info_adm@cosmo.dartmouth.edu
InfoSeek Robot 1.0 	
Agent: 	InfoSeek Robot 1.0
Host: 	corp-gw.infoseek.com
Email: 	stk@infoseek.com
Infoseek Sidewinder 	
Agent: 	Infoseek Sidewinder
Host: 	
Email: 	mna@infoseek.com
InfoSpiders 	
Agent: 	InfoSpiders/0.1
Host: 	*.ucsd.edu
Email: 	fil@cs.ucsd.edu
Inspector Web 	
Agent: 	inspectorwww/1.0 http://www.greenpac.com/inspectorwww.html
Host: 	www.corpsite.com, www.greenpac.com, 38.234.171.*
Email: 	doug@greenpac.com
IntelliAgent 	
Agent: 	'IAGENT/1.0'
Host: 	sand.it.bond.edu.au
Email: 	s1523@sand.it.bond.edu.au
I, Robot 	
Agent: 	I Robot 0.4 (irobot@chaos.dk)
Host: 	*.mame.dk, 206.161.121.*
Email: 	irobot@chaos.dk
Iron33 	
Agent: 	Iron33/0.0
Host: 	*.folon.ueda.info.waseda.ac.jp, 133.9.215.*
Email: 	watanabe@ueda.info.waseda.ac.jp
Israeli-search 	
Agent: 	IsraeliSearch/1.0
Host: 	dylan.ius.cs.cmu.edu
Email: 	etamar@xpert.co
JavaBee 	
Agent: 	JavaBee
Host: 	*
Email: 	info@objectbox.com
JBot Java Web Robot 	
Agent: 	JBot (but can be changed by the user)
Host: 	*
Email: 	daniel@matuschek.net
JCrawler 	
Agent: 	JCrawler/0.2
Host: 	db.netimages.com
Email: 	snowhare@netimages.com
AskJeeves 	
Agent: 	Mozilla/2.0 (compatible; Ask Jeeves/Teoma)
Host: 	ez*.directhit.com
Email: 	postmaster@ask.com
JoBo Java Web Robot 	
Agent: 	JoBo (can be modified by the user)
Host: 	*
Email: 	daniel@matuschek.net
Jobot 	
Agent: 	Jobot/0.1alpha libwww-perl/4.0
Host: 	supernova.micrognosis.com
Email: 	ajack@corp.micrognosis.com
JoeBot 	
Agent: 	JoeBot/x.x,
Host: 	
Email: 	rwaldin@primenet.com
The Jubii Indexing Robot 	
Agent: 	JubiiRobot/version#
Host: 	any host in the cybernet.dk domain
Email: 	jakob@jubii.dk
JumpStation 	
Agent: 	jumpstation
Host: 	*.stir.ac.uk
Email: 	j.fletcher@stirling.ac.uk
image.kapsi.net 	
Agent: 	image.kapsi.net/1.0
Host: 	addr-212-50-142-138.suomi.net
Email: 	Jaakko.Heusala@kapsi.net
Katipo 	
Agent: 	Katipo/1.0
Host: 	*
Email: 	Michael.Newbery@vuw.ac.nz
KDD-Explorer 	
Agent: 	KDD-Explorer/0.1
Host: 	mlc.kddvw.kcom.or.jp
Email: 	matsu@lab.kdd.co.jp
Kilroy 	
Agent: 	yes
Host: 	*.oclc.org
Email: 	kilroy@oclc.org
KO_Yappo_Robot 	
Agent: 	KO_Yappo_Robot/1.0.4(http://yappo.com/info/robot.html)
Host: 	yappo.com,209.25.40.1
Email: 	office_KO@yappo.com
LabelGrabber 	
Agent: 	LabelGrab/1.1
Host: 	head.w3.org
Email: 	jamieson@mit.edu
larbin 	
Agent: 	larbin (+mail)
Host: 	*
Email: 	sebastien.ailleret@inria.fr
legs 	
Agent: 	legs
Host: 	
Email: 	admin@magportal.com
Link Validator 	
Agent: 	Linkidator/0.93
Host: 	*.mitre.org
Email: 	tgimon@mitre.org
LinkScan 	
Agent: 	LinkScan Server/5.5 | LinkScan Workstation/5.5
Host: 	*
Email: 	sales@elsop.com
LinkWalker 	
Agent: 	LinkWalker
Host: 	*.seventwentyfour.com
Email: 	rbryant@seventwentyfour.com
Lockon 	
Agent: 	Lockon/xxxxx
Host: 	*.hitech.tuis.ac.jp
Email: 	search@rsch.tuis.ac.jp
logo.gif Crawler 	
Agent: 	logo.gif crawler
Host: 	*.inm.de
Email: 	sevo@inm.de
Lycos 	
Agent: 	Lycos/x.x
Host: 	fuzine.mt.cs.cmu.edu, lycos.com
Email: 	fuzzy@cmu.edu
Mac WWWWorm 	
Agent: 	
Host: 	
Email: 	lemieuse@ERE.UMontreal.CA
Magpie 	
Agent: 	Magpie/1.0
Host: 	*.blueberry.co.uk, 194.70.52.*, 193.131.167.144
Email: 	Keith.Jones@blueberry.co.uk
marvin/infoseek 	
Agent: 	marvin/infoseek (marvin-team@webseek.de)
Host: 	arthur*.sda.t-online.de
Email: 	marvin-team@webseek.de
Mattie 	
Agent: 	M/3.8
Host: 	mattie.mcw.aarkayn.org
Email: 	matt@mcw.aarkayn.org
MediaFox 	
Agent: 	MediaFox/x.y
Host: 	141.99.*.*
Email: 	sfx@uni-media.de
MerzScope 	
Agent: 	MerzScope
Host: 	(Client Based)
Email: 	
NEC-MeshExplorer 	
Agent: 	NEC-MeshExplorer
Host: 	meshsv300.tk.mesh.ad.jp
Email: 	web-dir@mxa.meshnet.or.jp
MindCrawler 	
Agent: 	MindCrawler
Host: 	*
Email: 	support@mindpass.com
mnoGoSearch search engine software 	
Agent: 	UdmSearch
Host: 	*
Email: 	support@mnogosearch.org
moget 	
Agent: 	moget/1.0
Host: 	*.goo.ne.jp
Email: 	moget@goo.ne.jp
MOMspider 	
Agent: 	MOMspider/1.00 libwww-perl/0.40
Host: 	*
Email: 	fielding@ics.uci.edu
Monster 	
Agent: 	Monster/vX.X.X -$TYPE ($OSTYPE)
Host: 	wild.stu.neva.ru
Email: 	diwil@wild.stu.neva.ru
Motor 	
Agent: 	Motor/0.2
Host: 	Michael.cybercon.technopark.gmd.de
Email: 	Motor@cybercon.technopark.gmd.de
MSNBot 	
Agent: 	MSNBOT/0.1 (http://search.msn.com/msnbot.htm)
Host: 	
Email: 	msnbot@microsoft.com
Muncher 	
Agent: 	yes
Host: 	www.goodlookingcooking.co.uk
Email: 	muncher@ridings.org.uk
Muninn 	
Agent: 	Muninn/0.1 libwww-perl-5.76 (http://people.freenet.de/Muninn/)
Host: 	santana.dynalias.net, 80.185.*, *
Email: 	muninn_bot@gmx.net
Muscat Ferret 	
Agent: 	MuscatFerret/
Host: 	193.114.89.*, 194.168.54.11
Email: 	olly@muscat.co.uk
Mwd.Search 	
Agent: 	MwdSearch/0.1
Host: 	*.fifi.net
Email: 	Antti.Westerberg@mwd.sci.fi
Internet Shinchakubin 	
Agent: 	User-Agent: Mozilla/4.0 (compatible; sharp-info-agent v1.0; )
Host: 	*
Email: 	shinchakubin-request@isl.nara.sharp.co.jp
NDSpider 	
Agent: 	NDSpider/1.5
Host: 	Blowfish.NationalDirectory.net
Email: 	dns3@NationalDirectory.com
NetCarta WebMap Engine 	
Agent: 	NetCarta CyberPilot Pro
Host: 	
Email: 	info@netcarta.com
NetMechanic 	
Agent: 	NetMechanic
Host: 	206.26.168.18
Email: 	tdahm@iquest.com
NetScoop 	
Agent: 	NetScoop/1.0 libwww/5.0a
Host: 	alpha.is.tokushima-u.ac.jp, beta.is.tokushima-u.ac.jp
Email: 	kita@is.tokushima-u.ac.jp
newscan-online 	
Agent: 	newscan-online/1.1
Host: 	*newscan-online.de
Email: 	mueller@newscan-online.de
NHSE Web Forager 	
Agent: 	NHSEWalker/3.0
Host: 	*.mcs.anl.gov
Email: 	olson@mcs.anl.gov
Nomad 	
Agent: 	Nomad-V2.x
Host: 	*.cs.colostate.edu
Email: 	sonnen@cs.colostat.edu
The NorthStar Robot 	
Agent: 	NorthStar
Host: 	frognot.utdallas.edu, utdallas.edu, cnidir.org
Email: 	barrie@unr.edu
ObjectsSearch 	
Agent: 	ObjectsSearch/0.01
Host: 	
Email: 	support@thesoftwareobjects.com
Occam 	
Agent: 	Occam/1.0
Host: 	gentian.cs.washington.edu, sekiu.cs.washington.edu, saxifrage.cs.washington.edu
Email: 	friedman@cs.washington.edu
HKU WWW Octopus 	
Agent: 	HKU WWW Robot,
Host: 	phoenix.cs.hku.hk
Email: 	jax@cs.hku.hk
OntoSpider 	
Agent: 	OntoSpider/1.0 libwww-perl/5.65
Host: 	ontospider.i-n.info
Email: 	ontospider@int-org.com
Openfind data gatherer 	
Agent: 	Openfind data gatherer, Openbot/3.0+(robot-response@openfind.com.tw;+http://www.openfind.com.tw/robot.html)
Host: 	66.7.131.132
Email: 	robot-response@openfind.com.tw
Orb Search 	
Agent: 	Orbsearch/1.0
Host: 	cow.dyn.ml.org, *.dyn.ml.org
Email: 	webernet@geocities.com
Pack Rat 	
Agent: 	PackRat/1.0
Host: 	cps.msu.edu
Email: 	dexterte@cps.msu.edu
PageBoy 	
Agent: 	PageBoy/1.0
Host: 	*.webdocs.org
Email: 	pageboy@webdocs.org
ParaSite 	
Agent: 	ParaSite/0.21 (http://www.ianett.com/parasite/)
Host: 	*.ianett.com
Email: 	parasite@ianett.com
Patric 	
Agent: 	Patric/0.01a
Host: 	*.nwnet.net
Email: 	webmaster@nwnet.net
pegasus 	
Agent: 	web robot PEGASUS
Host: 	*
Email: 	shannon@opensource.or.id
The Peregrinator 	
Agent: 	Peregrinator-Mathematics/0.7
Host: 	
Email: 	jimr@maths.su.oz.au
PerlCrawler 1.0 	
Agent: 	PerlCrawler/1.0 Xavatoria/2.0
Host: 	server5.hypermart.net
Email: 	webmaster@perlsearch.hypermart.net
Phantom 	
Agent: 	Duppies
Host: 	
Email: 	lburke@aktiv.com
PhpDig 	
Agent: 	phpdig/x.x.x
Host: 	yes
Email: 	phpdig@toiletoine.net
PiltdownMan 	
Agent: 	PiltdownMan/1.0 profitnet@myezmail.com
Host: 	62.36.128.*, 194.133.59.*, 212.106.215.*
Email: 	profitnet@myezmail.com
Pimptrain.com's robot 	
Agent: 	Mozilla/4.0 (compatible: Pimptrain's robot)
Host: 	pimtprain.com
Email: 	webmaster@pimptrain.com
Pioneer 	
Agent: 	Pioneer
Host: 	*.uncfsu.edu or flyer.ncsc.org
Email: 	micah@sequent.uncfsu.edu
html_analyzer 	
Agent: 	
Host: 	
Email: 	pitkow@aries.colorado.edu
Portal Juice Spider 	
Agent: 	PortalJuice.com/4.0
Host: 	*.portaljuice.com, *.nextopia.com
Email: 	pjspider@portaljuice.com
PGP Key Agent 	
Agent: 	PGP-KA/1.2
Host: 	salerno.starnet.it
Email: 	puma@comm2000.it
PlumtreeWebAccessor 	
Agent: 	PlumtreeWebAccessor/0.9
Host: 	
Email: 	josephs@plumtree.com
Poppi 	
Agent: 	Poppi/1.0
Host: 	=20
Email: 	
PortalB Spider 	
Agent: 	PortalBSpider/1.0 (spider@portalb.com)
Host: 	spider1.portalb.com, spider2.portalb.com, etc.
Email: 	spider@portalb.com
psbot 	
Agent: 	psbot/0.X (+http://www.picsearch.com/bot.html)
Host: 	*.picsearch.com
Email: 	psbot@picsearch.com
GetterroboPlus Puu 	
Agent: 	straight FLASH!! GetterroboPlus 1.5
Host: 	straight FLASH!! Getterrobo-Plus, *.homing.net
Email: 	marunaka@homing.net
The Python Robot 	
Agent: 	
Host: 	
Email: 	guido@python.org
Raven Search 	
Agent: 	Raven-v2
Host: 	192.168.1.*
Email: 	ravensearch@hotmail.com
RBSE Spider 	
Agent: 	
Host: 	rbse.jsc.nasa.gov (192.88.42.10)
Email: 	eichmann@rbse.jsc.nasa.gov
Resume Robot 	
Agent: 	Resume Robot
Host: 	
Email: 	proquest@onramp.net
RoadHouse Crawling System 	
Agent: 	RHCS/1.0a
Host: 	stage.perceval.be
Email: 	helpdesk@perceval.be
RixBot 	
Agent: 	RixBot (http://www.oops-as.no/rix/)
Host: 	www.oops-as.no
Email: 	
Road Runner: The ImageScape Robot 	
Agent: 	Road Runner: ImageScape Robot (lim@cs.leidenuniv.nl)
Host: 	
Email: 	lim@cs.leidenuniv.nl
Robbie the Robot 	
Agent: 	Robbie/0.1
Host: 	*.lmco.com
Email: 	robert.h.pollack@lmco.com
ComputingSite Robi/1.0 	
Agent: 	ComputingSite Robi/1.0 (robi@computingsite.com)
Host: 	robi.computingsite.com
Email: 	robi@computingsite.com
RoboCrawl Spider 	
Agent: 	RoboCrawl (http://www.canadiancontent.net)
Host: 	ncc.canadiancontent.net, ncc.air-net.no, canadiancontent.net, spider.canadiancontent.net
Email: 	staff@canadiancontent.net
RoboFox 	
Agent: 	Robofox v2.0
Host: 	*
Email: 	robo_fox@hotmail.com
Robozilla 	
Agent: 	Robozilla/1.0
Host: 	directory.mozilla.org
Email: 	robozilla@dmozed.org
Roverbot 	
Agent: 	Roverbot
Host: 	roverbot.com
Email: 	gmd@spyder.net
RuLeS 	
Agent: 	RuLeS/1.0 libwww/4.0
Host: 	www.rules.be
Email: 	marc@rules.be
SafetyNet Robot 	
Agent: 	SafetyNet Robot 0.1,
Host: 	*.urlabs.com
Email: 	m.l.nelson@urlabs.com
Scooter 	
Agent: 	Scooter/2.0 G.R.A.B. V1.1.0
Host: 	*.av.pa-x.dec.com
Email: 	scooter@pa.dec.com
Search.Aus-AU.COM 	
Agent: 	not available
Host: 	Search.Aus-AU.COM, 203.55.124.29, 203.2.239.29
Email: 	dez@geko.com
Sleek 	
Agent: 	Mozilla/4.0 (Sleek Spider/1.2)
Host: 	yes
Email: 	lawrence.hughes@search-info.com
SearchProcess 	
Agent: 	searchprocess/0.9
Host: 	searchprocess.com
Email: 	bruno@intelligence-process.com
Senrigan 	
Agent: 	Senrigan/xxxxxx
Host: 	aniki.olu.info.waseda.ac.jp
Email: 	kent@muraoka.info.waseda.ac.jp
SG-Scout 	
Agent: 	SG-Scout
Host: 	beta.xerox.com
Email: 	ptbb@ai.mit.edu, beebee@parc.xerox.com
ShagSeeker 	
Agent: 	Shagseeker at http://www.shagseek.com /1.0
Host: 	shagseek.com
Email: 	joe.reynolds@shagseek.com
Shai'Hulud 	
Agent: 	Shai'Hulud
Host: 	*.rdtex.ru
Email: 	shawdow@usa.net
Sift 	
Agent: 	libwww-perl-5.41
Host: 	www.worthy.com
Email: 	bworthy@worthy.com
Simmany Robot Ver1.0 	
Agent: 	SimBot/1.0
Host: 	sansam.hnc.net
Email: 	ailove@hnc.co.kr
Site Valet 	
Agent: 	Site Valet
Host: 	valet.webthing.com,valet.*
Email: 	nick@webthing.com
SiteTech-Rover 	
Agent: 	SiteTech-Rover
Host: 	
Email: 	adasilva@sitetech.com
Skymob.com 	
Agent: 	aWapClient
Host: 	www.skymob.com
Email: 	searchmaster@skymob.com
SLCrawler 	
Agent: 	SLCrawler
Host: 	n/a
Email: 	kng@inxight.com
Inktomi Slurp 	
Agent: 	Slurp/2.0
Host: 	*.inktomi.com
Email: 	slurp@inktomi.com
Smart Spider 	
Agent: 	ESISmartSpider/2.0
Host: 	207.16.241.*
Email: 	ken@engsoftware.com
Snooper 	
Agent: 	Snooper/b97_01
Host: 	
Email: 	melnicki@sit.ca
Solbot 	
Agent: 	Solbot/1.0 LWP/5.07
Host: 	robot*.sol.no
Email: 	ftj@sys.sol.no
Speedy Spider 	
Agent: 	Speedy Spider ( http://www.entireweb.com/speedy.html )
Host: 	router-00.sverige.net, 193.15.210.29, *.entireweb.com, *.worldlight.com
Email: 	speedy@worldlight.com
spider_monkey 	
Agent: 	mouse.house/7.1
Host: 	snowball.ionsys.com
Email: 	mprm@ionsys.com
SpiderBot 	
Agent: 	SpiderBot/1.0
Host: 	*
Email: 	spidrboticruzado@solaria.emp.ubu.es
Spiderline Crawler 	
Agent: 	spiderline/3.1.3
Host: 	*.spiderline.com, *.spiderline.org
Email: 	ben@spiderline.com
SpiderMan 	
Agent: 	SpiderMan 1.0
Host: 	NA
Email: 	leunghok@comp.nus.edu.sg
SpiderView(tm) 	
Agent: 	Mozilla/4.0 (compatible; SpiderView 1.0;unix)
Host: 	bobmin.quad2.iuinc.com, *
Email: 	webmaster@northernwebs.com
Spry Wizard Robot 	
Agent: 	no
Host: 	wizard.spry.com or tiger.spry.com
Email: 	info@spry.com
Site Searcher 	
Agent: 	ssearcher100
Host: 	*
Email: 	zackware@hotmail.com
Suke 	
Agent: 	suke/*.*
Host: 	*
Email: 	robot@kensaku.org
suntek search engine 	
Agent: 	suntek/1.0
Host: 	search.suntek.com.hk
Email: 	karen@suntek.com.hk
Sven 	
Agent: 	
Host: 	24.113.12.29
Email: 	rhondle@home.com
Sygol 	
Agent: 	http://www.sygol.com
Host: 	http://www.sygol.com
Email: 	webmaster@sygol.com
TACH Black Widow 	
Agent: 	Mozilla/3.0 (Black Widow v1.1.0; Linux 2.0.27; Dec 31 1997 12:25:00
Host: 	*.theautochannel.com
Email: 	mjenn@theautochannel.com
Tarantula 	
Agent: 	Tarantula/1.0
Host: 	yes
Email: 	Markus.Hoevener@evision.de
tarspider 	
Agent: 	tarspider
Host: 	
Email: 	chakl@fu-berlin.de
Tcl W3 Robot 	
Agent: 	dlw3robot/x.y (in TclX by http://hplyot.obspm.fr/~dl/)
Host: 	hplyot.obspm.fr
Email: 	dl@hplyot.obspm.fr
TechBOT 	
Agent: 	TechBOT
Host: 	techaid.net
Email: 	techbot@techaid.net
Templeton 	
Agent: 	Templeton/{version} for {platform}
Host: 	*
Email: 	nealk@net66.com
TitIn 	
Agent: 	TitIn/0.2
Host: 	barok.foi.hr
Email: 	dpavlin@foi.hr
TITAN 	
Agent: 	TITAN/0.1
Host: 	nlptitan.isl.ntt.jp
Email: 	hayashi@nttnly.isl.ntt.jp
The TkWWW Robot 	
Agent: 	
Host: 	
Email: 	scott@cs.sunyit.edu
TLSpider 	
Agent: 	TLSpider/1.1
Host: 	tlspider.topiclink.com (not avalible yet)
Email: 	tlspider@outtel.com
UCSD Crawl 	
Agent: 	UCSD-Crawler
Host: 	nuthaus.mib.org scilib.ucsd.edu
Email: 	atilghma@mib.org
UdmSearch 	
Agent: 	UdmSearch/2.1.1
Host: 	*
Email: 	bar@izhcom.ru
UptimeBot 	
Agent: 	uptimebot
Host: 	uptimebot.com
Email: 	luft_master@ukr.net
URL Check 	
Agent: 	urlck/1.2.3
Host: 	*
Email: 	dave@cutternet.com
URL Spider Pro 	
Agent: 	URL Spider Pro
Host: 	*
Email: 	greg@innerprise.net
Valkyrie 	
Agent: 	Valkyrie/1.0 libwww-perl/0.40
Host: 	*.c.u-tokyo.ac.jp
Email: 	harada@graco.c.u-tokyo.ac.jp
Verticrawl 	
Agent: 	Verticrawlbot
Host: 	http://www.verticrawl.com/
Email: 	webmaster@verticrawl.com
Victoria 	
Agent: 	Victoria/1.0
Host: 	
Email: 	adrianh@oneworld.co.uk
vision-search 	
Agent: 	vision-search/3.0'
Host: 	dylan.ius.cs.cmu.edu
Email: 	har@cs.cmu.edu
void-bot 	
Agent: 	void-bot/0.1 (bot@void.be; http://www.void.be/)
Host: 	void.be
Email: 	bot@void.be
Voyager 	
Agent: 	Voyager/0.0
Host: 	*.lisa.co.jp
Email: 	voyager@lisa.co.jp
VWbot 	
Agent: 	VWbot_K/4.2
Host: 	vancouver-webpages.com
Email: 	andrew@vancouver-webpages.com
The NWI Robot 	
Agent: 	w3index
Host: 	nwi.ub2.lu.se, mars.dtv.dk and a few others
Email: 	siglun@munin.ub2.lu.se
W3M2 	
Agent: 	W3M2/x.xxx
Host: 	*
Email: 	tronche@lri.fr
WallPaper (alias crawlpaper) 	
Agent: 	CrawlPaper/n.n.n (Windows n)
Host: 	
Email: 	lpiergentili@yahoo.com
the World Wide Web Wanderer 	
Agent: 	WWWWanderer v3.0
Host: 	*.mit.edu
Email: 	mkgray@mit.edu
w@pSpider by wap4.com 	
Agent: 	w@pSpider/xxx (unix) by wap4.com
Host: 	*.wap4.com, *.mopilot.com
Email: 	info@wap4.com
WebBandit Web Spider 	
Agent: 	WebBandit/1.0
Host: 	ix.netcom.com
Email: 	wooger@ix.netcom.com
WebCatcher 	
Agent: 	WebCatcher/1.0
Host: 	oscar.lang.nagoya-u.ac.jp
Email: 	reiji@infonia.ne.jp
WebCopy 	
Agent: 	WebCopy/(version)
Host: 	*
Email: 	vparada@inf.utfsm.cl
webfetcher 	
Agent: 	WebFetcher/0.8,
Host: 	*
Email: 	webfetch@ontv.com
The Webfoot Robot 	
Agent: 	
Host: 	phoenix.doc.ic.ac.uk
Email: 	L.McLoughlin@doc.ic.ac.uk
Webinator 	
Agent: 	
Host: 	several
Email: 	
weblayers 	
Agent: 	weblayers/0.0
Host: 	
Email: 	loic@afp.com
WebLinker 	
Agent: 	WebLinker/0.0 libwww-perl/0.1
Host: 	
Email: 	jcasey@maths.tcd.ie
WebMirror 	
Agent: 	no
Host: 	
Email: 	sfchan@mailhost.net
The Web Moose 	
Agent: 	WebMoose/0.0.0000
Host: 	msn.com
Email: 	mikeblas@nwlink.com
WebQuest 	
Agent: 	WebQuest/1.0
Host: 	210.121.146.2, 210.113.104.1, 210.113.104.2
Email: 	cty@cosmonet.co.kr
Digimarc MarcSpider 	
Agent: 	Digimarc WebReader/1.2
Host: 	206.102.3.*
Email: 	wmreader@digimarc.com
WebReaper 	
Agent: 	WebReaper [webreaper@otway.com]
Host: 	*
Email: 	webreaper@otway.com
webs 	
Agent: 	webs@recruit.co.jp
Host: 	lemon.recruit.co.jp
Email: 	dew@wwwadmin.rnet.or.jp
Websnarf 	
Agent: 	
Host: 	
Email: 	charles@fma.com
WebSpider 	
Agent: 	
Host: 	several
Email: 	u610468@csi.uottawa.ca
WebVac 	
Agent: 	webvac/1.0
Host: 	
Email: 	tim@federated.com
webwalk 	
Agent: 	webwalk
Host: 	
Email: 	
WebWalker 	
Agent: 	WebWalker/1.10
Host: 	*
Email: 	fccheong@cs.berkeley.edu
WebWatch 	
Agent: 	WebWatch
Host: 	
Email: 	janos@specter.com
Wget 	
Agent: 	Wget/1.4.0
Host: 	*
Email: 	hniksic@srce.hr
whatUseek Winona 	
Agent: 	whatUseek_winona/3.0
Host: 	*.whatuseek.com, *.aol2.com
Email: 	neil@whatUseek.com
WhoWhere Robot 	
Agent: 	
Host: 	spica.whowhere.com
Email: 	rupesh@whowhere.com
Weblog Monitor 	
Agent: 	wlm-1.1
Host: 	blossom.metastatic.org
Email: 	rsdio@metastatic.org
w3mir 	
Agent: 	w3mir
Host: 	
Email: 	w3mir-core@usit.uio.no
WebStolperer 	
Agent: 	WOLP/1.0 mda/1.0
Host: 	www.suchfibel.de
Email: 	mda@suchfibel.de
The Web Wombat 	
Agent: 	no
Host: 	qwerty.intercom.com.au
Email: 	phill@intercom.com.au
The World Wide Web Worm 	
Agent: 	
Host: 	piper.cs.colorado.edu
Email: 	mcbryan@piper.cs.colorado.edu
WWWC Ver 0.2.5 	
Agent: 	WWWC/0.25 (Win95)
Host: 	
Email: 	naka@kinet.or.jp
WebZinger 	
Agent: 	none
Host: 	http://www.imaginon.com/wzindex.html *
Email: 	info@imaginon.com
XGET 	
Agent: 	XGET/0.7
Host: 	*
Email: 	shige@mh1.117.ne.jp
Nederland.zoek 	
Agent: 	Nederland.zoek
Host: 	193.67.110.*
Email: 	zoek@nederland.net


http://www.robotstxt.org/wc/active/html/raw.html