# this file is generated from a master file, please modify it and regenerate it with the generate_lists.rb script
bot
spider
crawl
[^a]fish
^voyager\/
ADmantX
alexa
Alexandria(\s|\+)prototype(\s|\+)project
AllenTrack
almaden
appie
API[\+\s]scraper
Arachmo
architext
ArchiveTeam
arks
asterias
atomz
BDFetch
baidu
biglotron
BingPreview
binlar
Blackboard[\+\s]Safeassign
blaiz\-bee
bloglines
blogpulse
boitho\.com\-dc
bookmark\-manager
Brutus\/AET
BUbiNG
bwh3_user_agent
celestial
cfnetwork
checklink
checkprivacy
China\sLocal\sBrowse\s2\.6
cloakDetect
coccoc\/1\.0
collection@infegy.com
com\.plumanalytics
combine
contentmatch
ContentSmartz
convera
core
CoverScout
cursor
custo
DataCha0s\/2\.0
daumoa
DeuSu\/
Docoloc
docomo
DSurf
DTS Agent
easydl
EmailSiphon
EmailWolf
Embedly
EThOS\+\(British\+Library\)
facebookexternalhit\/
feedburner
FeedFetcher
feedreader
ferret
findlinks
Fulltext
Funnelback
G-i-g-a-b-o-t
Goldfire(\s|\+)Server
google
Grammarly
grub
gulliver
harvest
heritrix
holmes
htdig
htmlparser
HTTPFetcher
httrack
ia_archiver
ichiro
iktomi
ilse
^integrity\/\d
internetseer
intute
iSiloX
iskanie
jeeves
jobo
kyluka
larbin
lilina
link.?check
LinkLint-checkonly
^LinkParser\/
^LinkSaver\/
linkscan
LinkTiger
linkwalker
lipperhey
livejournal\.com
LOCKSS
ltx71
lycos[\_\+]
mail.ru
mediapartners\-google
megite
MetaURI[\+\s]API\/\d\.\d
mimas
mnogosearch
moget
motor
MuscatFerre
myweb
nagios
^NetAnts\/\d
netcraft
netluchs
Ning
nomad
nutch
^oaDOI$
ocelli
Offline(\s|\+)Navigator
onetszukaj
OurBrowser
panscient
parsijoo
EasyBib[\+\s]AutoCite[\+\s]
perman
pioneer
playmusic\.com
playstarmusic\.com
^Postgenomic(\s|\+)v2
powermarks
proximic
Qwantify
Readpaper
redalert
Riddler
robozilla
scan4mail
scientificcommons
scirus
scooter
Scrapy\/\d
^scrutiny\/\d
SearchBloxIntra
shoutcast
SkypeUriPreview
slurp
sogou
speedy
Strider
summify
sunrise
Sysomos
T\-H\-U\-N\-D\-E\-R\-S\-T\-O\-N\-E
tailrank
Teleport(\s|\+)Pro
Teoma
titan
^Traackr\.com$
Trove
twiceler
ucsd
ultraseek
urlaliasbuilder
validator
virus.detector
voila
^voltron$
voyager\/
w3af.org
Wanadoo
Web(\s|\+)Downloader
WebCloner
webcollage
WebCopier
Webinator
weblayers
Webmetrics
webmirror
webmon
webreaper
WebStripper
WebZIP
worm
www.gnip.com
WWW\-Mechanize
xenu
y!j
yacy
yahoo
yandex
zeus
zyborg