[ { "pattern": "Googlebot\\/", "url": "http://www.google.com/bot.html", "instances": [ "Googlebot/2.1 (+http://www.google.com/bot.html)", "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)", "Mozilla/5.0 (iPhone; CPU iPhone OS 6_0 like Mac OS X) AppleWebKit/536.26 (KHTML, like Gecko) Version/6.0 Mobile/10A5376e Safari/8536.25 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)", "Mozilla/5.0 (iPhone; CPU iPhone OS 8_3 like Mac OS X) AppleWebKit/537.36 (KHTML, like Gecko) Version/8.0 Mobile/12F70 Safari/600.1.4 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)", "Mozilla/5.0 (iPhone; CPU iPhone OS 8_3 like Mac OS X) AppleWebKit/600.1.4 (KHTML, like Gecko) Version/8.0 Mobile/12F70 Safari/600.1.4 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)", "Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/27.0.1453 Mobile Safari/537.36 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)", "Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2272.96 Mobile Safari/537.36 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)", "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Googlebot/2.1; +http://www.google.com/bot.html) Safari/537.36" ], "description": "Google's main web crawling bot for search indexing", "tags": [ "search-engine" ] }, { "pattern": "Googlebot-Mobile", "instances": [ "DoCoMo/2.0 N905i(c100;TB;W24H16) (compatible; Googlebot-Mobile/2.1; +http://www.google.com/bot.html)", "Mozilla/5.0 (iPhone; CPU iPhone OS 6_0 like Mac OS X) AppleWebKit/536.26 (KHTML, like Gecko) Version/6.0 Mobile/10A5376e Safari/8536.25 (compatible; Googlebot-Mobile/2.1; +http://www.google.com/bot.html)", "Mozilla/5.0 (iPhone; U; CPU iPhone OS 4_1 like Mac OS X; en-us) AppleWebKit/532.9 (KHTML, like Gecko) Version/4.0.5 Mobile/8B117 Safari/6531.22.7 (compatible; Googlebot-Mobile/2.1; +http://www.google.com/bot.html)", "Nokia6820/2.0 (4.83) Profile/MIDP-1.0 Configuration/CLDC-1.0 (compatible; Googlebot-Mobile/2.1; +http://www.google.com/bot.html)", "SAMSUNG-SGH-E250/1.0 Profile/MIDP-2.0 Configuration/CLDC-1.1 UP.Browser/6.2.3.3.c.1.101 (GUI) MMP/2.0 (compatible; Googlebot-Mobile/2.1; +http://www.google.com/bot.html)" ], "description": "Google's legacy mobile crawler for Google Search indexing", "tags": [ "search-engine" ] }, { "pattern": "Googlebot-Image", "instances": [ "Googlebot-Image/1.0" ], "description": "Google's image-specific web crawling bot for image search indexing", "tags": [ "search-engine" ] }, { "pattern": "Googlebot-News", "instances": [ "Googlebot-News" ], "description": "Google's news-specific web crawling bot for Google News indexing", "tags": [ "search-engine" ] }, { "pattern": "Googlebot-Video", "instances": [ "Googlebot-Video/1.0" ], "description": "Google's video crawler for video-related Google Search features", "tags": [ "search-engine" ] }, { "pattern": "AdsBot-Google([^-]|$)", "url": "https://support.google.com/webmasters/answer/1061943?hl=en", "instances": [ "AdsBot-Google (+http://www.google.com/adsbot.html)" ], "description": "Google's Ads bot for checking web page ad quality", "tags": [ "advertising" ] }, { "pattern": "AdsBot-Google-Mobile", "addition_date": "2017/08/21", "url": "https://support.google.com/adwords/answer/2404197", "instances": [ "AdsBot-Google-Mobile-Apps", "Mozilla/5.0 (Linux; Android 5.0; SM-G920A) AppleWebKit (KHTML, like Gecko) Chrome Mobile Safari (compatible; AdsBot-Google-Mobile; +http://www.google.com/mobile/adsbot.html)", "Mozilla/5.0 (iPhone; CPU iPhone OS 9_1 like Mac OS X) AppleWebKit/601.1.46 (KHTML, like Gecko) Version/9.0 Mobile/13B143 Safari/601.1 (compatible; AdsBot-Google-Mobile; +http://www.google.com/mobile/adsbot.html)" ], "description": "Google's mobile Ads bot for crawling mobile pages to serve targeted ads", "tags": [ "advertising" ] }, { "pattern": "Feedfetcher-Google", "addition_date": "2018/06/27", "url": "https://support.google.com/webmasters/answer/178852", "instances": [ "Feedfetcher-Google; (+http://www.google.com/feedfetcher.html; 1 subscribers; feed-id=728742641706423)" ], "description": "Google's feed fetcher bot for fetching RSS and Atom feeds for Google services", "tags": [ "feed-reader" ] }, { "pattern": "Mediapartners-Google", "url": "https://support.google.com/webmasters/answer/1061943?hl=en", "instances": [ "Mediapartners-Google", "Mozilla/5.0 (compatible; MSIE or Firefox mutant; not on Windows server;) Daumoa/4.0 (Following Mediapartners-Google)", "Mozilla/5.0 (iPhone; U; CPU iPhone OS 10_0 like Mac OS X; en-us) AppleWebKit/602.1.38 (KHTML, like Gecko) Version/10.0 Mobile/14A5297c Safari/602.1 (compatible; Mediapartners-Google/2.1; +http://www.google.com/bot.html)", "Mozilla/5.0 (iPhone; U; CPU iPhone OS 4_1 like Mac OS X; en-us) AppleWebKit/532.9 (KHTML, like Gecko) Version/4.0.5 Mobile/8B117 Safari/6531.22.7 (compatible; Mediapartners-Google/2.1; +http://www.google.com/bot.html)" ], "description": "Google's Mediapartners bot for AdSense and AdMob crawling", "tags": [ "advertising" ] }, { "pattern": "Mediapartners \\(Googlebot\\)", "addition_date": "2017/08/08", "url": "https://support.google.com/webmasters/answer/1061943?hl=en", "instances": [], "description": "Google's Mediapartners bot variant for AdSense and AdMob crawling", "tags": [ "advertising" ] }, { "pattern": "APIs-Google", "addition_date": "2017/08/08", "url": "https://support.google.com/webmasters/answer/1061943?hl=en", "instances": [ "APIs-Google (+https://developers.google.com/webmasters/APIs-Google.html)" ], "description": "Google's APIs bot for crawling API documentation and services", "tags": [ "search-engine" ] }, { "pattern": "Google-InspectionTool", "url": "https://developers.google.com/search/docs/crawling-indexing/overview-google-crawlers", "instances": [ "Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/W.X.Y.Z Mobile Safari/537.36 (compatible; Google-InspectionTool/1.0)", "Mozilla/5.0 (compatible; Google-InspectionTool/1.0)" ], "description": "Google's inspection tool bot for testing and debugging search indexing", "tags": [ "search-engine" ] }, { "pattern": "Storebot-Google", "url": "https://developers.google.com/search/docs/crawling-indexing/overview-google-crawlers", "instances": [ "Mozilla/5.0 (X11; Linux x86_64; Storebot-Google/1.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.88 Safari/537.36", "Mozilla/5.0 (Linux; Android 8.0; Pixel 2 Build/OPD3.170816.012; Storebot-Google/1.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.138 Mobile Safari/537.36" ], "description": "Google's Storebot for crawling product and e-commerce pages", "tags": [ "search-engine" ] }, { "pattern": "GoogleOther", "url": "https://developers.google.com/search/docs/crawling-indexing/overview-google-crawlers", "instances": [ "GoogleOther" ], "description": "Google's other bots and services for various Google search features", "tags": [ "search-engine" ] }, { "pattern": "bingbot", "url": "http://www.bing.com/bingbot.htm", "instances": [ "Mozilla/5.0 (Windows Phone 8.1; ARM; Trident/7.0; Touch; rv:11.0; IEMobile/11.0; NOKIA; Lumia 530) like Gecko (compatible; adidxbot/2.0; +http://www.bing.com/bingbot.htm)", "Mozilla/5.0 (compatible; adidxbot/2.0; http://www.bing.com/bingbot.htm)", "Mozilla/5.0 (compatible; adidxbot/2.0; +http://www.bing.com/bingbot.htm)", "Mozilla/5.0 (compatible; bingbot/2.0; http://www.bing.com/bingbot.htm)", "Mozilla/5.0 (compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm", "Mozilla/5.0 (compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm)", "Mozilla/5.0 (compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm) SitemapProbe", "Mozilla/5.0 (iPhone; CPU iPhone OS 7_0 like Mac OS X) AppleWebKit/537.51.1 (KHTML, like Gecko) Version/7.0 Mobile/11A465 Safari/9537.53 (compatible; adidxbot/2.0; http://www.bing.com/bingbot.htm)", "Mozilla/5.0 (iPhone; CPU iPhone OS 7_0 like Mac OS X) AppleWebKit/537.51.1 (KHTML, like Gecko) Version/7.0 Mobile/11A465 Safari/9537.53 (compatible; adidxbot/2.0; +http://www.bing.com/bingbot.htm)", "Mozilla/5.0 (iPhone; CPU iPhone OS 7_0 like Mac OS X) AppleWebKit/537.51.1 (KHTML, like Gecko) Version/7.0 Mobile/11A465 Safari/9537.53 (compatible; bingbot/2.0; http://www.bing.com/bingbot.htm)", "Mozilla/5.0 (iPhone; CPU iPhone OS 7_0 like Mac OS X) AppleWebKit/537.51.1 (KHTML, like Gecko) Version/7.0 Mobile/11A465 Safari/9537.53 (compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm)", "Mozilla/5.0 (seoanalyzer; compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm)", "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm) Safari/537.36", "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm) Chrome/103.0.5060.134 Safari/537.36" ], "description": "Microsoft's web crawling bot for Bing search indexing", "tags": [ "search-engine" ] }, { "pattern": "Slurp", "url": "http://help.yahoo.com/help/us/ysearch/slurp", "instances": [ "Mozilla/5.0 (compatible; Yahoo! Slurp/3.0; http://help.yahoo.com/help/us/ysearch/slurp)", "Mozilla/5.0 (compatible; Yahoo! Slurp; http://help.yahoo.com/help/us/ysearch/slurp)", "Mozilla/5.0 (compatible; Yahoo! Slurp China; http://misc.yahoo.com.cn/help.html)" ], "description": "Yahoo's web crawling bot for Yahoo search indexing", "tags": [ "search-engine" ] }, { "pattern": "[wW]get", "instances": [ "WGETbot/1.0 (+http://wget.alanreed.org)", "Wget/1.14 (linux-gnu)", "Wget/1.20.3 (linux-gnu)" ], "description": "GNU Wget command-line tool for downloading web content", "tags": [ "http-library" ] }, { "pattern": "LinkedInBot", "instances": [ "LinkedInBot/1.0 (compatible; Mozilla/5.0; Jakarta Commons-HttpClient/3.1 +http://www.linkedin.com)", "LinkedInBot/1.0 (compatible; Mozilla/5.0; Jakarta Commons-HttpClient/4.3 +http://www.linkedin.com)", "LinkedInBot/1.0 (compatible; Mozilla/5.0; Apache-HttpClient +http://www.linkedin.com)" ], "description": "LinkedIn's bot for crawling professional content and profiles", "tags": [ "social-preview" ] }, { "pattern": "Python-urllib", "instances": [ "Python-urllib/1.17", "Python-urllib/2.5", "Python-urllib/2.6", "Python-urllib/2.7", "Python-urllib/3.1", "Python-urllib/3.2", "Python-urllib/3.3", "Python-urllib/3.4", "Python-urllib/3.5", "Python-urllib/3.6", "Python-urllib/3.7" ], "description": "Python's built-in URL library for HTTP requests", "tags": [ "http-library" ] }, { "pattern": "python-requests", "addition_date": "2018/05/27", "instances": [ "python-requests/2.9.2", "python-requests/2.11.1", "python-requests/2.18.4", "python-requests/2.19.1", "python-requests/2.20.0", "python-requests/2.21.0", "python-requests/2.22.0" ], "description": "Popular Python HTTP library for making web requests", "tags": [ "http-library" ] }, { "pattern": "aiohttp", "addition_date": "2019/12/23", "instances": [ "Python/3.9 aiohttp/3.7.3", "Python/3.8 aiohttp/3.7.2", "Python/3.7 aiohttp/3.6.2a2" ], "url": "https://docs.aiohttp.org/en/stable/", "description": "Asynchronous HTTP client library for Python", "tags": [ "http-library" ] }, { "pattern": "httpx", "addition_date": "2019/12/23", "instances": [ "python-httpx/0.16.1", "python-httpx/0.13.0.dev1" ], "url": "https://www.python-httpx.org", "description": "Modern Python HTTP client with async support", "tags": [ "http-library" ] }, { "pattern": "libwww-perl", "instances": [ "2Bone_LinkChecker/1.0 libwww-perl/6.03", "2Bone_LinkChkr/1.0 libwww-perl/6.03", "amibot - http://www.amidalla.de - tech@amidalla.com libwww-perl/5.831" ], "description": "Perl library for making HTTP requests and web crawling", "tags": [ "http-library" ] }, { "pattern": "httpunit", "instances": [ "httpunit/1.x" ], "description": "Java library for automated web application testing", "tags": [ "http-library" ] }, { "pattern": "Nutch", "instances": [ "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/605.1.16 (KHTML, like Gecko; compatible; Friendly_Crawler/2.0) Chrome/120.0.6099.217 Safari/605.1.15/Nutch-1.20-SNAPSHOT", "NutchCVS/0.7.1 (Nutch; http://lucene.apache.org/nutch/bot.html; nutch-agent@lucene.apache.org)", "istellabot-nutch/Nutch-1.10" ], "description": "Apache Nutch open-source web crawler framework", "tags": [ "search-engine" ] }, { "pattern": "Go-http-client", "addition_date": "2016/03/26", "url": "https://golang.org/pkg/net/http/", "instances": [ "Go-http-client/1.1", "Go-http-client/2.0" ], "description": "Go programming language HTTP client library", "tags": [ "http-library" ] }, { "pattern": "phpcrawl", "addition_date": "2012/09/17", "url": "http://phpcrawl.cuab.de/", "instances": [ "phpcrawl" ], "description": "PHP web crawler library for scraping websites", "tags": [ "http-library" ] }, { "pattern": "msnbot", "url": "http://search.msn.com/msnbot.htm", "instances": [ "adidxbot/1.1 (+http://search.msn.com/msnbot.htm)", "adidxbot/2.0 (+http://search.msn.com/msnbot.htm)", "librabot/1.0 (+http://search.msn.com/msnbot.htm)", "librabot/2.0 (+http://search.msn.com/msnbot.htm)", "msnbot-NewsBlogs/2.0b (+http://search.msn.com/msnbot.htm)", "msnbot-UDiscovery/2.0b (+http://search.msn.com/msnbot.htm)", "msnbot-media/1.0 (+http://search.msn.com/msnbot.htm)", "msnbot-media/1.1 (+http://search.msn.com/msnbot.htm)", "msnbot-media/2.0b (+http://search.msn.com/msnbot.htm)", "msnbot/1.0 (+http://search.msn.com/msnbot.htm)", "msnbot/1.1 (+http://search.msn.com/msnbot.htm)", "msnbot/2.0b (+http://search.msn.com/msnbot.htm)", "msnbot/2.0b (+http://search.msn.com/msnbot.htm).", "msnbot/2.0b (+http://search.msn.com/msnbot.htm)._" ], "description": "Microsoft's search engine bot for web indexing", "tags": [ "search-engine" ] }, { "pattern": "jyxobot", "instances": [], "description": "Jyxo search engine bot for web crawling", "tags": [ "search-engine" ] }, { "pattern": "FAST-WebCrawler", "instances": [ "FAST-WebCrawler/3.6/FirstPage (atw-crawler at fast dot no;http://fast.no/support/crawler.asp)", "FAST-WebCrawler/3.7 (atw-crawler at fast dot no; http://fast.no/support/crawler.asp)", "FAST-WebCrawler/3.7/FirstPage (atw-crawler at fast dot no;http://fast.no/support/crawler.asp)", "FAST-WebCrawler/3.8" ], "description": "FAST search engine web crawler for indexing", "tags": [ "search-engine" ] }, { "pattern": "FAST Enterprise Crawler", "instances": [ "FAST Enterprise Crawler 6 / Scirus scirus-crawler@fast.no; http://www.scirus.com/srsapp/contactus/", "FAST Enterprise Crawler 6 used by Schibsted (webcrawl@schibstedsok.no)" ], "description": "FAST enterprise-grade web crawler for search", "tags": [ "search-engine" ] }, { "pattern": "BIGLOTRON", "instances": [ "BIGLOTRON (Beta 2;GNU/Linux)" ], "description": "Biglotron search engine web crawler bot", "tags": [ "search-engine" ] }, { "pattern": "Teoma", "instances": [ "Mozilla/2.0 (compatible; Ask Jeeves/Teoma; +http://sp.ask.com/docs/about/tech_crawling.html)", "Mozilla/2.0 (compatible; Ask Jeeves/Teoma; +http://about.ask.com/en/docs/about/webmasters.shtml)" ], "url": "http://about.ask.com/en/docs/about/webmasters.shtml", "description": "Ask Jeeves Teoma search engine web crawler", "tags": [ "search-engine" ] }, { "pattern": "convera", "instances": [ "ConveraCrawler/0.9e (+http://ews.converasearch.com/crawl.htm)" ], "url": "http://ews.converasearch.com/crawl.htm", "description": "Convera search engine web crawler bot", "tags": [ "search-engine" ] }, { "pattern": "^Seekbot", "instances": [ "Seekbot/1.0 (http://www.seekbot.net/bot.html) RobotsTxtFetcher/1.2" ], "url": "http://www.seekbot.net/bot.html", "description": "Seekbot search engine web crawler for indexing", "tags": [ "search-engine" ] }, { "pattern": "Gigabot", "instances": [ "Gigabot/1.0", "Gigabot/2.0 (http://www.gigablast.com/spider.html)" ], "url": "http://www.gigablast.com/spider.html", "description": "Gigablast search engine web crawler bot", "tags": [ "search-engine" ] }, { "pattern": "Gigablast", "instances": [ "GigablastOpenSource/1.0" ], "url": "https://github.com/gigablast/open-source-search-engine", "description": "Gigablast open-source search engine crawler", "tags": [ "search-engine" ] }, { "pattern": "exabot", "instances": [ "Mozilla/5.0 (compatible; Alexabot/1.0; +http://www.alexa.com/help/certifyscan; certifyscan@alexa.com)", "Mozilla/5.0 (compatible; Exabot PyExalead/3.0; +http://www.exabot.com/go/robot)", "Mozilla/5.0 (compatible; Exabot-Images/3.0; +http://www.exabot.com/go/robot)", "Mozilla/5.0 (compatible; Exabot/3.0 (BiggerBetter); +http://www.exabot.com/go/robot)", "Mozilla/5.0 (compatible; Exabot/3.0; +http://www.exabot.com/go/robot)", "Mozilla/5.0 (compatible; Exabot/3.0; http://www.exabot.com/go/robot)" ], "description": "Exabot search engine web crawler for indexing", "tags": [ "search-engine" ] }, { "pattern": "ia_archiver", "instances": [ "ia_archiver (+http://www.alexa.com/site/help/webmasters; crawler@alexa.com)", "ia_archiver-web.archive.org" ], "description": "Internet Archive Wayback Machine web crawler", "tags": [ "archiver" ] }, { "pattern": "GingerCrawler", "instances": [ "GingerCrawler/1.0 (Language Assistant for Dyslexics; www.gingersoftware.com/crawler_agent.htm; support at ginger software dot com)" ], "description": "Ginger Software's language assistant web crawler", "tags": [ "seo" ] }, { "pattern": "webmon ", "instances": [], "description": "Webmon website monitoring and crawling bot", "tags": [ "monitoring" ] }, { "pattern": "HTTrack", "instances": [ "Mozilla/4.5 (compatible; HTTrack 3.0x; Windows 98)" ], "description": "HTTrack website copier for offline browsing", "tags": [ "http-library" ] }, { "pattern": "grub\\.org", "instances": [ "Mozilla/4.0 (compatible; grub-client-0.3.0; Crawl your own stuff with http://grub.org)", "Mozilla/4.0 (compatible; grub-client-1.0.4; Crawl your own stuff with http://grub.org)", "Mozilla/4.0 (compatible; grub-client-1.0.5; Crawl your own stuff with http://grub.org)", "Mozilla/4.0 (compatible; grub-client-1.0.6; Crawl your own stuff with http://grub.org)", "Mozilla/4.0 (compatible; grub-client-1.0.7; Crawl your own stuff with http://grub.org)", "Mozilla/4.0 (compatible; grub-client-1.1.1; Crawl your own stuff with http://grub.org)", "Mozilla/4.0 (compatible; grub-client-1.2.1; Crawl your own stuff with http://grub.org)", "Mozilla/4.0 (compatible; grub-client-1.3.1; Crawl your own stuff with http://grub.org)", "Mozilla/4.0 (compatible; grub-client-1.3.7; Crawl your own stuff with http://grub.org)", "Mozilla/4.0 (compatible; grub-client-1.4.3; Crawl your own stuff with http://grub.org)", "Mozilla/4.0 (compatible; grub-client-1.5.3; Crawl your own stuff with http://grub.org)" ], "description": "Grub search engine web crawler for indexing", "tags": [ "search-engine" ] }, { "pattern": "UsineNouvelleCrawler", "instances": [], "description": "Usine Nouvelle news site web crawler", "tags": [ "search-engine" ] }, { "pattern": "antibot", "instances": [], "description": "Antibot web crawler for content discovery", "tags": [ "scanner" ] }, { "pattern": "netresearchserver", "instances": [], "description": "Net Research Server web crawler bot", "tags": [ "seo" ] }, { "pattern": "speedy", "instances": [ "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US) Speedy Spider (http://www.entireweb.com/about/search_tech/speedy_spider/)", "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US) Speedy Spider for SpeedyAds (http://www.entireweb.com/about/search_tech/speedy_spider/)", "Mozilla/5.0 (compatible; Speedy Spider; http://www.entireweb.com/about/search_tech/speedy_spider/)", "Speedy Spider (Entireweb; Beta/1.2; http://www.entireweb.com/about/search_tech/speedyspider/)", "Speedy Spider (http://www.entireweb.com/about/search_tech/speedy_spider/)" ], "description": "Entireweb Speedy Spider web crawler bot", "tags": [ "search-engine" ] }, { "pattern": "fluffy", "instances": [], "description": "Fluffy search engine web crawler bot", "tags": [ "search-engine" ] }, { "pattern": "findlink", "instances": [ "findlinks/1.0 (+http://wortschatz.uni-leipzig.de/findlinks/)", "findlinks/1.1.3-beta8 (+http://wortschatz.uni-leipzig.de/findlinks/)", "findlinks/1.1.3-beta9 (+http://wortschatz.uni-leipzig.de/findlinks/)", "findlinks/1.1.5-beta7 (+http://wortschatz.uni-leipzig.de/findlinks/)", "findlinks/1.1.6-beta1 (+http://wortschatz.uni-leipzig.de/findlinks/)", "findlinks/1.1.6-beta1 (+http://wortschatz.uni-leipzig.de/findlinks/; YaCy 0.1; yacy.net)", "findlinks/1.1.6-beta2 (+http://wortschatz.uni-leipzig.de/findlinks/)", "findlinks/1.1.6-beta3 (+http://wortschatz.uni-leipzig.de/findlinks/)", "findlinks/1.1.6-beta4 (+http://wortschatz.uni-leipzig.de/findlinks/)", "findlinks/1.1.6-beta5 (+http://wortschatz.uni-leipzig.de/findlinks/)", "findlinks/1.1.6-beta6 (+http://wortschatz.uni-leipzig.de/findlinks/)", "findlinks/2.0 (+http://wortschatz.uni-leipzig.de/findlinks/)", "findlinks/2.0.1 (+http://wortschatz.uni-leipzig.de/findlinks/)", "findlinks/2.0.2 (+http://wortschatz.uni-leipzig.de/findlinks/)", "findlinks/2.0.4 (+http://wortschatz.uni-leipzig.de/findlinks/)", "findlinks/2.0.5 (+http://wortschatz.uni-leipzig.de/findlinks/)", "findlinks/2.0.9 (+http://wortschatz.uni-leipzig.de/findlinks/)", "findlinks/2.1 (+http://wortschatz.uni-leipzig.de/findlinks/)", "findlinks/2.1.3 (+http://wortschatz.uni-leipzig.de/findlinks/)", "findlinks/2.1.5 (+http://wortschatz.uni-leipzig.de/findlinks/)", "findlinks/2.2 (+http://wortschatz.uni-leipzig.de/findlinks/)", "findlinks/2.5 (+http://wortschatz.uni-leipzig.de/findlinks/)", "findlinks/2.6 (+http://wortschatz.uni-leipzig.de/findlinks/)" ], "description": "Findlinks web crawler for link discovery", "tags": [ "seo" ] }, { "pattern": "msrbot", "instances": [], "description": "Microsoft Research web crawler bot", "tags": [ "search-engine" ] }, { "pattern": "panscient", "instances": [ "panscient.com" ], "description": "Panscient web crawler for content analysis", "tags": [ "seo" ] }, { "pattern": "yacybot", "instances": [ "yacybot (/global; amd64 FreeBSD 10.3-RELEASE; java 1.8.0_77; GMT/en) http://yacy.net/bot.html", "yacybot (/global; amd64 FreeBSD 10.3-RELEASE-p7; java 1.7.0_95; GMT/en) http://yacy.net/bot.html", "yacybot (-global; amd64 FreeBSD 9.2-RELEASE-p10; java 1.7.0_65; Europe/en) http://yacy.net/bot.html", "yacybot (/global; amd64 Linux 2.6.32-042stab093.4; java 1.7.0_65; Etc/en) http://yacy.net/bot.html", "yacybot (/global; amd64 Linux 2.6.32-042stab094.8; java 1.7.0_79; America/en) http://yacy.net/bot.html", "yacybot (/global; amd64 Linux 2.6.32-042stab108.8; java 1.7.0_91; America/en) http://yacy.net/bot.html", "yacybot (-global; amd64 Linux 2.6.32-042stab111.11; java 1.7.0_79; Europe/en) http://yacy.net/bot.html", "yacybot (-global; amd64 Linux 2.6.32-042stab116.1; java 1.7.0_79; Europe/en) http://yacy.net/bot.html", "yacybot (/global; amd64 Linux 2.6.32-573.3.1.el6.x86_64; java 1.7.0_85; Europe/en) http://yacy.net/bot.html", "yacybot (-global; amd64 Linux 3.10.0-229.4.2.el7.x86_64; java 1.7.0_79; Europe/en) http://yacy.net/bot.html", "yacybot (-global; amd64 Linux 3.10.0-229.4.2.el7.x86_64; java 1.8.0_45; Europe/en) http://yacy.net/bot.html", "yacybot (/global; amd64 Linux 3.10.0-229.7.2.el7.x86_64; java 1.8.0_45; Europe/en) http://yacy.net/bot.html", "yacybot (/global; amd64 Linux 3.10.0-327.22.2.el7.x86_64; java 1.7.0_101; Etc/en) http://yacy.net/bot.html", "yacybot (/global; amd64 Linux 3.11.10-21-desktop; java 1.7.0_51; America/en) http://yacy.net/bot.html", "yacybot (/global; amd64 Linux 3.12.1; java 1.7.0_65; Europe/en) http://yacy.net/bot.html", "yacybot (/global; amd64 Linux 3.13.0-042stab093.4; java 1.7.0_79; Europe/de) http://yacy.net/bot.html", "yacybot (/global; amd64 Linux 3.13.0-042stab093.4; java 1.7.0_79; Europe/en) http://yacy.net/bot.html", "yacybot (/global; amd64 Linux 3.13.0-45-generic; java 1.7.0_75; Europe/en) http://yacy.net/bot.html", "yacybot (-global; amd64 Linux 3.13.0-61-generic; java 1.7.0_79; Europe/en) http://yacy.net/bot.html", "yacybot (/global; amd64 Linux 3.13.0-74-generic; java 1.7.0_91; Europe/en) http://yacy.net/bot.html", "yacybot (/global; amd64 Linux 3.13.0-83-generic; java 1.7.0_95; Europe/de) http://yacy.net/bot.html", "yacybot (/global; amd64 Linux 3.13.0-83-generic; java 1.7.0_95; Europe/en) http://yacy.net/bot.html", "yacybot (/global; amd64 Linux 3.13.0-85-generic; java 1.7.0_101; Europe/en) http://yacy.net/bot.html", "yacybot (/global; amd64 Linux 3.13.0-85-generic; java 1.7.0_95; Europe/en) http://yacy.net/bot.html", "yacybot (/global; amd64 Linux 3.13.0-88-generic; java 1.7.0_101; Europe/en) http://yacy.net/bot.html", "yacybot (/global; amd64 Linux 3.14-0.bpo.1-amd64; java 1.7.0_55; Europe/de) http://yacy.net/bot.html", "yacybot (/global; amd64 Linux 3.14.32-xxxx-grs-ipv6-64; java 1.7.0_75; Europe/en) http://yacy.net/bot.html", "yacybot (-global; amd64 Linux 3.14.32-xxxx-grs-ipv6-64; java 1.8.0_111; Europe/de) http://yacy.net/bot.html", "yacybot (/global; amd64 Linux 3.16.0-4-amd64; java 1.7.0_111; Europe/de) http://yacy.net/bot.html", "yacybot (/global; amd64 Linux 3.16.0-4-amd64; java 1.7.0_75; America/en) http://yacy.net/bot.html", "yacybot (-global; amd64 Linux 3.16.0-4-amd64; java 1.7.0_75; Europe/en) http://yacy.net/bot.html", "yacybot (/global; amd64 Linux 3.16.0-4-amd64; java 1.7.0_75; Europe/en) http://yacy.net/bot.html", "yacybot (/global; amd64 Linux 3.16.0-4-amd64; java 1.7.0_79; Europe/de) http://yacy.net/bot.html", "yacybot (/global; amd64 Linux 3.16.0-4-amd64; java 1.7.0_79; Europe/en) http://yacy.net/bot.html", "yacybot (/global; amd64 Linux 3.16.0-4-amd64; java 1.7.0_91; Europe/de) http://yacy.net/bot.html", "yacybot (/global; amd64 Linux 3.16.0-4-amd64; java 1.7.0_95; Europe/en) http://yacy.net/bot.html", "yacybot (/global; amd64 Linux 3.16.0-4-amd64; java 1.8.0_111; Europe/en) http://yacy.net/bot.html", "yacybot (/global; amd64 Linux 3.16-0.bpo.2-amd64; java 1.7.0_65; Europe/en) http://yacy.net/bot.html", "yacybot (-global; amd64 Linux 3.19.0-15-generic; java 1.8.0_45-internal; Europe/de) http://yacy.net/bot.html", "yacybot (-global; amd64 Linux 3.2.0-4-amd64; java 1.7.0_65; Europe/en) http://yacy.net/bot.html", "yacybot (-global; amd64 Linux 3.2.0-4-amd64; java 1.7.0_67; Europe/en) http://yacy.net/bot.html", "yacybot (-global; amd64 Linux 4.4.0-57-generic; java 9-internal; Europe/en) http://yacy.net/bot.html", "yacybot (-global; amd64 Windows 8.1 6.3; java 1.7.0_55; Europe/de) http://yacy.net/bot.html", "yacybot (-global; amd64 Windows 8 6.2; java 1.7.0_55; Europe/de) http://yacy.net/bot.html", "yacybot (-global; amd64 Linux 5.2.8-Jinsol; java 12.0.2; Europe/en) http://yacy.net/bot.html", "yacybot (-global; amd64 Linux 5.2.9-Jinsol; java 12.0.2; Europe/en) http://yacy.net/bot.html", "yacybot (-global; amd64 Linux 5.2.11-Jinsol; java 12.0.2; Europe/en) http://yacy.net/bot.html" ], "description": "YaCy decentralized search engine web crawler", "tags": [ "search-engine" ] }, { "pattern": "AISearchBot", "instances": [], "description": "AI-powered search engine web crawler bot", "tags": [ "search-engine", "ai-crawler" ] }, { "pattern": "ips-agent", "instances": [ "BlackBerry9000/4.6.0.167 Profile/MIDP-2.0 Configuration/CLDC-1.1 VendorID/102 ips-agent", "Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.7.12; ips-agent) Gecko/20050922 Fedora/1.0.7-1.1.fc4 Firefox/1.0.7", "Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.9.1.3; ips-agent) Gecko/20090824 Fedora/1.0.7-1.1.fc4 Firefox/3.5.3", "Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.9.2.24; ips-agent) Gecko/20111107 Ubuntu/10.04 (lucid) Firefox/3.6.24", "Mozilla/5.0 (X11; Ubuntu; Linux i686; rv:14.0; ips-agent) Gecko/20100101 Firefox/14.0.1" ], "description": "IPS agent web crawler for content indexing", "tags": [ "seo" ] }, { "pattern": "tagoobot", "instances": [], "description": "Tagoo search engine web crawler bot", "tags": [ "search-engine" ] }, { "pattern": "MJ12bot", "instances": [ "MJ12bot/v1.2.0 (http://majestic12.co.uk/bot.php?+)", "Mozilla/5.0 (compatible; MJ12bot/v1.2.1; http://www.majestic12.co.uk/bot.php?+)", "Mozilla/5.0 (compatible; MJ12bot/v1.2.3; http://www.majestic12.co.uk/bot.php?+)", "Mozilla/5.0 (compatible; MJ12bot/v1.2.4; http://www.majestic12.co.uk/bot.php?+)", "Mozilla/5.0 (compatible; MJ12bot/v1.2.5; http://www.majestic12.co.uk/bot.php?+)", "Mozilla/5.0 (compatible; MJ12bot/v1.3.0; http://www.majestic12.co.uk/bot.php?+)", "Mozilla/5.0 (compatible; MJ12bot/v1.3.1; http://www.majestic12.co.uk/bot.php?+)", "Mozilla/5.0 (compatible; MJ12bot/v1.3.2; http://www.majestic12.co.uk/bot.php?+)", "Mozilla/5.0 (compatible; MJ12bot/v1.3.3; http://www.majestic12.co.uk/bot.php?+)", "Mozilla/5.0 (compatible; MJ12bot/v1.4.0; http://www.majestic12.co.uk/bot.php?+)", "Mozilla/5.0 (compatible; MJ12bot/v1.4.1; http://www.majestic12.co.uk/bot.php?+)", "Mozilla/5.0 (compatible; MJ12bot/v1.4.2; http://www.majestic12.co.uk/bot.php?+)", "Mozilla/5.0 (compatible; MJ12bot/v1.4.3; http://www.majestic12.co.uk/bot.php?+)", "Mozilla/5.0 (compatible; MJ12bot/v1.4.4 (domain ownership verifier); http://www.majestic12.co.uk/bot.php?+)", "Mozilla/5.0 (compatible; MJ12bot/v1.4.4; http://www.majestic12.co.uk/bot.php?+)", "Mozilla/5.0 (compatible; MJ12bot/v1.4.5; http://www.majestic12.co.uk/bot.php?+)", "Mozilla/5.0 (compatible; MJ12bot/v1.4.6; http://mj12bot.com/)", "Mozilla/5.0 (compatible; MJ12bot/v1.4.7; http://mj12bot.com/)", "Mozilla/5.0 (compatible; MJ12bot/v1.4.7; http://www.majestic12.co.uk/bot.php?+)", "Mozilla/5.0 (compatible; MJ12bot/v1.4.8; http://mj12bot.com/)" ], "description": "Majestic-12 search engine web crawler bot", "tags": [ "seo" ] }, { "pattern": "woriobot", "instances": [ "Mozilla/5.0 (compatible; woriobot +http://worio.com)", "Mozilla/5.0 (compatible; woriobot support [at] zite [dot] com +http://zite.com)" ], "description": "Worio search engine web crawler bot", "tags": [ "search-engine" ] }, { "pattern": "yanga", "instances": [ "Yanga WorldSearch Bot v1.1/beta (http://www.yanga.co.uk/)" ], "description": "Yanga search engine web crawler bot", "tags": [ "search-engine" ] }, { "pattern": "buzzbot", "instances": [ "Buzzbot/1.0 (Buzzbot; http://www.buzzstream.com; buzzbot@buzzstream.com)" ], "description": "Buzzstream web crawler for link research", "tags": [ "seo" ] }, { "pattern": "mlbot", "instances": [ "MLBot (www.metadatalabs.com/mlbot)" ], "description": "Metadata Labs web crawler for analysis", "tags": [ "seo" ] }, { "pattern": "yandex\\.com\\/bots", "url": "https://yandex.ru/support/webmaster/robot-workings/check-yandex-robots.html#robot-in-logs", "instances": [ "Mozilla/5.0 (compatible; YandexBot/3.0; +http://yandex.com/bots)", "Mozilla/5.0 (compatible; YandexBot/3.0; MirrorDetector; +http://yandex.com/bots)", "Mozilla/5.0 (iPhone; CPU iPhone OS 8_1 like Mac OS X) AppleWebKit/600.1.4 (KHTML, like Gecko) Version/8.0 Mobile/12B411 Safari/600.1.4 (compatible; YandexBot/3.0; +http://yandex.com/bots)", "Mozilla/5.0 (compatible; YandexImages/3.0; +http://yandex.com/bots)", "Mozilla/5.0 (compatible; YandexAccessibilityBot/3.0; +http://yandex.com/bots", "Mozilla/5.0 (compatible; YandexUserproxy; robot; +http://yandex.com/bots", "Mozilla/5.0 (iPhone; CPU iPhone OS 8_1 like Mac OS X) AppleWebKit/600.1.4 (KHTML, like Gecko) Version/8.0 Mobile/12B411 Safari/600.1.4 (compatible; YandexMobileBot/3.0; +http://yandex.com/bots)", "Mozilla/5.0 (compatible; YandexMetrika/2.0; +http://yandex.com/bots)", "Mozilla/5.0 (compatible; YandexMetrika/2.0; +http://yandex.com/bots yabs01)", "Mozilla/5.0 (compatible; YandexMetrika/3.0; +http://yandex.com/bots)", "Mozilla/5.0 (compatible; YandexMetrika/4.0; +http://yandex.com/bots)", "Mozilla/5.0 (compatible; YandexTurbo/1.0; +http://yandex.com/bots)", "Mozilla/5.0 (compatible; YandexVideoParser/1.0; +http://yandex.com/bots)", "Mozilla/5.0 (compatible; YandexVideo/3.0; +http://yandex.com/bots)", "Mozilla/5.0 (compatible; YandexImageResizer/2.0; +http://yandex.com/bots)", "Mozilla/5.0 (compatible; YandexAdNet/1.0; +http://yandex.com/bots)", "Mozilla/5.0 (compatible; YandexBlogs/0.99; robot; +http://yandex.com/bots)", "Mozilla/5.0 (compatible; YandexCalendar/1.0; +http://yandex.com/bots)", "Mozilla/5.0 (compatible; YandexDirect/3.0; +http://yandex.com/bots)", "Mozilla/5.0 (compatible; YandexDirectDyn/1.0; +http://yandex.com/bots", "Mozilla/5.0 (compatible; YandexFavicons/1.0; +http://yandex.com/bots)", "Mozilla/5.0 (compatible; YaDirectFetcher/1.0; Dyatel; +http://yandex.com/bots)", "Mozilla/5.0 (compatible; YandexForDomain/1.0; +http://yandex.com/bots)", "Mozilla/5.0 (compatible; YandexMarket/1.0; +http://yandex.com/bots)", "Mozilla/5.0 (compatible; YandexMarket/2.0; +http://yandex.com/bots)", "Mozilla/5.0 (compatible; YandexMedia/3.0; +http://yandex.com/bots)", "Mozilla/5.0 (compatible; YandexMobileScreenShotBot/1.0; +http://yandex.com/bots)", "Mozilla/5.0 (compatible; YandexNews/4.0; +http://yandex.com/bots)", "Mozilla/5.0 (compatible; YandexOntoDB/1.0; +http://yandex.com/bots)", "Mozilla/5.0 (compatible; YandexOntoDBAPI/1.0; +http://yandex.com/bots)", "Mozilla/5.0 (compatible; YandexPagechecker/1.0; +http://yandex.com/bots)", "Mozilla/5.0 (compatible; YandexPartner/3.0; +http://yandex.com/bots)", "Mozilla/5.0 (compatible; YandexRCA/1.0; +http://yandex.com/bots)", "Mozilla/5.0 (compatible; YandexSearchShop/1.0; +http://yandex.com/bots)", "Mozilla/5.0 (compatible; YandexSitelinks; Dyatel; +http://yandex.com/bots)", "Mozilla/5.0 (compatible; YandexSpravBot/1.0; +http://yandex.com/bots)", "Mozilla/5.0 (compatible; YandexTracker/1.0; +http://yandex.com/bots)", "Mozilla/5.0 (compatible; YandexVertis/3.0; +http://yandex.com/bots)", "Mozilla/5.0 (compatible; YandexVerticals/1.0; +http://yandex.com/bots)", "Mozilla/5.0 (compatible; YandexWebmaster/2.0; +http://yandex.com/bots)", "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2228.0 Safari/537.36 (compatible; YandexScreenshotBot/3.0; +http://yandex.com/bots)" ], "addition_date": "2015/04/14", "description": "Yandex search engine web crawler bots", "tags": [ "search-engine" ] }, { "pattern": "purebot", "addition_date": "2010/01/19", "instances": [], "description": "Pure web crawler for content discovery", "tags": [ "search-engine" ] }, { "pattern": "Linguee Bot", "addition_date": "2010/01/26", "url": "http://www.linguee.com/bot", "instances": [ "Linguee Bot (http://www.linguee.com/bot)", "Linguee Bot (http://www.linguee.com/bot; bot@linguee.com)" ], "description": "Linguee translation web crawler bot", "tags": [ "search-engine" ] }, { "pattern": "CyberPatrol", "addition_date": "2010/02/11", "url": "http://www.cyberpatrol.com/cyberpatrolcrawler.asp", "instances": [ "CyberPatrol SiteCat Webbot (http://www.cyberpatrol.com/cyberpatrolcrawler.asp)" ], "description": "CyberPatrol web content filtering bot", "tags": [ "scanner" ] }, { "pattern": "voilabot", "addition_date": "2010/05/18", "instances": [ "Mozilla/5.0 (Windows NT 5.1; U; Win64; fr; rv:1.8.1) VoilaBot BETA 1.2 (support.voilabot@orange-ftgroup.com)", "Mozilla/5.0 (Windows; U; Windows NT 5.1; fr; rv:1.8.1) VoilaBot BETA 1.2 (support.voilabot@orange-ftgroup.com)" ], "description": "Voila search engine web crawler bot", "tags": [ "search-engine" ] }, { "pattern": "Baiduspider", "addition_date": "2010/07/15", "url": "http://www.baidu.jp/spider/", "instances": [ "Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)", "Mozilla/5.0 (compatible; Baiduspider-render/2.0; +http://www.baidu.com/search/spider.html)" ], "description": "Baidu search engine web crawler bot", "tags": [ "search-engine" ] }, { "pattern": "citeseerxbot", "addition_date": "2010/07/17", "instances": [], "description": "CiteSeerX academic web crawler bot", "tags": [ "academic" ] }, { "pattern": "spbot", "addition_date": "2010/07/31", "url": "http://www.seoprofiler.com/bot", "instances": [ "Mozilla/5.0 (compatible; spbot/1.0; +http://www.seoprofiler.com/bot/ )", "Mozilla/5.0 (compatible; spbot/1.1; +http://www.seoprofiler.com/bot/ )", "Mozilla/5.0 (compatible; spbot/1.2; +http://www.seoprofiler.com/bot/ )", "Mozilla/5.0 (compatible; spbot/2.0.1; +http://www.seoprofiler.com/bot/ )", "Mozilla/5.0 (compatible; spbot/2.0.2; +http://www.seoprofiler.com/bot/ )", "Mozilla/5.0 (compatible; spbot/2.0.3; +http://www.seoprofiler.com/bot/ )", "Mozilla/5.0 (compatible; spbot/2.0.4; +http://www.seoprofiler.com/bot )", "Mozilla/5.0 (compatible; spbot/2.0; +http://www.seoprofiler.com/bot/ )", "Mozilla/5.0 (compatible; spbot/2.1; +http://www.seoprofiler.com/bot )", "Mozilla/5.0 (compatible; spbot/3.0; +http://www.seoprofiler.com/bot )", "Mozilla/5.0 (compatible; spbot/3.1; +http://www.seoprofiler.com/bot )", "Mozilla/5.0 (compatible; spbot/4.0.1; +http://www.seoprofiler.com/bot )", "Mozilla/5.0 (compatible; spbot/4.0.2; +http://www.seoprofiler.com/bot )", "Mozilla/5.0 (compatible; spbot/4.0.3; +http://www.seoprofiler.com/bot )", "Mozilla/5.0 (compatible; spbot/4.0.4; +http://www.seoprofiler.com/bot )", "Mozilla/5.0 (compatible; spbot/4.0.5; +http://www.seoprofiler.com/bot )", "Mozilla/5.0 (compatible; spbot/4.0.6; +http://www.seoprofiler.com/bot )", "Mozilla/5.0 (compatible; spbot/4.0.7; +http://OpenLinkProfiler.org/bot )", "Mozilla/5.0 (compatible; spbot/4.0.7; +https://www.seoprofiler.com/bot )", "Mozilla/5.0 (compatible; spbot/4.0.8; +http://OpenLinkProfiler.org/bot )", "Mozilla/5.0 (compatible; spbot/4.0.9; +http://OpenLinkProfiler.org/bot )", "Mozilla/5.0 (compatible; spbot/4.0; +http://www.seoprofiler.com/bot )", "Mozilla/5.0 (compatible; spbot/4.0a; +http://www.seoprofiler.com/bot )", "Mozilla/5.0 (compatible; spbot/4.0b; +http://www.seoprofiler.com/bot )", "Mozilla/5.0 (compatible; spbot/4.1.0; +http://OpenLinkProfiler.org/bot )", "Mozilla/5.0 (compatible; spbot/4.2.0; +http://OpenLinkProfiler.org/bot )", "Mozilla/5.0 (compatible; spbot/4.3.0; +http://OpenLinkProfiler.org/bot )", "Mozilla/5.0 (compatible; spbot/4.4.0; +http://OpenLinkProfiler.org/bot )", "Mozilla/5.0 (compatible; spbot/4.4.1; +http://OpenLinkProfiler.org/bot )", "Mozilla/5.0 (compatible; spbot/4.4.2; +http://OpenLinkProfiler.org/bot )", "Mozilla/5.0 (compatible; spbot/5.0.1; +http://OpenLinkProfiler.org/bot )", "Mozilla/5.0 (compatible; spbot/5.0.2; +http://OpenLinkProfiler.org/bot )", "Mozilla/5.0 (compatible; spbot/5.0.3; +http://OpenLinkProfiler.org/bot )", "Mozilla/5.0 (compatible; spbot/5.0; +http://OpenLinkProfiler.org/bot )" ], "description": "SEO Profiler web crawler for analysis", "tags": [ "seo" ] }, { "pattern": "twengabot", "addition_date": "2010/08/03", "url": "http://www.twenga.com/bot.html", "instances": [], "description": "Twenga shopping web crawler bot", "tags": [ "seo" ] }, { "pattern": "postrank", "addition_date": "2010/08/03", "url": "http://www.postrank.com", "instances": [ "PostRank/2.0 (postrank.com)", "PostRank/2.0 (postrank.com; 1 subscribers)" ], "description": "PostRank web crawler for content ranking", "tags": [ "seo" ] }, { "pattern": "Turnitin", "addition_date": "2010/09/26", "url": "http://www.turnitin.com", "instances": [ "TurnitinBot (https://turnitin.com/robot/crawlerinfo.html)", "Turnitin (https://bit.ly/2UvnfoQ)" ], "description": "Turnitin plagiarism detection web crawler", "tags": [ "academic" ] }, { "pattern": "scribdbot", "addition_date": "2010/09/28", "url": "http://www.scribd.com", "instances": [], "description": "Scribd document web crawler bot", "tags": [ "search-engine" ] }, { "pattern": "page2rss", "addition_date": "2010/10/07", "url": "http://www.page2rss.com", "instances": [ "Mozilla/5.0 (compatible; Page2RSS/0.7; +http://page2rss.com/)" ], "description": "Page2RSS web crawler for RSS conversion", "tags": [ "feed-reader" ] }, { "pattern": "sitebot", "addition_date": "2010/12/15", "url": "http://www.sitebot.org", "instances": [ "Mozilla/5.0 (compatible; Whoiswebsitebot/0.1; +http://www.whoiswebsite.net)" ], "description": "Sitebot web crawler for site analysis", "tags": [ "seo" ] }, { "pattern": "linkdex", "addition_date": "2011/01/06", "url": "http://www.linkdex.com", "instances": [ "Mozilla/5.0 (compatible; linkdexbot/2.0; +http://www.linkdex.com/about/bots/)", "Mozilla/5.0 (compatible; linkdexbot/2.0; +http://www.linkdex.com/bots/)", "Mozilla/5.0 (compatible; linkdexbot/2.1; +http://www.linkdex.com/about/bots/)", "Mozilla/5.0 (compatible; linkdexbot/2.1; +http://www.linkdex.com/bots/)", "Mozilla/5.0 (compatible; linkdexbot/2.2; +http://www.linkdex.com/bots/)", "linkdex.com/v2.0", "linkdexbot/Nutch-1.0-dev (http://www.linkdex.com/; crawl at linkdex dot com)" ], "description": "Linkdex SEO tool web crawler for link analysis", "tags": [ "seo" ] }, { "pattern": "Adidxbot", "url": "https://www.bing.com/webmasters/help/which-crawlers-does-bing-use-8c184ec0", "instances": [], "description": "Bing's advertising index web crawler bot", "tags": [ "advertising", "search-engine" ] }, { "pattern": "ezooms", "addition_date": "2011/04/27", "url": "http://www.phpbb.com/community/viewtopic.php?f=64&t=935605&start=450#p12948289", "instances": [ "Mozilla/5.0 (compatible; Ezooms/1.0; ezooms.bot@gmail.com)" ], "description": "Ezooms search engine web crawler bot", "tags": [ "seo" ] }, { "pattern": "dotbot", "addition_date": "2011/04/27", "instances": [ "Mozilla/5.0 (compatible; DotBot/1.1; http://www.opensiteexplorer.org/dotbot, help@moz.com)", "dotbot" ], "description": "Moz DotBot web crawler for SEO analysis", "tags": [ "seo" ] }, { "pattern": "Mail\\.RU_Bot", "addition_date": "2011/04/27", "instances": [ "Mozilla/5.0 (compatible; Linux x86_64; Mail.RU_Bot/2.0; +http://go.mail.ru/help/robots)", "Mozilla/5.0 (compatible; Linux x86_64; Mail.RU_Bot/2.0; +http://go.mail.ru/", "Mozilla/5.0 (compatible; Mail.RU_Bot/2.0; +http://go.mail.ru/", "Mozilla/5.0 (compatible; Linux x86_64; Mail.RU_Bot/Robots/2.0; +http://go.mail.ru/help/robots)" ], "description": "Mail.RU search engine web crawler bot", "tags": [ "search-engine" ] }, { "pattern": "discobot", "addition_date": "2011/05/03", "url": "http://discoveryengine.com/discobot.html", "instances": [ "Mozilla/5.0 (compatible; discobot/1.0; +http://discoveryengine.com/discobot.html)", "Mozilla/5.0 (compatible; discobot/2.0; +http://discoveryengine.com/discobot.html)", "mozilla/5.0 (compatible; discobot/1.1; +http://discoveryengine.com/discobot.html)" ], "description": "Discovery Engine web crawler for content discovery", "tags": [ "search-engine" ] }, { "pattern": "heritrix", "addition_date": "2011/06/21", "url": "https://github.com/internetarchive/heritrix3/wiki", "instances": [ "Mozilla/5.0 (compatible; heritrix/1.12.1 +http://www.webarchiv.cz)", "Mozilla/5.0 (compatible; heritrix/1.12.1b +http://netarkivet.dk/website/info.html)", "Mozilla/5.0 (compatible; heritrix/1.14.2 +http://rjpower.org)", "Mozilla/5.0 (compatible; heritrix/1.14.2 +http://www.webarchiv.cz)", "Mozilla/5.0 (compatible; heritrix/1.14.3 +http://archive.org)", "Mozilla/5.0 (compatible; heritrix/1.14.3 +http://www.accelobot.com)", "Mozilla/5.0 (compatible; heritrix/1.14.3 +http://www.webarchiv.cz)", "Mozilla/5.0 (compatible; heritrix/1.14.3.r6601 +http://www.buddybuzz.net/yptrino)", "Mozilla/5.0 (compatible; heritrix/1.14.4 +http://parsijoo.ir)", "Mozilla/5.0 (compatible; heritrix/1.14.4 +http://www.exif-search.com)", "Mozilla/5.0 (compatible; heritrix/2.0.2 +http://aihit.com)", "Mozilla/5.0 (compatible; heritrix/2.0.2 +http://seekda.com)", "Mozilla/5.0 (compatible; heritrix/3.0.0-SNAPSHOT-20091120.021634 +http://crawler.archive.org)", "Mozilla/5.0 (compatible; heritrix/3.1.0-RC1 +http://boston.lti.cs.cmu.edu/crawler_12/)", "Mozilla/5.0 (compatible; heritrix/3.1.1 +http://places.tomtom.com/crawlerinfo)", "Mozilla/5.0 (compatible; heritrix/3.1.1 +http://www.mixdata.com)", "Mozilla/5.0 (compatible; heritrix/3.1.1; UniLeipzigASV +http://corpora.informatik.uni-leipzig.de/crawler_faq.html)", "Mozilla/5.0 (compatible; heritrix/3.2.0 +http://www.crim.ca)", "Mozilla/5.0 (compatible; heritrix/3.2.0 +http://www.exif-search.com)", "Mozilla/5.0 (compatible; heritrix/3.2.0 +http://www.mixdata.com)", "Mozilla/5.0 (compatible; heritrix/3.3.0-SNAPSHOT-20160309-0050; UniLeipzigASV +http://corpora.informatik.uni-leipzig.de/crawler_faq.html)", "Mozilla/5.0 (compatible; sukibot_heritrix/3.1.1 +http://suki.ling.helsinki.fi/eng/webmasters.html)" ], "description": "Internet Archive's Heritrix web crawler framework", "tags": [ "archiver" ] }, { "pattern": "findthatfile", "addition_date": "2011/06/21", "url": "http://www.findthatfile.com/", "instances": [], "description": "FindThatFile web crawler for file discovery", "tags": [ "search-engine" ] }, { "pattern": "europarchive\\.org", "addition_date": "2011/06/21", "url": "", "instances": [ "Mozilla/5.0 (compatible; MSIE 7.0 +http://www.europarchive.org)" ], "description": "European Archive web crawler for preservation", "tags": [ "archiver" ] }, { "pattern": "NerdByNature\\.Bot", "addition_date": "2011/07/12", "url": "http://www.nerdbynature.net/bot", "instances": [ "Mozilla/5.0 (compatible; NerdByNature.Bot; http://www.nerdbynature.net/bot)" ], "description": "NerdByNature web crawler for content indexing", "tags": [ "seo" ] }, { "pattern": "(sistrix|SISTRIX) [cC]rawler", "addition_date": "2011/08/02", "url": "https://www.sistrix.com/tutorials/crawling-errors-in-the-optimizer/", "instances": [ "Mozilla/5.0 (compatible; SISTRIX Crawler; http://crawler.sistrix.net/)" ], "description": "SISTRIX SEO tool web crawler for analysis", "tags": [ "seo" ] }, { "pattern": "Ahrefs(Bot|SiteAudit)", "addition_date": "2011/08/28", "instances": [ "Mozilla/5.0 (compatible; AhrefsBot/6.1; +http://ahrefs.com/robot/)", "Mozilla/5.0 (compatible; AhrefsSiteAudit/6.1; +http://ahrefs.com/robot/)", "Mozilla/5.0 (compatible; AhrefsBot/5.2; News; +http://ahrefs.com/robot/)", "Mozilla/5.0 (compatible; AhrefsBot/5.2; +http://ahrefs.com/robot/)", "Mozilla/5.0 (compatible; AhrefsSiteAudit/5.2; +http://ahrefs.com/robot/)", "Mozilla/5.0 (compatible; AhrefsBot/6.1; News; +http://ahrefs.com/robot/)", "Mozilla/5.0 (compatible; AhrefsBot/7.0; +http://ahrefs.com/robot/)" ], "description": "Ahrefs SEO tool web crawler for link analysis", "tags": [ "seo" ] }, { "pattern": "fuelbot", "addition_date": "2018/06/28", "instances": [ "fuelbot" ], "description": "Fuel web crawler for content discovery", "tags": [ "seo" ] }, { "pattern": "^CrunchBot", "addition_date": "2018/06/28", "instances": [ "CrunchBot/1.0 (+http://www.leadcrunch.com/crunchbot)" ], "description": "LeadCrunch web crawler for lead generation", "tags": [ "seo" ] }, { "pattern": "IndeedBot", "addition_date": "2018/06/28", "instances": [ "Mozilla/5.0 (Windows NT 6.1; rv:38.0) Gecko/20100101 Firefox/38.0 (IndeedBot 1.1)" ], "description": "Indeed job search web crawler bot", "tags": [ "search-engine" ] }, { "pattern": "mappydata", "addition_date": "2018/06/28", "instances": [ "Mozilla/5.0 (compatible; Mappy/1.0; +http://mappydata.net/bot/)" ], "description": "Mappy web crawler for mapping data", "tags": [ "seo" ] }, { "pattern": "woobot", "addition_date": "2018/06/28", "instances": [ "woobot" ], "description": "Woo web crawler for content analysis", "tags": [ "seo" ] }, { "pattern": "ZoominfoBot", "addition_date": "2018/06/28", "instances": [ "ZoominfoBot (zoominfobot at zoominfo dot com)" ], "description": "ZoomInfo web crawler for business intelligence", "tags": [ "seo" ] }, { "pattern": "PrivacyAwareBot", "addition_date": "2018/06/28", "instances": [ "Mozilla/5.0 (compatible; PrivacyAwareBot/1.1; +http://www.privacyaware.org)" ], "description": "PrivacyAware web crawler for privacy analysis", "tags": [ "seo" ] }, { "pattern": "Multiviewbot", "addition_date": "2018/06/28", "instances": [ "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Multiviewbot" ], "description": "Multiview web crawler for content discovery", "tags": [ "seo" ] }, { "pattern": "SWIMGBot", "addition_date": "2018/06/28", "instances": [ "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/45.0.2454.101 Safari/537.36 SWIMGBot" ], "description": "SWIMG web crawler for image discovery", "tags": [ "seo" ] }, { "pattern": "Grobbot", "addition_date": "2018/06/28", "instances": [ "Mozilla/5.0 (compatible; Grobbot/2.2; +https://grob.it)" ], "description": "Grob web crawler for content analysis", "tags": [ "seo" ] }, { "pattern": "eright", "addition_date": "2018/06/28", "instances": [ "Mozilla/5.0 (compatible; eright/1.0; +bot@eright.com)" ], "description": "Eright web crawler for content discovery", "tags": [ "seo" ] }, { "pattern": "Apercite", "addition_date": "2018/06/28", "instances": [ "Mozilla/5.0 (compatible; Apercite; +http://www.apercite.fr/robot/index.html)" ], "description": "Apercite web crawler for content analysis", "tags": [ "seo" ] }, { "pattern": "semanticbot", "addition_date": "2018/06/28", "instances": [ "semanticbot", "semanticbot (info@semanticaudience.com)" ], "description": "Semantic web crawler for content analysis", "tags": [ "seo" ] }, { "pattern": "Aboundex", "addition_date": "2011/09/28", "url": "http://www.aboundex.com/crawler/", "instances": [ "Aboundex/0.2 (http://www.aboundex.com/crawler/)", "Aboundex/0.3 (http://www.aboundex.com/crawler/)" ], "description": "Aboundex web crawler for content discovery", "tags": [ "search-engine" ] }, { "pattern": "domaincrawler", "addition_date": "2011/10/21", "instances": [ "CipaCrawler/3.0 (info@domaincrawler.com; http://www.domaincrawler.com/www.example.com)" ], "description": "Domain Crawler web crawler for analysis", "tags": [ "seo" ] }, { "pattern": "wbsearchbot", "addition_date": "2011/12/21", "url": "http://www.warebay.com/bot.html", "instances": [], "description": "Warebay search web crawler bot", "tags": [ "search-engine" ] }, { "pattern": "summify", "addition_date": "2012/01/04", "url": "http://summify.com", "instances": [ "Summify (Summify/1.0.1; +http://summify.com)" ], "description": "Summify web crawler for content summarization", "tags": [ "seo" ] }, { "pattern": "CCBot", "addition_date": "2012/02/05", "url": "http://www.commoncrawl.org/bot.html", "instances": [ "CCBot/2.0 (http://commoncrawl.org/faq/)", "CCBot/2.0 (https://commoncrawl.org/faq/)" ], "description": "Common Crawl web crawler for indexing", "tags": [ "ai-crawler" ] }, { "pattern": "edisterbot", "addition_date": "2012/02/25", "instances": [], "description": "Edister web crawler for content discovery", "tags": [ "search-engine" ] }, { "pattern": "SeznamBot", "addition_date": "2012/03/14", "instances": [ "Mozilla/5.0 (compatible; SeznamBot/3.2-test1-1; +http://napoveda.seznam.cz/en/seznambot-intro/)", "Mozilla/5.0 (compatible; SeznamBot/3.2-test1; +http://napoveda.seznam.cz/en/seznambot-intro/)", "Mozilla/5.0 (compatible; SeznamBot/3.2-test2; +http://napoveda.seznam.cz/en/seznambot-intro/)", "Mozilla/5.0 (compatible; SeznamBot/3.2-test4; +http://napoveda.seznam.cz/en/seznambot-intro/)", "Mozilla/5.0 (compatible; SeznamBot/3.2; +http://napoveda.seznam.cz/en/seznambot-intro/)", "Mozilla/5.0 (compatible; SeznamBot/4.0; +http://napoveda.seznam.cz/seznambot-intro/)" ], "description": "Seznam search engine web crawler bot", "tags": [ "search-engine" ] }, { "pattern": "ec2linkfinder", "addition_date": "2012/03/22", "instances": [ "ec2linkfinder" ], "description": "EC2 link finder web crawler bot", "tags": [ "scanner" ] }, { "pattern": "gslfbot", "addition_date": "2012/04/03", "instances": [], "description": "GSLFBOT web crawler for content discovery", "tags": [ "seo" ] }, { "pattern": "aiHitBot", "addition_date": "2012/04/16", "instances": [ "Mozilla/5.0 (compatible; aiHitBot/2.9; +https://www.aihitdata.com/about)" ], "description": "AiHit web crawler for data collection", "tags": [ "search-engine" ] }, { "pattern": "intelium_bot", "addition_date": "2012/05/07", "instances": [], "description": "Intelium web crawler for content discovery", "tags": [ "seo" ] }, { "pattern": "facebookexternalhit", "addition_date": "2012/05/07", "instances": [ "facebookexternalhit/1.0 (+http://www.facebook.com/externalhit_uatext.php)", "facebookexternalhit/1.1", "facebookexternalhit/1.1 (+http://www.facebook.com/externalhit_uatext.php)" ], "url": "https://developers.facebook.com/docs/sharing/webmasters/crawler/", "description": "Facebook external hit web crawler bot", "tags": [ "social-preview" ] }, { "pattern": "Yeti", "addition_date": "2012/05/07", "url": "http://naver.me/bot", "instances": [ "Mozilla/5.0 (compatible; Yeti/1.1; +http://naver.me/bot)" ], "description": "Naver Yeti search engine web crawler bot", "tags": [ "search-engine" ] }, { "pattern": "RetrevoPageAnalyzer", "addition_date": "2012/05/07", "instances": [ "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; RetrevoPageAnalyzer; +http://www.retrevo.com/content/about-us)" ], "description": "Retrevo page analyzer web crawler bot", "tags": [ "seo" ] }, { "pattern": "lb-spider", "addition_date": "2012/05/07", "instances": [], "description": "LB spider web crawler for content discovery", "tags": [ "search-engine" ] }, { "pattern": "Sogou", "addition_date": "2012/05/13", "url": "http://www.sogou.com/docs/help/webmasters.htm#07", "instances": [ "Sogou News Spider/4.0(+http://www.sogou.com/docs/help/webmasters.htm#07)", "Sogou Pic Spider/3.0(+http://www.sogou.com/docs/help/webmasters.htm#07)", "Sogou web spider/4.0(+http://www.sogou.com/docs/help/webmasters.htm#07)" ], "description": "Sogou search engine web crawler bot", "tags": [ "search-engine" ] }, { "pattern": "lssbot", "addition_date": "2012/05/15", "url": "https://www.lssbot.com/", "instances": [], "description": "LSS web crawler for content discovery", "tags": [ "seo" ] }, { "pattern": "careerbot", "addition_date": "2012/05/23", "url": "http://www.career-x.de/bot.html", "instances": [], "description": "Career-X web crawler for job discovery", "tags": [ "seo" ] }, { "pattern": "wotbox", "addition_date": "2012/06/12", "url": "http://www.wotbox.com", "instances": [ "Wotbox/2.0 (bot@wotbox.com; http://www.wotbox.com)", "Wotbox/2.01 (+http://www.wotbox.com/bot/)" ], "description": "Wotbox web crawler for content discovery", "tags": [ "search-engine" ] }, { "pattern": "wocbot", "addition_date": "2012/07/25", "url": "http://www.wocodi.com/crawler", "instances": [], "description": "Wocodi web crawler for content discovery", "tags": [ "search-engine" ] }, { "pattern": "ichiro", "addition_date": "2012/08/28", "url": "http://help.goo.ne.jp/help/article/1142", "instances": [ "DoCoMo/2.0 P900i(c100;TB;W24H11) (compatible; ichiro/mobile goo; +http://help.goo.ne.jp/help/article/1142/)", "DoCoMo/2.0 P900i(c100;TB;W24H11) (compatible; ichiro/mobile goo; +http://search.goo.ne.jp/option/use/sub4/sub4-1/)", "DoCoMo/2.0 P900i(c100;TB;W24H11) (compatible; ichiro/mobile goo;+http://search.goo.ne.jp/option/use/sub4/sub4-1/)", "DoCoMo/2.0 P900i(c100;TB;W24H11)(compatible; ichiro/mobile goo;+http://help.goo.ne.jp/door/crawler.html)", "DoCoMo/2.0 P901i(c100;TB;W24H11) (compatible; ichiro/mobile goo; +http://help.goo.ne.jp/door/crawler.html)", "KDDI-CA31 UP.Browser/6.2.0.7.3.129 (GUI) MMP/2.0 (compatible; ichiro/mobile goo; +http://help.goo.ne.jp/help/article/1142/)", "KDDI-CA31 UP.Browser/6.2.0.7.3.129 (GUI) MMP/2.0 (compatible; ichiro/mobile goo; +http://search.goo.ne.jp/option/use/sub4/sub4-1/)", "KDDI-CA31 UP.Browser/6.2.0.7.3.129 (GUI) MMP/2.0 (compatible; ichiro/mobile goo;+http://search.goo.ne.jp/option/use/sub4/sub4-1/)", "ichiro/2.0 (http://help.goo.ne.jp/door/crawler.html)", "ichiro/2.0 (ichiro@nttr.co.jp)", "ichiro/3.0 (http://help.goo.ne.jp/door/crawler.html)", "ichiro/3.0 (http://help.goo.ne.jp/help/article/1142)", "ichiro/3.0 (http://search.goo.ne.jp/option/use/sub4/sub4-1/)", "ichiro/4.0 (http://help.goo.ne.jp/door/crawler.html)", "ichiro/5.0 (http://help.goo.ne.jp/door/crawler.html)" ], "description": "Goo ichiro search engine web crawler bot", "tags": [ "search-engine" ] }, { "pattern": "DuckDuckBot", "addition_date": "2012/09/19", "url": "http://duckduckgo.com/duckduckbot.html", "instances": [ "DuckDuckBot/1.0; (+http://duckduckgo.com/duckduckbot.html)", "DuckDuckBot/1.1; (+http://duckduckgo.com/duckduckbot.html)", "Mozilla/5.0 (compatible; DuckDuckBot-Https/1.1; https://duckduckgo.com/duckduckbot)", "'Mozilla/5.0 (compatible; DuckDuckBot-Https/1.1; https://duckduckgo.com/duckduckbot)'" ], "description": "DuckDuckGo search engine web crawler bot", "tags": [ "search-engine" ] }, { "pattern": "lssrocketcrawler", "addition_date": "2012/09/24", "instances": [], "description": "LSS Rocket web crawler for content", "tags": [ "seo" ] }, { "pattern": "drupact", "addition_date": "2012/09/27", "url": "http://www.arocom.de/drupact", "instances": [ "drupact/0.7; http://www.arocom.de/drupact" ], "description": "Drupact web crawler for content discovery", "tags": [ "seo" ] }, { "pattern": "webcompanycrawler", "addition_date": "2012/10/03", "instances": [], "description": "Web Company web crawler for analysis", "tags": [ "seo" ] }, { "pattern": "acoonbot", "addition_date": "2012/10/07", "url": "http://www.acoon.de/robot.asp", "instances": [], "description": "Acoon web crawler for content discovery", "tags": [ "search-engine" ] }, { "pattern": "openindexspider", "addition_date": "2012/10/26", "url": "http://www.openindex.io/en/webmasters/spider.html", "instances": [], "description": "OpenIndex web crawler for indexing", "tags": [ "search-engine" ] }, { "pattern": "gnam gnam spider", "addition_date": "2012/10/31", "instances": [], "description": "Gnam Gnam web crawler for discovery", "tags": [ "seo" ] }, { "pattern": "web-archive-net\\.com\\.bot", "instances": [], "description": "Web Archive web crawler for preservation", "tags": [ "archiver" ] }, { "pattern": "backlinkcrawler", "addition_date": "2013/01/04", "url": "http://www.backlinktest.com/crawler.html", "instances": [], "description": "Backlink Crawler web crawler for analysis", "tags": [ "seo" ] }, { "pattern": "coccoc", "addition_date": "2013/01/04", "url": "http://help.coccoc.vn/", "instances": [ "Mozilla/5.0 (compatible; coccoc/1.0; +http://help.coccoc.com/)", "Mozilla/5.0 (compatible; coccoc/1.0; +http://help.coccoc.com/searchengine)", "Mozilla/5.0 (compatible; coccocbot-image/1.0; +http://help.coccoc.com/searchengine)", "Mozilla/5.0 (compatible; coccocbot-web/1.0; +http://help.coccoc.com/searchengine)", "Mozilla/5.0 (compatible; image.coccoc/1.0; +http://help.coccoc.com/)", "Mozilla/5.0 (compatible; imagecoccoc/1.0; +http://help.coccoc.com/)", "Mozilla/5.0 (compatible; imagecoccoc/1.0; +http://help.coccoc.com/searchengine)", "coccoc", "coccoc/1.0 ()", "coccoc/1.0 (http://help.coccoc.com/)", "coccoc/1.0 (http://help.coccoc.vn/)" ], "description": "Coccoc search engine web crawler bot", "tags": [ "search-engine" ] }, { "pattern": "integromedb", "addition_date": "2013/01/10", "url": "http://www.integromedb.org/Crawler", "instances": [ "www.integromedb.org/Crawler" ], "description": "IntegromeDB web crawler for research", "tags": [ "academic" ] }, { "pattern": "content crawler spider", "addition_date": "2013/01/11", "instances": [], "description": "Content Crawler web crawler for discovery", "tags": [ "seo" ] }, { "pattern": "toplistbot", "addition_date": "2013/02/05", "instances": [], "description": "TopList web crawler for ranking", "tags": [ "seo" ] }, { "pattern": "it2media-domain-crawler", "addition_date": "2013/03/12", "instances": [ "it2media-domain-crawler/1.0 on crawler-prod.it2media.de", "it2media-domain-crawler/2.0" ], "description": "IT2Media domain web crawler bot", "tags": [ "seo" ] }, { "pattern": "ip-web-crawler\\.com", "addition_date": "2013/03/22", "instances": [], "description": "IP Web Crawler web crawler bot", "tags": [ "scanner" ] }, { "pattern": "siteexplorer\\.info", "addition_date": "2013/05/01", "instances": [ "Mozilla/5.0 (compatible; SiteExplorer/1.0b; +http://siteexplorer.info/)", "Mozilla/5.0 (compatible; SiteExplorer/1.1b; +http://siteexplorer.info/Backlink-Checker-Spider/)" ], "description": "Site Explorer web crawler for analysis", "tags": [ "seo" ] }, { "pattern": "elisabot", "addition_date": "2013/06/27", "instances": [], "description": "Elisa web crawler for content discovery", "tags": [ "seo" ] }, { "pattern": "proximic", "addition_date": "2013/09/12", "url": "http://www.proximic.com/info/spider.php", "instances": [ "Mozilla/5.0 (compatible; proximic; +http://www.proximic.com)", "Mozilla/5.0 (compatible; proximic; +http://www.proximic.com/info/spider.php)" ], "description": "Proximic web crawler for content analysis", "tags": [ "advertising" ] }, { "pattern": "changedetection", "addition_date": "2013/09/13", "url": "http://www.changedetection.com/bot.html", "instances": [ "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; http://www.changedetection.com/bot.html )" ], "description": "ChangeDetection web crawler for monitoring", "tags": [ "monitoring" ] }, { "pattern": "arabot", "addition_date": "2013/10/09", "instances": [], "description": "Arabot web crawler for content discovery", "tags": [ "search-engine" ] }, { "pattern": "WeSEE:Search", "addition_date": "2013/11/18", "instances": [ "WeSEE:Search", "WeSEE:Search/0.1 (Alpha, http://www.wesee.com/en/support/bot/)" ], "description": "WeSEE search engine web crawler bot", "tags": [ "search-engine" ] }, { "pattern": "niki-bot", "addition_date": "2014/01/01", "instances": [], "description": "Niki web crawler for content discovery", "tags": [ "seo" ] }, { "pattern": "CrystalSemanticsBot", "addition_date": "2014/02/17", "url": "http://www.crystalsemantics.com/user-agent/", "instances": [], "description": "Crystal Semantics web crawler bot", "tags": [ "seo" ] }, { "pattern": "rogerbot", "addition_date": "2014/02/28", "url": "http://moz.com/help/pro/what-is-rogerbot-", "instances": [ "Mozilla/5.0 (compatible; rogerBot/1.0; UrlCrawler; http://www.seomoz.org/dp/rogerbot)", "rogerbot/1.0 (http://moz.com/help/pro/what-is-rogerbot-, rogerbot-crawler+partager@moz.com)", "rogerbot/1.0 (http://moz.com/help/pro/what-is-rogerbot-, rogerbot-crawler+shiny@moz.com)", "rogerbot/1.0 (http://moz.com/help/pro/what-is-rogerbot-, rogerbot-wherecat@moz.com", "rogerbot/1.0 (http://moz.com/help/pro/what-is-rogerbot-, rogerbot-wherecat@moz.com)", "rogerbot/1.0 (http://www.moz.com/dp/rogerbot, rogerbot-crawler@moz.com)", "rogerbot/1.0 (http://www.seomoz.org/dp/rogerbot, rogerbot-crawler+shiny@seomoz.org)", "rogerbot/1.0 (http://www.seomoz.org/dp/rogerbot, rogerbot-crawler@seomoz.org)", "rogerbot/1.0 (http://www.seomoz.org/dp/rogerbot, rogerbot-wherecat@moz.com)", "rogerbot/1.1 (http://moz.com/help/guides/search-overview/crawl-diagnostics#more-help, rogerbot-crawler+pr2-crawler-05@moz.com)", "rogerbot/1.1 (http://moz.com/help/guides/search-overview/crawl-diagnostics#more-help, rogerbot-crawler+pr4-crawler-11@moz.com)", "rogerbot/1.1 (http://moz.com/help/guides/search-overview/crawl-diagnostics#more-help, rogerbot-crawler+pr4-crawler-15@moz.com)", "rogerbot/1.2 (http://moz.com/help/pro/what-is-rogerbot-, rogerbot-crawler+phaser-testing-crawler-01@moz.com)" ], "description": "Moz RogerBot web crawler for analysis", "tags": [ "seo" ] }, { "pattern": "360Spider", "addition_date": "2014/03/14", "url": "http://needs-be.blogspot.co.uk/2013/02/how-to-block-spider360.html", "instances": [ "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/21.0.1180.89 Safari/537.1; 360Spider", "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/21.0.1180.89 Safari/537.1; 360Spider(compatible; HaosouSpider; http://www.haosou.com/help/help_3_2.html)", "Mozilla/5.0 (Windows NT 6.2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/31.0.1650.63 Safari/537.36 QIHU 360SE; 360Spider", "Mozilla/5.0 (Windows; U; Windows NT 5.1; zh-CN; ) Firefox/1.5.0.11; 360Spider", "Mozilla/5.0 (Windows; U; Windows NT 5.1; zh-CN; rv:1.8.0.11) Firefox/1.5.0.11; 360Spider", "Mozilla/5.0 (Windows; U; Windows NT 5.1; zh-CN; rv:1.8.0.11) Firefox/1.5.0.11 360Spider;", "Mozilla/5.0 (Windows; U; Windows NT 5.1; zh-CN; rv:1.8.0.11) Gecko/20070312 Firefox/1.5.0.11; 360Spider", "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0); 360Spider", "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0); 360Spider(compatible; HaosouSpider; http://www.haosou.com/help/help_3_2.html)", "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/50.0.2661.102 Safari/537.36; 360Spider" ], "description": "360 search engine web crawler bot", "tags": [ "search-engine" ] }, { "pattern": "psbot", "addition_date": "2014/03/31", "url": "http://www.picsearch.com/bot.html", "instances": [ "psbot-image (+http://www.picsearch.com/bot.html)", "psbot-page (+http://www.picsearch.com/bot.html)", "psbot/0.1 (+http://www.picsearch.com/bot.html)" ], "description": "PicSearch web crawler for image discovery", "tags": [ "search-engine" ] }, { "pattern": "InterfaxScanBot", "addition_date": "2014/03/31", "url": "http://scan-interfax.ru", "instances": [], "description": "Interfax scan web crawler bot", "tags": [ "seo" ] }, { "pattern": "CC Metadata Scaper", "addition_date": "2014/04/01", "url": "http://wiki.creativecommons.org/Metadata_Scraper", "instances": [ "CC Metadata Scaper http://wiki.creativecommons.org/Metadata_Scraper" ], "description": "Creative Commons metadata web crawler", "tags": [ "archiver" ] }, { "pattern": "g00g1e\\.net", "addition_date": "2014/04/01", "url": "http://www.g00g1e.net/", "instances": [], "description": "G00g1e web crawler for content discovery", "tags": [ "seo" ] }, { "pattern": "GrapeshotCrawler", "addition_date": "2014/04/01", "url": "http://www.grapeshot.co.uk/crawler.php", "instances": [ "Mozilla/5.0 (compatible; GrapeshotCrawler/2.0; +http://www.grapeshot.co.uk/crawler.php)" ], "description": "Grapeshot web crawler for content analysis", "tags": [ "advertising" ] }, { "pattern": "urlappendbot", "addition_date": "2014/05/10", "url": "http://www.profound.net/urlappendbot.html", "instances": [ "Mozilla/5.0 (compatible; URLAppendBot/1.0; +http://www.profound.net/urlappendbot.html)" ], "description": "URL Append web crawler for analysis", "tags": [ "seo" ] }, { "pattern": "brainobot", "addition_date": "2014/06/24", "instances": [], "description": "Braino web crawler for content discovery", "tags": [ "seo" ] }, { "pattern": "fr-crawler", "addition_date": "2014/07/31", "instances": [ "Mozilla/5.0 (compatible; fr-crawler/1.1)" ], "description": "FR Crawler web crawler for content discovery", "tags": [ "search-engine" ] }, { "pattern": "binlar", "addition_date": "2014/09/12", "instances": [ "binlar_2.6.3 binlar2.6.3@unspecified.mail", "binlar_2.6.3 binlar_2.6.3@unspecified.mail", "binlar_2.6.3 larbin2.6.3@unspecified.mail", "binlar_2.6.3 phanendra_kalapala@McAfee.com", "binlar_2.6.3 test@mgmt.mic" ], "description": "Binlar web crawler for content discovery", "tags": [ "scanner" ] }, { "pattern": "SimpleCrawler", "addition_date": "2014/09/12", "instances": [ "SimpleCrawler/0.1" ], "description": "Simple Crawler web crawler framework", "tags": [ "http-library" ] }, { "pattern": "Twitterbot", "addition_date": "2014/09/12", "url": "https://dev.twitter.com/cards/getting-started", "instances": [ "Twitterbot/0.1", "Twitterbot/1.0" ], "description": "Twitter web crawler for link previews", "tags": [ "social-preview" ] }, { "pattern": "cXensebot", "addition_date": "2014/10/05", "instances": [ "cXensebot/1.1a" ], "url": "http://www.cxense.com/bot.html", "description": "CXense web crawler for content analysis", "tags": [ "advertising" ] }, { "pattern": "smtbot", "addition_date": "2014/10/04", "instances": [ "Mozilla/5.0 (compatible; SMTBot/1.0; +http://www.similartech.com/smtbot)", "SMTBot (similartech.com/smtbot)", "Mozilla/5.0 (iPhone; CPU iPhone OS 6_0 like Mac OS X) AppleWebKit/536.26 (KHTML, like Gecko) Version/6.0 Mobile/10A5376e Safari/8536.25 (compatible; SMTBot/1.0; +http://www.similartech.com/smtbot)", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.75 Safari/537.36 (compatible; SMTBot/1.0; +http://www.similartech.com/smtbot)", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.75 Safari/537.36 (compatible; SMTBot/1.0; http://www.similartech.com/smtbot)" ], "url": "http://www.similartech.com/smtbot", "description": "SimilarTech web crawler for technology detection", "tags": [ "seo" ] }, { "pattern": "bnf\\.fr_bot", "addition_date": "2014/11/18", "url": "http://www.bnf.fr/fr/outils/a.dl_web_capture_robot.html", "instances": [ "Mozilla/5.0 (compatible; bnf.fr_bot; +http://bibnum.bnf.fr/robot/bnf.html)", "Mozilla/5.0 (compatible; bnf.fr_bot; +http://www.bnf.fr/fr/outils/a.dl_web_capture_robot.html)" ], "description": "BNF French National Library web crawler", "tags": [ "archiver" ] }, { "pattern": "A6-Indexer", "addition_date": "2014/12/05", "url": "http://www.a6corp.com/a6-web-scraping-policy/", "instances": [ "A6-Indexer" ], "description": "A6 Corporation web crawler for indexing", "tags": [ "search-engine" ] }, { "pattern": "ADmantX", "addition_date": "2014/12/05", "url": "http://www.admantx.com", "instances": [ "ADmantX Platform Semantic Analyzer - ADmantX Inc. - www.admantx.com - support@admantx.com" ], "description": "ADmantX semantic analyzer web crawler", "tags": [ "advertising" ] }, { "pattern": "Facebot", "url": "https://developers.facebook.com/docs/sharing/best-practices#crawl", "addition_date": "2014/12/30", "instances": [ "Facebot/1.0" ], "description": "Facebook's web crawler for social sharing", "tags": [ "social-preview" ] }, { "pattern": "OrangeBot\\/", "instances": [ "Mozilla/5.0 (compatible; OrangeBot/2.0; support.orangebot@orange.com" ], "addition_date": "2015/01/12", "description": "Orange search engine web crawler bot", "tags": [ "search-engine" ] }, { "pattern": "memorybot", "url": "http://mignify.com/bot.htm", "instances": [ "Mozilla/5.0 (compatible; memorybot/1.21.14 +http://mignify.com/bot.html)" ], "addition_date": "2015/02/01", "description": "Mignify memory web crawler bot", "tags": [ "seo" ] }, { "pattern": "AdvBot", "url": "http://advbot.net/bot.html", "instances": [ "Mozilla/5.0 (compatible; AdvBot/2.0; +http://advbot.net/bot.html)" ], "addition_date": "2015/02/01", "description": "AdvBot web crawler for advertising analysis", "tags": [ "advertising" ] }, { "pattern": "MegaIndex", "url": "https://www.megaindex.ru/?tab=linkAnalyze", "instances": [ "Mozilla/5.0 (compatible; MegaIndex.ru/2.0; +https://www.megaindex.ru/?tab=linkAnalyze)", "Mozilla/5.0 (compatible; MegaIndex.ru/2.0; +http://megaindex.com/crawler)" ], "addition_date": "2015/03/28", "description": "MegaIndex SEO tool web crawler for analysis", "tags": [ "seo" ] }, { "pattern": "SemanticScholarBot", "url": "https://www.semanticscholar.org/crawler", "instances": [ "SemanticScholarBot/1.0 (+http://s2.allenai.org/bot.html)", "Mozilla/5.0 (compatible) SemanticScholarBot (+https://www.semanticscholar.org/crawler)" ], "addition_date": "2015/03/28", "description": "Semantic Scholar web crawler for academic content", "tags": [ "academic" ] }, { "pattern": "ltx71", "url": "http://ltx71.com/", "instances": [ "ltx71 - (http://ltx71.com/)" ], "addition_date": "2015/04/04", "description": "LTX71 web crawler for content discovery", "tags": [ "scanner" ] }, { "pattern": "nerdybot", "url": "http://nerdybot.com/", "instances": [ "nerdybot" ], "addition_date": "2015/04/05", "description": "NerdyBot web crawler for content analysis", "tags": [ "seo" ] }, { "pattern": "xovibot", "url": "http://www.xovibot.net/", "instances": [ "Mozilla/5.0 (compatible; XoviBot/2.0; +http://www.xovibot.net/)" ], "addition_date": "2015/04/05", "description": "Xovi SEO tool web crawler for analysis", "tags": [ "seo" ] }, { "pattern": "BUbiNG", "url": "http://law.di.unimi.it/BUbiNG.html", "instances": [ "BUbiNG (+http://law.di.unimi.it/BUbiNG.html)" ], "addition_date": "2015/04/06", "description": "BUbiNG web crawler framework for research", "tags": [ "academic" ] }, { "pattern": "Qwantify", "url": "https://www.qwant.com/", "instances": [ "Mozilla/5.0 (compatible; Qwantify/2.0n; +https://www.qwant.com/)/*", "Mozilla/5.0 (compatible; Qwantify/2.4w; +https://www.qwant.com/)/2.4w", "Mozilla/5.0 (compatible; Qwantify/Bleriot/1.1; +https://help.qwant.com/bot)", "Mozilla/5.0 (compatible; Qwantify/Bleriot/1.2.1; +https://help.qwant.com/bot)" ], "addition_date": "2015/04/06", "description": "Qwant search engine web crawler bot", "tags": [ "search-engine" ] }, { "pattern": "archive\\.org_bot", "url": "http://www.archive.org/details/archive.org_bot", "depends_on": [ "heritrix" ], "instances": [ "Mozilla/5.0 (compatible; heritrix/3.1.1-SNAPSHOT-20120116.200628 +http://www.archive.org/details/archive.org_bot)", "Mozilla/5.0 (compatible; archive.org_bot/heritrix-1.15.4 +http://www.archive.org)", "Mozilla/5.0 (compatible; heritrix/3.3.0-SNAPSHOT-20140702-2247 +http://archive.org/details/archive.org_bot)", "Mozilla/5.0 (compatible; archive.org_bot +http://www.archive.org/details/archive.org_bot)", "Mozilla/5.0 (compatible; archive.org_bot +http://archive.org/details/archive.org_bot)", "Mozilla/5.0 (compatible; special_archiver/3.1.1 +http://www.archive.org/details/archive.org_bot)" ], "addition_date": "2015/04/14", "description": "Archive.org web crawler for preservation", "tags": [ "archiver" ] }, { "pattern": "Applebot", "url": "http://www.apple.com/go/applebot", "addition_date": "2015/04/15", "instances": [ "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/600.2.5 (KHTML, like Gecko) Version/8.0.2 Safari/600.2.5 (Applebot/0.1)", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/600.2.5 (KHTML, like Gecko) Version/8.0.2 Safari/600.2.5 (Applebot/0.1; +http://www.apple.com/go/applebot)", "Mozilla/5.0 (compatible; Applebot/0.3; +http://www.apple.com/go/applebot)", "Mozilla/5.0 (iPhone; CPU iPhone OS 6_0 like Mac OS X) AppleWebKit/536.26 (KHTML, like Gecko) Version/6.0 Mobile/10A5376e Safari/8536.25 (compatible; Applebot/0.3; +http://www.apple.com/go/applebot)", "Mozilla/5.0 (iPhone; CPU iPhone OS 8_1 like Mac OS X) AppleWebKit/600.1.4 (KHTML, like Gecko) Version/8.0 Mobile/12B410 Safari/600.1.4 (Applebot/0.1; +http://www.apple.com/go/applebot)" ], "description": "Apple's web crawler for Siri and search", "tags": [ "search-engine" ] }, { "pattern": "TweetmemeBot", "url": "http://datasift.com/bot.html", "instances": [ "Mozilla/5.0 (TweetmemeBot/4.0; +http://datasift.com/bot.html) Gecko/20100101 Firefox/31.0" ], "addition_date": "2015/04/15", "description": "TweetMeme web crawler for social content", "tags": [ "social-preview" ] }, { "pattern": "crawler4j", "url": "https://github.com/yasserg/crawler4j", "instances": [ "crawler4j (http://code.google.com/p/crawler4j/)", "crawler4j (https://github.com/yasserg/crawler4j/)" ], "addition_date": "2015/05/07", "description": "Crawler4j Java web crawler framework", "tags": [ "http-library" ] }, { "pattern": "findxbot", "url": "http://www.findxbot.com", "instances": [ "Mozilla/5.0 (compatible; Findxbot/1.0; +http://www.findxbot.com)" ], "addition_date": "2015/05/07", "description": "FindX web crawler for content discovery", "tags": [ "search-engine" ] }, { "pattern": "S[eE][mM]rushBot", "url": "http://www.semrush.com/bot.html", "instances": [ "Mozilla/5.0 (compatible; SemrushBot-SA/0.97; +http://www.semrush.com/bot.html)", "Mozilla/5.0 (compatible; SemrushBot-SI/0.97; +http://www.semrush.com/bot.html)", "Mozilla/5.0 (compatible; SemrushBot/3~bl; +http://www.semrush.com/bot.html)", "Mozilla/5.0 (compatible; SemrushBot/0.98~bl; +http://www.semrush.com/bot.html)", "Mozilla/5.0 (compatible; SemrushBot-BA; +http://www.semrush.com/bot.html)", "Mozilla/5.0 (compatible; SemrushBot/6~bl; +http://www.semrush.com/bot.html)", "Mozilla/5.0 (compatible; SemrushBot/7~bl; +http://www.semrush.com/bot.html)", "SEMrushBot" ], "addition_date": "2015/05/26", "description": "Semrush SEO tool web crawler for analysis", "tags": [ "seo" ] }, { "pattern": "yoozBot", "url": "http://yooz.ir", "instances": [ "Mozilla/5.0 (compatible; yoozBot-2.2; http://yooz.ir; info@yooz.ir)" ], "addition_date": "2015/05/26", "description": "Yooz web crawler for content analysis", "tags": [ "seo" ] }, { "pattern": "lipperhey", "url": "http://www.lipperhey.com/", "instances": [ "Mozilla/5.0 (compatible; Lipperhey Link Explorer; http://www.lipperhey.com/)", "Mozilla/5.0 (compatible; Lipperhey SEO Service; http://www.lipperhey.com/)", "Mozilla/5.0 (compatible; Lipperhey Site Explorer; http://www.lipperhey.com/)", "Mozilla/5.0 (compatible; Lipperhey-Kaus-Australis/5.0; +https://www.lipperhey.com/en/about/)" ], "addition_date": "2015/08/26", "description": "Lipperhey SEO tool web crawler for analysis", "tags": [ "seo" ] }, { "pattern": "Y!J", "url": "https://www.yahoo-help.jp/app/answers/detail/p/595/a_id/42716/~/%E3%82%A6%E3%82%A7%E3%83%96%E3%83%9A%E3%83%BC%E3%82%B8%E3%81%AB%E3%82%A2%E3%82%AF%E3%82%BB%E3%82%B9%E3%81%99%E3%82%8B%E3%82%B7%E3%82%B9%E3%83%86%E3%83%A0%E3%81%AE%E3%83%A6%E3%83%BC%E3%82%B6%E3%83%BC%E3%82%A8%E3%83%BC%E3%82%B8%E3%82%A7%E3%83%B3%E3%83%88%E3%81%AB%E3%81%A4%E3%81%84%E3%81%A6", "instances": [ "Y!J-ASR/0.1 crawler (http://www.yahoo-help.jp/app/answers/detail/p/595/a_id/42716/)", "Y!J-BRJ/YATS crawler (http://help.yahoo.co.jp/help/jp/search/indexing/indexing-15.html)", "Y!J-PSC/1.0 crawler (http://help.yahoo.co.jp/help/jp/search/indexing/indexing-15.html)", "Y!J-BRW/1.0 crawler (http://help.yahoo.co.jp/help/jp/search/indexing/indexing-15.html)", "Mozilla/5.0 (iPhone; Y!J-BRY/YATSH crawler; http://help.yahoo.co.jp/help/jp/search/indexing/indexing-15.html)", "Mozilla/5.0 (compatible; Y!J SearchMonkey/1.0 (Y!J-AGENT; http://help.yahoo.co.jp/help/jp/search/indexing/indexing-15.html))" ], "addition_date": "2015/05/26", "description": "Yahoo Japan search engine web crawler bot", "tags": [ "search-engine" ] }, { "pattern": "Domain Re-Animator Bot", "url": "http://domainreanimator.com", "instances": [ "Domain Re-Animator Bot (http://domainreanimator.com) - support@domainreanimator.com" ], "addition_date": "2015/04/14", "description": "Domain Re-Animator web crawler for domain analysis", "tags": [ "seo" ] }, { "pattern": "AddThis", "url": "https://www.addthis.com", "instances": [ "AddThis.com robot tech.support@clearspring.com" ], "addition_date": "2015/06/02", "description": "AddThis social sharing web crawler bot", "tags": [ "social-preview" ] }, { "pattern": "Screaming Frog SEO Spider", "url": "http://www.screamingfrog.co.uk/seo-spider", "instances": [ "Screaming Frog SEO Spider/5.1" ], "addition_date": "2016/01/08", "description": "Screaming Frog SEO tool web crawler", "tags": [ "seo" ] }, { "pattern": "MetaURI", "url": "http://www.useragentstring.com/MetaURI_id_17683.php", "instances": [ "MetaURI API/2.0 +metauri.com" ], "addition_date": "2016/01/02", "description": "MetaURI API web crawler for metadata", "tags": [ "seo" ] }, { "pattern": "Scrapy", "url": "http://scrapy.org/", "instances": [ "Scrapy/1.0.3 (+http://scrapy.org)" ], "addition_date": "2016/01/02", "description": "Scrapy Python web crawler framework", "tags": [ "http-library" ] }, { "pattern": "Livelap[bB]ot", "url": "http://site.livelap.com/crawler", "instances": [ "LivelapBot/0.2 (http://site.livelap.com/crawler)", "Livelapbot/0.1" ], "addition_date": "2016/01/02", "description": "Livelap web crawler for content discovery", "tags": [ "seo" ] }, { "pattern": "OpenHoseBot", "url": "http://www.openhose.org/bot.html", "instances": [ "Mozilla/5.0 (compatible; OpenHoseBot/2.1; +http://www.openhose.org/bot.html)" ], "addition_date": "2016/01/02", "description": "OpenHose web crawler for content analysis", "tags": [ "search-engine" ] }, { "pattern": "CapsuleChecker", "url": "http://www.capsulink.com/about", "instances": [ "CapsuleChecker (http://www.capsulink.com/)" ], "addition_date": "2016/01/02", "description": "Capsule web crawler for link checking", "tags": [ "monitoring" ] }, { "pattern": "collection@infegy\\.com", "url": "http://infegy.com/", "instances": [ "Mozilla/5.0 (compatible) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/47.0.2526.73 Safari/537.36 collection@infegy.com" ], "addition_date": "2016/01/03", "description": "Infegy web crawler for social listening", "tags": [ "seo" ] }, { "pattern": "IstellaBot", "url": "http://www.tiscali.it/", "instances": [ "Mozilla/5.0 (compatible; IstellaBot/1.23.15 +http://www.tiscali.it/)" ], "addition_date": "2016/01/09", "description": "Istella web crawler for search indexing", "tags": [ "search-engine" ] }, { "pattern": "DeuSu\\/", "addition_date": "2016/01/23", "url": "https://deusu.de/robot.html", "instances": [ "Mozilla/5.0 (compatible; DeuSu/0.1.0; +https://deusu.org)", "Mozilla/5.0 (compatible; DeuSu/5.0.2; +https://deusu.de/robot.html)" ], "description": "DeuSu web crawler for search indexing", "tags": [ "search-engine" ] }, { "pattern": "betaBot", "addition_date": "2016/01/23", "instances": [], "description": "BetaBot web crawler for testing", "tags": [ "seo" ] }, { "pattern": "Cliqzbot\\/", "addition_date": "2016/01/23", "url": "http://cliqz.com/company/cliqzbot", "instances": [ "Mozilla/5.0 (compatible; Cliqzbot/2.0; +http://cliqz.com/company/cliqzbot)", "Cliqzbot/0.1 (+http://cliqz.com +cliqzbot@cliqz.com)", "Cliqzbot/0.1 (+http://cliqz.com/company/cliqzbot)", "Mozilla/5.0 (compatible; Cliqzbot/0.1 +http://cliqz.com/company/cliqzbot)", "Mozilla/5.0 (compatible; Cliqzbot/1.0 +http://cliqz.com/company/cliqzbot)" ], "description": "Cliqz search engine web crawler bot", "tags": [ "search-engine" ] }, { "pattern": "MojeekBot\\/", "addition_date": "2016/01/23", "url": "https://www.mojeek.com/bot.html", "instances": [ "MojeekBot/0.2 (archi; http://www.mojeek.com/bot.html)", "Mozilla/5.0 (compatible; MojeekBot/0.2; http://www.mojeek.com/bot.html#relaunch)", "Mozilla/5.0 (compatible; MojeekBot/0.2; http://www.mojeek.com/bot.html)", "Mozilla/5.0 (compatible; MojeekBot/0.5; http://www.mojeek.com/bot.html)", "Mozilla/5.0 (compatible; MojeekBot/0.6; +https://www.mojeek.com/bot.html)", "Mozilla/5.0 (compatible; MojeekBot/0.6; http://www.mojeek.com/bot.html)" ], "description": "Mojeek search engine web crawler bot", "tags": [ "search-engine" ] }, { "pattern": "netEstate NE Crawler", "addition_date": "2016/01/23", "url": "http://www.website-datenbank.de/", "instances": [ "netEstate NE Crawler (+http://www.sengine.info/)", "netEstate NE Crawler (+http://www.website-datenbank.de/)" ], "description": "NetEstate web crawler for domain analysis", "tags": [ "seo" ] }, { "pattern": "SafeSearch microdata crawler", "addition_date": "2016/01/23", "url": "https://safesearch.avira.com", "instances": [ "SafeSearch microdata crawler (https://safesearch.avira.com, safesearch-abuse@avira.com)" ], "description": "Avira SafeSearch web crawler for safety", "tags": [ "search-engine" ] }, { "pattern": "Gluten Free Crawler\\/", "addition_date": "2016/01/23", "url": "http://glutenfreepleasure.com/", "instances": [ "Mozilla/5.0 (compatible; Gluten Free Crawler/1.0; +http://glutenfreepleasure.com/)" ], "description": "Gluten Free Pleasure web crawler bot", "tags": [ "search-engine" ] }, { "pattern": "Sonic", "addition_date": "2016/02/08", "url": "http://www.yama.info.waseda.ac.jp/~crawler/info.html", "instances": [ "Mozilla/5.0 (compatible; RankSonicSiteAuditor/1.0; +https://ranksonic.com/ranksonic_sab.html)", "Mozilla/5.0 (compatible; Sonic/1.0; http://www.yama.info.waseda.ac.jp/~crawler/info.html)", "Mozzila/5.0 (compatible; Sonic/1.0; http://www.yama.info.waseda.ac.jp/~crawler/info.html)" ], "description": "Sonic web crawler for ranking analysis", "tags": [ "seo" ] }, { "pattern": "Sysomos", "addition_date": "2016/02/08", "url": "http://www.sysomos.com", "instances": [ "Mozilla/5.0 (compatible; Sysomos/1.0; +http://www.sysomos.com/; Sysomos)" ], "description": "Sysomos web crawler for social media analysis", "tags": [ "seo" ] }, { "pattern": "Trove", "addition_date": "2016/02/08", "url": "http://www.trove.com", "instances": [], "description": "Trove web crawler for content discovery", "tags": [ "archiver" ] }, { "pattern": "deadlinkchecker", "addition_date": "2016/02/08", "url": "http://www.deadlinkchecker.com", "instances": [ "www.deadlinkchecker.com Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/46.0.2490.86 Safari/537.36", "www.deadlinkchecker.com XMLHTTP/1.0", "www.deadlinkchecker.com XMLHTTP/1.0 Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/46.0.2490.86 Safari/537.36" ], "description": "Dead Link Checker web crawler for link validation", "tags": [ "seo" ] }, { "pattern": "Slack-ImgProxy", "addition_date": "2016/04/25", "url": "https://api.slack.com/robots", "instances": [ "Slack-ImgProxy (+https://api.slack.com/robots)", "Slack-ImgProxy 0.59 (+https://api.slack.com/robots)", "Slack-ImgProxy 0.66 (+https://api.slack.com/robots)", "Slack-ImgProxy 1.106 (+https://api.slack.com/robots)", "Slack-ImgProxy 1.138 (+https://api.slack.com/robots)", "Slack-ImgProxy 149 (+https://api.slack.com/robots)" ], "description": "Slack's image proxy web crawler bot", "tags": [ "social-preview" ] }, { "pattern": "Embedly", "addition_date": "2016/04/25", "url": "http://support.embed.ly", "instances": [ "Embedly +support@embed.ly", "Mozilla/5.0 (compatible; Embedly/0.2; +http://support.embed.ly/)", "Mozilla/5.0 (compatible; Embedly/0.2; snap; +http://support.embed.ly/)" ], "description": "Embedly web crawler for content embedding", "tags": [ "social-preview" ] }, { "pattern": "RankActiveLinkBot", "addition_date": "2016/06/20", "url": "https://rankactive.com/resources/rankactive-linkbot", "instances": [ "Mozilla/5.0 (compatible; RankActiveLinkBot; +https://rankactive.com/resources/rankactive-linkbot)" ], "description": "RankActive web crawler for link analysis", "tags": [ "seo" ] }, { "pattern": "iskanie", "addition_date": "2016/09/02", "url": "http://www.iskanie.com", "instances": [ "iskanie (+http://www.iskanie.com)" ], "description": "Iskanie web crawler for content discovery", "tags": [ "seo" ] }, { "pattern": "SafeDNSBot", "addition_date": "2016/09/10", "url": "https://www.safedns.com/searchbot", "instances": [ "SafeDNSBot (https://www.safedns.com/searchbot)" ], "description": "SafeDNS web crawler for security analysis", "tags": [ "scanner" ] }, { "pattern": "SkypeUriPreview", "addition_date": "2016/10/10", "instances": [ "Mozilla/5.0 (Windows NT 6.1; WOW64) SkypeUriPreview Preview/0.5" ], "description": "Skype's URI preview web crawler bot", "tags": [ "social-preview" ] }, { "pattern": "Veoozbot", "addition_date": "2016/11/03", "url": "http://www.veooz.com/veoozbot.html", "instances": [ "Mozilla/5.0 (compatible; Veoozbot/1.0; +http://www.veooz.com/veoozbot.html)" ], "description": "Veooz web crawler for marketing analysis", "tags": [ "seo" ] }, { "pattern": "Slackbot", "addition_date": "2016/11/03", "url": "https://api.slack.com/robots", "instances": [ "Slackbot-LinkExpanding (+https://api.slack.com/robots)", "Slackbot-LinkExpanding 1.0 (+https://api.slack.com/robots)", "Slackbot 1.0 (+https://api.slack.com/robots)" ], "description": "Slack's link expansion web crawler bot", "tags": [ "social-preview" ] }, { "pattern": "redditbot", "addition_date": "2016/11/03", "url": "http://www.reddit.com/feedback", "instances": [ "Mozilla/5.0 (compatible; redditbot/1.0; +http://www.reddit.com/feedback)" ], "description": "Reddit's web crawler for content sharing", "tags": [ "social-preview" ] }, { "pattern": "datagnionbot", "addition_date": "2016/11/03", "url": "http://www.datagnion.com/bot.html", "instances": [ "datagnionbot (+http://www.datagnion.com/bot.html)" ], "description": "Datagnion web crawler for data analysis", "tags": [ "seo" ] }, { "pattern": "Google-Adwords-Instant", "addition_date": "2016/11/03", "url": "http://www.google.com/adsbot.html", "instances": [ "Google-Adwords-Instant (+http://www.google.com/adsbot.html)" ], "description": "Google AdWords instant web crawler bot", "tags": [ "advertising" ] }, { "pattern": "adbeat_bot", "addition_date": "2016/11/04", "instances": [ "Mozilla/5.0 (compatible; adbeat_bot; +support@adbeat.com; support@adbeat.com)", "adbeat_bot" ], "description": "AdBeat web crawler for advertising analysis", "tags": [ "advertising" ] }, { "pattern": "WhatsApp", "addition_date": "2016/11/15", "url": "https://www.whatsapp.com/", "instances": [ "WhatsApp", "WhatsApp/0.3.4479 N", "WhatsApp/0.3.4679 N", "WhatsApp/0.3.4941 N", "WhatsApp/2.12.15/i", "WhatsApp/2.12.16/i", "WhatsApp/2.12.17/i", "WhatsApp/2.12.449 A", "WhatsApp/2.12.453 A", "WhatsApp/2.12.510 A", "WhatsApp/2.12.540 A", "WhatsApp/2.12.548 A", "WhatsApp/2.12.555 A", "WhatsApp/2.12.556 A", "WhatsApp/2.16.1/i", "WhatsApp/2.16.13 A", "WhatsApp/2.16.2/i", "WhatsApp/2.16.42 A", "WhatsApp/2.16.57 A", "WhatsApp/2.19.92 i", "WhatsApp/2.19.175 A", "WhatsApp/2.19.244 A", "WhatsApp/2.19.258 A", "WhatsApp/2.19.308 A", "WhatsApp/2.19.330 A" ], "description": "WhatsApp's web crawler for link previews", "tags": [ "social-preview" ] }, { "pattern": "contxbot", "addition_date": "2017/02/25", "instances": [ "Mozilla/5.0 (compatible;contxbot/1.0)" ], "description": "Contx web crawler for content discovery", "tags": [ "seo" ] }, { "pattern": "pinterest\\.com\\/bot", "addition_date": "2017/03/03", "instances": [ "Mozilla/5.0 (compatible; Pinterestbot/1.0; +http://www.pinterest.com/bot.html)", "Pinterest/0.2 (+http://www.pinterest.com/bot.html)" ], "url": "http://www.pinterest.com/bot.html", "description": "Pinterest web crawler for content discovery", "tags": [ "social-preview" ] }, { "pattern": "electricmonk", "addition_date": "2017/03/04", "instances": [ "Mozilla/5.0 (compatible; electricmonk/3.2.0 +https://www.duedil.com/our-crawler/)" ], "url": "https://www.duedil.com/our-crawler/", "description": "DueDil electricmonk web crawler bot", "tags": [ "seo" ] }, { "pattern": "GarlikCrawler", "addition_date": "2017/03/18", "instances": [ "GarlikCrawler/1.2 (http://garlik.com/, crawler@garlik.com)" ], "url": "http://garlik.com/", "description": "Garlik web crawler for content discovery", "tags": [ "seo" ] }, { "pattern": "BingPreview\\/", "addition_date": "2017/04/23", "url": "https://www.bing.com/webmaster/help/which-crawlers-does-bing-use-8c184ec0", "instances": [ "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/534+ (KHTML, like Gecko) BingPreview/1.0b", "Mozilla/5.0 (Windows NT 6.3; WOW64; Trident/7.0; rv:11.0; BingPreview/1.0b) like Gecko", "Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.2; Trident/6.0; WOW64; Trident/6.0; BingPreview/1.0b)", "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0; WOW64; Trident/5.0; BingPreview/1.0b)", "Mozilla/5.0 (iPhone; CPU iPhone OS 7_0 like Mac OS X) AppleWebKit/537.51.1 (KHTML, like Gecko) Version/7.0 Mobile/11A465 Safari/9537.53 BingPreview/1.0b" ], "description": "Bing preview web crawler bot", "tags": [ "social-preview" ] }, { "pattern": "vebidoobot", "addition_date": "2017/05/08", "instances": [ "Mozilla/5.0 (compatible; vebidoobot/1.0; +https://blog.vebidoo.de/vebidoobot/" ], "url": "https://blog.vebidoo.de/vebidoobot/", "description": "Vebidoo web crawler for content discovery", "tags": [ "seo" ] }, { "pattern": "FemtosearchBot", "addition_date": "2017/05/16", "instances": [ "Mozilla/5.0 (compatible; FemtosearchBot/1.0; http://femtosearch.com)" ], "url": "http://femtosearch.com", "description": "Femtosearch web crawler for content discovery", "tags": [ "search-engine" ] }, { "pattern": "Yahoo Link Preview", "addition_date": "2017/06/28", "instances": [ "Mozilla/5.0 (compatible; Yahoo Link Preview; https://help.yahoo.com/kb/mail/yahoo-link-preview-SLN23615.html)" ], "url": "https://help.yahoo.com/kb/mail/yahoo-link-preview-SLN23615.html", "description": "Yahoo link preview web crawler bot", "tags": [ "social-preview" ] }, { "pattern": "MetaJobBot", "addition_date": "2017/08/16", "instances": [ "Mozilla/5.0 (compatible; MetaJobBot; http://www.metajob.de/crawler)" ], "url": "http://www.metajob.de/the/crawler", "description": "MetaJob web crawler for job discovery", "tags": [ "seo" ] }, { "pattern": "DomainStatsBot", "addition_date": "2017/08/16", "instances": [ "DomainStatsBot/1.0 (http://domainstats.io/our-bot)" ], "url": "http://domainstats.io/our-bot", "description": "DomainStats web crawler for domain analysis", "tags": [ "seo" ] }, { "pattern": "mindUpBot", "addition_date": "2017/08/16", "instances": [ "mindUpBot (datenbutler.de)" ], "url": "http://www.datenbutler.de/", "description": "MindUp web crawler for content discovery", "tags": [ "seo" ] }, { "pattern": "Daum\\/", "addition_date": "2017/08/16", "instances": [ "Mozilla/5.0 (compatible; Daum/4.1; +http://cs.daum.net/faq/15/4118.html?faqId=28966)" ], "url": "http://cs.daum.net/faq/15/4118.html?faqId=28966", "description": "Daum search engine web crawler bot", "tags": [ "search-engine" ] }, { "pattern": "Jugendschutzprogramm-Crawler", "addition_date": "2017/08/16", "instances": [ "Jugendschutzprogramm-Crawler; Info: http://www.jugendschutzprogramm.de" ], "url": "http://www.jugendschutzprogramm.de", "description": "Jugendschutzprogramm web crawler bot", "tags": [ "scanner" ] }, { "pattern": "Xenu Link Sleuth", "addition_date": "2017/08/19", "instances": [ "Xenu Link Sleuth/1.3.8" ], "url": "http://home.snafu.de/tilman/xenulink.html", "description": "Xenu link checker web crawler tool", "tags": [ "seo" ] }, { "pattern": "Pcore-HTTP", "addition_date": "2017/08/19", "instances": [ "Pcore-HTTP/v0.40.3", "Pcore-HTTP/v0.44.0" ], "url": "https://bitbucket.org/softvisio/pcore/overview", "description": "Pcore HTTP web crawler library", "tags": [ "http-library" ] }, { "pattern": "moatbot", "addition_date": "2017/09/16", "instances": [ "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/40.0.2214.111 Safari/537.36 moatbot", "Mozilla/5.0 (iPhone; CPU iPhone OS 8_0 like Mac OS X) AppleWebKit/600.1.3 (KHTML, like Gecko) Version/8.0 Mobile/12A4345d Safari/600.1.4 moatbot" ], "url": "https://moat.com", "description": "Moat web crawler for advertising analysis", "tags": [ "advertising" ] }, { "pattern": "KosmioBot", "addition_date": "2017/09/16", "instances": [ "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/44.0.2403.125 Safari/537.36 (compatible; KosmioBot/1.0; +http://kosm.io/bot.html)" ], "url": "http://kosm.io/bot.html", "description": "Kosmio web crawler for content discovery", "tags": [ "seo" ] }, { "pattern": "[pP]ingdom", "addition_date": "2017/09/16", "instances": [ "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Ubuntu Chromium/59.0.3071.109 Chrome/59.0.3071.109 Safari/537.36 PingdomPageSpeed/1.0 (pingbot/2.0; +http://www.pingdom.com/)", "Mozilla/5.0 (compatible; pingbot/2.0; +http://www.pingdom.com/)", "Pingdom.com_bot_version_1.4_(http://www.pingdom.com/)", "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Ubuntu Chromium/61.0.3163.100 Chrome/61.0.3163.100 Safari/537.36 PingdomPageSpeed/1.0 (pingbot/2.0; +http://www.pingdom.com/)", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) browser/2020.2.1 Chrome/78.0.3904.130 Electron/7.3.2 Safari/537.36 PingdomTMS/2020.2", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) browser/2020.2.5 Chrome/78.0.3904.130 Electron/7.3.15 Safari/537.36 PingdomTMS/2020.2", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) browser/2020.2.0 Chrome/78.0.3904.130 Electron/7.1.7 Safari/537.36 PingdomTMS/2020.2", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) renderer/2020.2.0 Chrome/78.0.3904.130 Electron/7.1.7 Safari/537.36 PingdomTMS/2020.2", "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Ubuntu Chromium/61.0.3163.100 Chrome/61.0.3163.100 Safari/537.36 PingdomPageSpeed/1.0 (pingbot/2.0; http://www.pingdom.com/)" ], "url": "http://www.pingdom.com", "description": "Pingdom website monitoring web crawler", "tags": [ "monitoring" ] }, { "pattern": "AppInsights", "addition_date": "2019/03/09", "instances": [ "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0; AppInsights)" ], "url": "https://docs.microsoft.com/en-us/azure/azure-monitor/app/app-insights-overview", "description": "Microsoft AppInsights web crawler bot", "tags": [ "monitoring" ] }, { "pattern": "PhantomJS", "addition_date": "2017/09/18", "instances": [ "Mozilla/5.0 (Unknown; Linux x86_64) AppleWebKit/538.1 (KHTML, like Gecko) PhantomJS/2.1.1 Safari/538.1 bl.uk_lddc_renderbot/2.0.0 (+ http://www.bl.uk/aboutus/legaldeposit/websites/websites/faqswebmaster/index.html)" ], "url": "http://phantomjs.org/", "description": "PhantomJS headless browser web crawler", "tags": [ "browser-automation" ] }, { "pattern": "Gowikibot", "addition_date": "2017/10/26", "instances": [ "Mozilla/5.0 (compatible; Gowikibot/1.0; +http://www.gowikibot.com)" ], "url": "http://www.gowikibot.com", "description": "GoWiki web crawler for content discovery", "tags": [ "seo" ] }, { "pattern": "PiplBot", "addition_date": "2017/10/30", "instances": [ "PiplBot (+http://www.pipl.com/bot/)", "Mozilla/5.0+(compatible;+PiplBot;+http://www.pipl.com/bot/)" ], "url": "http://www.pipl.com/bot/", "description": "Pipl web crawler for people search", "tags": [ "seo" ] }, { "pattern": "Discordbot", "addition_date": "2017/09/22", "url": "https://discordapp.com", "instances": [ "Mozilla/5.0 (compatible; Discordbot/2.0; +https://discordapp.com)" ], "description": "Discord web crawler for link previews", "tags": [ "social-preview" ] }, { "pattern": "TelegramBot", "addition_date": "2017/10/01", "instances": [ "TelegramBot (like TwitterBot)" ], "description": "Telegram web crawler for link previews", "tags": [ "social-preview" ] }, { "pattern": "Jetslide", "addition_date": "2017/09/27", "url": "http://jetsli.de/crawler", "instances": [ "Mozilla/5.0 (compatible; Jetslide; +http://jetsli.de/crawler)" ], "description": "Jetslide web crawler for content discovery", "tags": [ "social-preview" ] }, { "pattern": "newsharecounts", "addition_date": "2017/09/30", "url": "http://newsharecounts.com/crawler", "instances": [ "Mozilla/5.0 (compatible; NewShareCounts.com/1.0; +http://newsharecounts.com/crawler)" ], "description": "NewShareCounts web crawler for sharing", "tags": [ "social-preview" ] }, { "pattern": "James BOT", "addition_date": "2017/10/12", "url": "http://cognitiveseo.com/bot.html", "instances": [ "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.8.1.6) Gecko/20070725 Firefox/2.0.0.6 - James BOT - WebCrawler http://cognitiveseo.com/bot.html" ], "description": "CognitiveSEO James web crawler bot", "tags": [ "seo" ] }, { "pattern": "Bark[rR]owler", "addition_date": "2017/10/09", "url": "http://www.exensa.com/crawl", "instances": [ "Barkrowler/0.5.1 (experimenting / debugging - sorry for your logs ) http://www.exensa.com/crawl - admin@exensa.com -- based on BuBiNG", "Barkrowler/0.7 (+http://www.exensa.com/crawl)", "BarkRowler/0.7 (+http://www.exensa.com/crawling)", "Barkrowler/0.9 (+http://www.exensa.com/crawl)" ], "description": "Barkrowler web crawler for content discovery", "tags": [ "seo" ] }, { "pattern": "TinEye", "addition_date": "2017/10/14", "url": "http://www.tineye.com/crawler.html", "instances": [ "Mozilla/5.0 (compatible; TinEye-bot/1.31; +http://www.tineye.com/crawler.html)", "TinEye/1.1 (http://tineye.com/crawler.html)" ], "description": "TinEye reverse image search web crawler", "tags": [ "seo" ] }, { "pattern": "SocialRankIOBot", "addition_date": "2017/10/19", "url": "http://socialrank.io/about", "instances": [ "SocialRankIOBot; http://socialrank.io/about" ], "description": "SocialRank web crawler for social analysis", "tags": [ "seo" ] }, { "pattern": "trendictionbot", "addition_date": "2017/10/30", "url": "http://www.trendiction.de/bot", "instances": [ "Mozilla/5.0 (Windows; U; Windows NT 6.0; en-GB; rv:1.0; trendictionbot0.5.0; trendiction search; http://www.trendiction.de/bot; please let us know of any problems; web at trendiction.com) Gecko/20071127 Firefox/3.0.0.11", "Mozilla/5.0 (Windows NT 10.0; Win64; x64; trendictionbot0.5.0; trendiction search; http://www.trendiction.de/bot; please let us know of any problems; web at trendiction.com) Gecko/20170101 Firefox/67.0" ], "description": "Trendiction web crawler for trend analysis", "tags": [ "seo" ] }, { "pattern": "Ocarinabot", "addition_date": "2017/09/27", "instances": [ "Ocarinabot" ], "description": "Ocarina web crawler for content discovery", "tags": [ "seo" ] }, { "pattern": "epicbot", "addition_date": "2017/10/31", "url": "http://www.epictions.com/epicbot", "instances": [ "Mozilla/5.0 (compatible; epicbot; +http://www.epictions.com/epicbot)" ], "description": "Epic web crawler for content discovery", "tags": [ "seo" ] }, { "pattern": "Primalbot", "addition_date": "2017/09/27", "url": "https://www.primal.com", "instances": [ "Mozilla/5.0 (compatible; Primalbot; +https://www.primal.com;)" ], "description": "Primal web crawler for content discovery", "tags": [ "seo" ] }, { "pattern": "DuckDuckGo-Favicons-Bot", "addition_date": "2017/10/06", "url": "http://duckduckgo.com", "instances": [ "Mozilla/5.0 (compatible; DuckDuckGo-Favicons-Bot/1.0; +http://duckduckgo.com)" ], "description": "DuckDuckGo favicon web crawler bot", "tags": [ "search-engine" ] }, { "pattern": "GnowitNewsbot", "addition_date": "2017/10/30", "url": "http://www.gnowit.com", "instances": [ "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:49.0) Gecko/20100101 Firefox/49.0 / GnowitNewsbot / Contact information at http://www.gnowit.com" ], "description": "Gnowit news web crawler bot", "tags": [ "seo" ] }, { "pattern": "Leikibot", "addition_date": "2017/09/24", "url": "http://www.leiki.com", "instances": [ "Mozilla/5.0 (Windows NT 6.3;compatible; Leikibot/1.0; +http://www.leiki.com)" ], "description": "Leiki web crawler for content discovery", "tags": [ "seo" ] }, { "pattern": "LinkArchiver", "addition_date": "2017/09/24", "url": "https://github.com/thisisparker/linkarchiver", "instances": [ "@LinkArchiver twitter bot" ], "description": "LinkArchiver web crawler for archiving", "tags": [ "archiver" ] }, { "pattern": "YaK\\/", "addition_date": "2017/09/25", "url": "http://linkfluence.com", "instances": [ "Mozilla/5.0 (compatible; YaK/1.0; http://linkfluence.com/; bot@linkfluence.com)" ], "description": "Linkfluence YaK web crawler bot", "tags": [ "seo" ] }, { "pattern": "PaperLiBot", "addition_date": "2017/09/25", "url": "http://support.paper.li/entries/20023257-what-is-paper-li", "instances": [ "Mozilla/5.0 (compatible; PaperLiBot/2.1; http://support.paper.li/entries/20023257-what-is-paper-li)", "Mozilla/5.0 (compatible; PaperLiBot/2.1; https://support.paper.li/entries/20023257-what-is-paper-li)" ], "description": "Paper.li web crawler for content curation", "tags": [ "seo" ] }, { "pattern": "Digg Deeper", "addition_date": "2017/09/26", "url": "http://digg.com/about", "instances": [ "Digg Deeper/v1 (http://digg.com/about)" ], "description": "Digg web crawler for content discovery", "tags": [ "seo" ] }, { "pattern": "^dcrawl", "addition_date": "2017/09/22", "url": "https://github.com/kgretzky/dcrawl", "instances": [ "dcrawl/1.0" ], "description": "dcrawl web crawler framework", "tags": [ "seo" ] }, { "pattern": "Snacktory", "addition_date": "2017/09/23", "url": "https://github.com/karussell/snacktory", "instances": [ "Mozilla/5.0 (compatible; Snacktory; +https://github.com/karussell/snacktory)" ], "description": "Snacktory web crawler for content extraction", "tags": [ "seo" ] }, { "pattern": "AndersPinkBot", "addition_date": "2017/09/24", "url": "http://anderspink.com/bot.html", "instances": [ "Mozilla/5.0 (compatible; AndersPinkBot/1.0; +http://anderspink.com/bot.html)" ], "description": "AndersPink web crawler for content discovery", "tags": [ "seo" ] }, { "pattern": "Fyrebot", "addition_date": "2017/09/22", "instances": [ "Fyrebot/1.0" ], "description": "Fyre web crawler for content discovery", "tags": [ "seo" ] }, { "pattern": "EveryoneSocialBot", "addition_date": "2017/09/22", "url": "http://everyonesocial.com", "instances": [ "Mozilla/5.0 (compatible; EveryoneSocialBot/1.0; support@everyonesocial.com http://everyonesocial.com/)" ], "description": "EveryoneSocial web crawler for sharing", "tags": [ "social-preview" ] }, { "pattern": "Mediatoolkitbot", "addition_date": "2017/10/06", "url": "http://mediatoolkit.com", "instances": [ "Mediatoolkitbot (complaints@mediatoolkit.com)" ], "description": "Mediatoolkit web crawler for monitoring", "tags": [ "monitoring" ] }, { "pattern": "Luminator-robots", "addition_date": "2017/09/22", "instances": [ "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_2) AppleWebKit/537.13 (KHTML, like Gecko) Chrome/30.0.1599.66 Safari/537.13 Luminator-robots/2.0" ], "description": "Luminator web crawler for content discovery", "tags": [ "seo" ] }, { "pattern": "ExtLinksBot", "addition_date": "2017/11/02", "url": "https://extlinks.com/Bot.html", "instances": [ "Mozilla/5.0 (compatible; ExtLinksBot/1.5 +https://extlinks.com/Bot.html)" ], "description": "ExtLinks web crawler for link analysis", "tags": [ "seo" ] }, { "pattern": "SurveyBot", "addition_date": "2017/11/02", "instances": [ "Mozilla/5.0 (Windows; U; Windows NT 5.1; en; rv:1.9.0.13) Gecko/2009073022 Firefox/3.5.2 (.NET CLR 3.5.30729) SurveyBot/2.3 (DomainTools)" ], "description": "DomainTools survey web crawler bot", "tags": [ "seo" ] }, { "pattern": "NING\\/", "addition_date": "2017/11/02", "instances": [ "NING/1.0" ], "description": "Ning web crawler for content discovery", "tags": [ "social-preview" ] }, { "pattern": "okhttp", "addition_date": "2017/11/02", "instances": [ "okhttp/2.5.0", "okhttp/2.7.5", "okhttp/3.2.0", "okhttp/3.5.0", "okhttp/4.1.0" ], "description": "OkHttp Java HTTP client library", "tags": [ "http-library" ] }, { "pattern": "Nuzzel", "addition_date": "2017/11/02", "instances": [ "Nuzzel" ], "description": "Nuzzel web crawler for news discovery", "tags": [ "seo" ] }, { "pattern": "omgili", "addition_date": "2017/11/02", "url": "http://omgili.com", "instances": [ "omgili/0.5 +http://omgili.com" ], "description": "Omgili web crawler for content discovery", "tags": [ "seo" ] }, { "pattern": "PocketParser", "addition_date": "2017/11/02", "url": "https://getpocket.com/pocketparser_ua", "instances": [ "PocketParser/2.0 (+https://getpocket.com/pocketparser_ua)" ], "description": "Pocket web crawler for content parsing", "tags": [ "seo" ] }, { "pattern": "YisouSpider", "addition_date": "2017/11/02", "instances": [ "YisouSpider", "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.81 YisouSpider/5.0 Safari/537.36" ], "description": "Yisou search engine web crawler bot", "tags": [ "search-engine" ] }, { "pattern": "um-LN", "addition_date": "2017/11/02", "instances": [ "Mozilla/5.0 (compatible; um-LN/1.0; mailto: techinfo@ubermetrics-technologies.com)" ], "description": "Ubermetrics web crawler for monitoring", "tags": [ "seo" ] }, { "pattern": "ToutiaoSpider", "addition_date": "2017/11/02", "url": "http://web.toutiao.com/media_cooperation/", "instances": [ "Mozilla/5.0 (compatible; ToutiaoSpider/1.0; http://web.toutiao.com/media_cooperation/;)" ], "description": "Toutiao news platform web crawler bot", "tags": [ "search-engine" ] }, { "pattern": "MuckRack", "addition_date": "2017/11/02", "url": "http://muckrack.com", "instances": [ "Mozilla/5.0 (compatible; MuckRack/1.0; +http://muckrack.com)" ], "description": "MuckRack journalist database web crawler", "tags": [ "seo" ] }, { "pattern": "Jamie's Spider", "addition_date": "2017/11/02", "url": "http://jamiembrown.com/", "instances": [ "Jamie's Spider (http://jamiembrown.com/)" ], "description": "Jamie Brown's personal web crawler bot", "tags": [ "seo" ] }, { "pattern": "AHC\\/", "addition_date": "2017/11/02", "url": "https://github.com/AsyncHttpClient/async-http-client", "instances": [ "AHC/2.0" ], "description": "Async HTTP Client Java library for HTTP requests", "tags": [ "http-library" ] }, { "pattern": "NetcraftSurveyAgent", "addition_date": "2017/11/02", "instances": [ "Mozilla/5.0 (compatible; NetcraftSurveyAgent/1.0; +info@netcraft.com)" ], "description": "Netcraft survey web crawler bot", "tags": [ "scanner" ] }, { "pattern": "Laserlikebot", "addition_date": "2017/11/02", "instances": [ "Mozilla/5.0 (iPhone; CPU iPhone OS 8_3 like Mac OS X) AppleWebKit/600.1.4 (KHTML, like Gecko) Version/8.0 Mobile/12F70 Safari/600.1.4 (compatible; Laserlikebot/0.1)" ], "description": "Laserlike web crawler for content discovery", "tags": [ "seo" ] }, { "pattern": "^Apache-HttpClient", "addition_date": "2017/11/02", "instances": [ "Apache-HttpClient/4.2.3 (java 1.5)", "Apache-HttpClient/4.2.5 (java 1.5)", "Apache-HttpClient/4.3.1 (java 1.5)", "Apache-HttpClient/4.3.3 (java 1.5)", "Apache-HttpClient/4.3.5 (java 1.5)", "Apache-HttpClient/4.4.1 (Java/1.8.0_65)", "Apache-HttpClient/4.5.2 (Java/1.8.0_65)", "Apache-HttpClient/4.5.2 (Java/1.8.0_151)", "Apache-HttpClient/4.5.2 (Java/1.8.0_161)", "Apache-HttpClient/4.5.2 (Java/1.8.0_181)", "Apache-HttpClient/4.5.3 (Java/1.8.0_121)", "Apache-HttpClient/4.5.3-SNAPSHOT (Java/1.8.0_152)", "Apache-HttpClient/4.5.7 (Java/11.0.3)", "Apache-HttpClient/4.5.10 (Java/1.8.0_201)" ], "description": "Apache HTTP Client Java library for requests", "tags": [ "http-library" ] }, { "pattern": "AppEngine-Google", "addition_date": "2017/11/02", "instances": [ "AppEngine-Google; (+http://code.google.com/appengine; appid: example)", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.181 Safari/537.36 AppEngine-Google; (+http://code.google.com/appengine; appid: s~feedly-nikon3)" ], "description": "Google App Engine web crawler bot", "tags": [ "search-engine" ] }, { "pattern": "Jetty", "addition_date": "2017/11/02", "instances": [ "Jetty/9.3.z-SNAPSHOT" ], "description": "Jetty web server HTTP client library", "tags": [ "http-library" ] }, { "pattern": "Upflow", "addition_date": "2017/11/02", "instances": [ "Upflow/1.0" ], "description": "Upflow web crawler for content analysis", "tags": [ "seo" ] }, { "pattern": "Thinklab", "addition_date": "2017/11/02", "url": "thinklab.com", "instances": [ "Thinklab (thinklab.com)" ], "description": "Thinklab web crawler for research", "tags": [ "academic" ] }, { "pattern": "Traackr\\.com", "addition_date": "2017/11/02", "url": "https://www.traackr.com/", "instances": [ "Traackr.com" ], "description": "Traackr influencer marketing web crawler", "tags": [ "seo" ] }, { "pattern": "Twurly", "addition_date": "2017/11/02", "url": "http://twurly.org", "instances": [ "Ruby, Twurly v1.1 (http://twurly.org)" ], "description": "Twurly Ruby web crawler framework", "tags": [ "seo" ] }, { "pattern": "Mastodon", "addition_date": "2017/11/02", "instances": [ "http.rb/2.2.2 (Mastodon/1.5.1; +https://example-masto-instance.org/)" ], "description": "Mastodon social network web crawler bot", "tags": [ "social-preview" ] }, { "pattern": "http_get", "addition_date": "2017/11/02", "instances": [ "http_get" ], "description": "HTTP GET command-line tool for requests", "tags": [ "http-library" ] }, { "pattern": "DnyzBot", "addition_date": "2017/11/20", "instances": [ "Mozilla/5.0 (compatible; DnyzBot/1.0)" ], "description": "Dnyz web crawler for content discovery", "tags": [ "seo" ] }, { "pattern": "botify", "addition_date": "2018/02/01", "instances": [ "Mozilla/5.0 (compatible; botify; http://botify.com)" ], "description": "Botify SEO tool web crawler for analysis", "tags": [ "seo" ] }, { "pattern": "007ac9 Crawler", "addition_date": "2018/02/09", "instances": [ "Mozilla/5.0 (compatible; 007ac9 Crawler; http://crawler.007ac9.net/)" ], "description": "007ac9 web crawler for content discovery", "tags": [ "seo" ] }, { "pattern": "BehloolBot", "addition_date": "2018/02/09", "instances": [ "Mozilla/5.0 (compatible; BehloolBot/beta; +http://www.webeaver.com/bot)" ], "description": "Behlool web crawler for content analysis", "tags": [ "seo" ] }, { "pattern": "BrandVerity", "addition_date": "2018/02/27", "instances": [ "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.10; rv:41.0) Gecko/20100101 Firefox/55.0 BrandVerity/1.0 (http://www.brandverity.com/why-is-brandverity-visiting-me)", "Mozilla/5.0 (iPhone; CPU iPhone OS 7_0 like Mac OS X) AppleWebKit/537.51.1 (KHTML, like Gecko) Mobile/11A465 Twitter for iPhone BrandVerity/1.0 (http://www.brandverity.com/why-is-brandverity-visiting-me)" ], "url": "http://www.brandverity.com/why-is-brandverity-visiting-me", "description": "BrandVerity web crawler for brand monitoring", "tags": [ "seo" ] }, { "pattern": "check_http", "addition_date": "2018/02/09", "instances": [ "check_http/v2.2.1 (nagios-plugins 2.2.1)" ], "description": "Nagios check_http monitoring plugin", "tags": [ "monitoring" ] }, { "pattern": "BDCbot", "addition_date": "2018/02/09", "instances": [ "Mozilla/5.0 (Windows NT 6.1; compatible; BDCbot/1.0; +http://bigweb.bigdatacorp.com.br/faq.aspx) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2272.118 Safari/537.36", "Mozilla/5.0 (Windows NT 10.0; Win64; x64; BDCbot/1.0; +http://bigweb.bigdatacorp.com.br/faq.aspx) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.100 Safari/537.36" ], "description": "BigDataCorp web crawler for data collection", "tags": [ "seo" ] }, { "pattern": "ZumBot", "addition_date": "2018/02/09", "instances": [ "Mozilla/5.0 (compatible; ZumBot/1.0; http://help.zum.com/inquiry)" ], "description": "Zum search engine web crawler bot", "tags": [ "search-engine" ] }, { "pattern": "EZID", "addition_date": "2018/02/09", "instances": [ "EZID (EZID link checker; https://ezid.cdlib.org/)" ], "description": "EZID link checker web crawler bot", "tags": [ "seo" ] }, { "pattern": "ICC-Crawler", "addition_date": "2018/02/28", "instances": [ "ICC-Crawler/2.0 (Mozilla-compatible; ; http://ucri.nict.go.jp/en/icccrawler.html)" ], "url": "http://ucri.nict.go.jp/en/icccrawler.html", "description": "ICC web crawler for language research", "tags": [ "academic" ] }, { "pattern": "ArchiveBot", "addition_date": "2018/02/28", "instances": [ "ArchiveTeam ArchiveBot/20170106.02 (wpull 2.0.2)" ], "url": "https://github.com/ArchiveTeam/ArchiveBot", "description": "Archive Team web crawler for preservation", "tags": [ "archiver" ] }, { "pattern": "^LCC ", "addition_date": "2018/02/28", "instances": [ "LCC (+http://corpora.informatik.uni-leipzig.de/crawler_faq.html)" ], "url": "http://corpora.informatik.uni-leipzig.de/crawler_faq.html", "description": "Leipzig Corpora Collection web crawler", "tags": [ "academic" ] }, { "pattern": "filterdb\\.iss\\.net\\/crawler", "addition_date": "2018/03/16", "instances": [ "Mozilla/5.0 (compatible; oBot/2.3.1; +http://filterdb.iss.net/crawler/)" ], "url": "http://filterdb.iss.net/crawler/", "description": "ISS filter database web crawler", "tags": [ "scanner" ] }, { "pattern": "BLP_bbot", "addition_date": "2018/03/27", "instances": [ "BLP_bbot/0.1" ], "description": "BLP web crawler for content discovery", "tags": [ "seo" ] }, { "pattern": "BomboraBot", "addition_date": "2018/03/27", "instances": [ "Mozilla/5.0 (compatible; BomboraBot/1.0; +http://www.bombora.com/bot)" ], "url": "http://www.bombora.com/bot", "description": "Bombora web crawler for business intelligence", "tags": [ "seo" ] }, { "pattern": "Buck\\/", "addition_date": "2018/03/27", "instances": [ "Buck/2.2; (+https://app.hypefactors.com/media-monitoring/about.html)" ], "url": "https://app.hypefactors.com/media-monitoring/about.html", "description": "Hypefactors media monitoring web crawler", "tags": [ "monitoring" ] }, { "pattern": "Companybook-Crawler", "addition_date": "2018/03/27", "instances": [ "Companybook-Crawler (+https://www.companybooknetworking.com/)" ], "url": "https://www.companybooknetworking.com/", "description": "Companybook networking web crawler", "tags": [ "seo" ] }, { "pattern": "Genieo", "addition_date": "2018/03/27", "instances": [ "Mozilla/5.0 (compatible; Genieo/1.0 http://www.genieo.com/webfilter.html)" ], "url": "http://www.genieo.com/webfilter.html", "description": "Genieo web filter web crawler bot", "tags": [ "seo" ] }, { "pattern": "magpie-crawler", "addition_date": "2018/03/27", "instances": [ "magpie-crawler/1.1 (U; Linux amd64; en-GB; +http://www.brandwatch.net)" ], "url": "http://www.brandwatch.net", "description": "Brandwatch magpie web crawler bot", "tags": [ "seo" ] }, { "pattern": "MeltwaterNews", "addition_date": "2018/03/27", "instances": [ "MeltwaterNews www.meltwater.com" ], "url": "http://www.meltwater.com", "description": "Meltwater news monitoring web crawler", "tags": [ "seo" ] }, { "pattern": "Moreover", "addition_date": "2018/03/27", "instances": [ "Mozilla/5.0 Moreover/5.1 (+http://www.moreover.com)" ], "url": "http://www.moreover.com", "description": "Moreover news aggregation web crawler", "tags": [ "seo" ] }, { "pattern": "newspaper\\/", "addition_date": "2018/03/27", "instances": [ "newspaper/0.1.0.7", "newspaper/0.2.5", "newspaper/0.2.6", "newspaper/0.2.8" ], "description": "Newspaper Python web scraping library", "tags": [ "http-library" ] }, { "pattern": "ScoutJet", "addition_date": "2018/03/27", "instances": [ "Mozilla/5.0 (compatible; ScoutJet; +http://www.scoutjet.com/)" ], "url": "http://www.scoutjet.com/", "description": "ScoutJet web crawler for content discovery", "tags": [ "seo" ] }, { "pattern": "(^| )sentry\\/", "addition_date": "2018/03/27", "instances": [ "sentry/8.22.0 (https://sentry.io)" ], "url": "https://sentry.io", "description": "Sentry error tracking web crawler bot", "tags": [ "monitoring" ] }, { "pattern": "StorygizeBot", "addition_date": "2018/03/27", "instances": [ "Mozilla/5.0 (compatible; StorygizeBot; http://www.storygize.com)" ], "url": "http://www.storygize.com", "description": "Storygize web crawler for content analysis", "tags": [ "seo" ] }, { "pattern": "UptimeRobot", "addition_date": "2018/03/27", "instances": [ "Mozilla/5.0+(compatible; UptimeRobot/2.0; http://www.uptimerobot.com/)" ], "url": "http://www.uptimerobot.com/", "description": "UptimeRobot website monitoring web crawler", "tags": [ "monitoring" ] }, { "pattern": "OutclicksBot", "addition_date": "2018/04/21", "instances": [ "OutclicksBot/2 +https://www.outclicks.net/agent/VjzDygCuk4ubNmg40ZMbFqT0sIh7UfOKk8s8ZMiupUR", "OutclicksBot/2 +https://www.outclicks.net/agent/gIYbZ38dfAuhZkrFVl7sJBFOUhOVct6J1SvxgmBZgCe", "OutclicksBot/2 +https://www.outclicks.net/agent/PryJzTl8POCRHfvEUlRN5FKtZoWDQOBEvFJ2wh6KH5J", "OutclicksBot/2 +https://www.outclicks.net/agent/p2i4sNUh7eylJF1S6SGgRs5mP40ExlYvsr9GBxVQG6h" ], "url": "https://www.outclicks.net", "description": "Outclicks web crawler for link tracking", "tags": [ "advertising" ] }, { "pattern": "seoscanners", "addition_date": "2018/05/27", "instances": [ "Mozilla/5.0 (compatible; seoscanners.net/1; +spider@seoscanners.net)" ], "url": "https://github.com/monperrus/crawler-user-agents/issues/384#issuecomment-2575367162", "description": "SEO Scanners web crawler for analysis", "tags": [ "seo" ] }, { "pattern": "Hatena", "addition_date": "2018/05/29", "instances": [ "Hatena Antenna/0.3", "Hatena::Russia::Crawler/0.01", "Hatena-Favicon/2 (http://www.hatena.ne.jp/faq/)", "Hatena::Scissors/0.01", "HatenaBookmark/4.0 (Hatena::Bookmark; Analyzer)", "Hatena::Fetcher/0.01 (master) Furl/3.13" ], "description": "Hatena web services web crawler bot", "tags": [ "social-preview" ] }, { "pattern": "Google Web Preview", "addition_date": "2018/05/31", "instances": [ "Mozilla/5.0 (Linux; U; Android 2.3.4; generic) AppleWebKit/537.36 (KHTML, like Gecko; Google Web Preview) Version/4.0 Mobile Safari/537.36", "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko; Google Web Preview) Chrome/27.0.1453 Safari/537.36" ], "description": "Google web preview web crawler bot", "tags": [ "social-preview" ] }, { "pattern": "MauiBot", "addition_date": "2018/06/06", "instances": [ "MauiBot (crawler.feedback+wc@gmail.com)" ], "description": "Maui web crawler for content discovery", "tags": [ "seo" ] }, { "pattern": "AlphaBot", "addition_date": "2018/05/27", "instances": [ "Mozilla/5.0 (compatible; AlphaBot/3.2; +http://alphaseobot.com/bot.html)" ], "url": "http://alphaseobot.com/bot.html", "description": "AlphaBot SEO web crawler for analysis", "tags": [ "seo" ] }, { "pattern": "SBL-BOT", "addition_date": "2018/06/06", "instances": [ "SBL-BOT (http://sbl.net)" ], "url": "http://sbl.net", "description": "Bot of SoftByte BlackWidow", "tags": [ "scanner" ] }, { "pattern": "IAS crawler", "addition_date": "2018/06/06", "instances": [ "IAS crawler (ias_crawler; http://integralads.com/site-indexing-policy/)" ], "url": "http://integralads.com/site-indexing-policy/", "description": "Bot of Integral Ad Science, Inc.", "tags": [ "advertising" ] }, { "pattern": "adscanner", "addition_date": "2018/06/24", "instances": [ "Mozilla/5.0 (compatible; adscanner/)" ], "description": "AdScanner web crawler for ad analysis", "tags": [ "advertising" ] }, { "pattern": "Netvibes", "addition_date": "2018/06/24", "instances": [ "Netvibes (crawler/bot; http://www.netvibes.com", "Netvibes (crawler; http://www.netvibes.com)" ], "url": "http://www.netvibes.com", "description": "Netvibes web crawler for content aggregation", "tags": [ "feed-reader" ] }, { "pattern": "acapbot", "addition_date": "2018/06/27", "instances": [ "Mozilla/5.0 (compatible;acapbot/0.1;treat like Googlebot)", "Mozilla/5.0 (compatible;acapbot/0.1.;treat like Googlebot)" ], "description": "ACAP web crawler for content analysis", "tags": [ "seo" ] }, { "pattern": "Baidu-YunGuanCe", "addition_date": "2018/06/27", "instances": [ "Baidu-YunGuanCe-Bot(ce.baidu.com)", "Baidu-YunGuanCe-SLABot(ce.baidu.com)", "Baidu-YunGuanCe-ScanBot(ce.baidu.com)", "Baidu-YunGuanCe-PerfBot(ce.baidu.com)", "Baidu-YunGuanCe-VSBot(ce.baidu.com)" ], "url": "https://ce.baidu.com/topic/topic20150908", "description": "Baidu Cloud Watch", "tags": [ "monitoring" ] }, { "pattern": "bitlybot", "addition_date": "2018/06/27", "instances": [ "bitlybot/3.0 (+http://bit.ly/)", "bitlybot/2.0", "bitlybot" ], "url": "http://bit.ly/", "description": "Bit.ly web crawler for link tracking", "tags": [ "seo" ] }, { "pattern": "blogmuraBot", "addition_date": "2018/06/27", "instances": [ "blogmuraBot (+http://www.blogmura.com)" ], "url": "http://www.blogmura.com", "description": "A blog ranking site which links to blogs on just about every theme possible.", "tags": [ "seo" ] }, { "pattern": "Bot\\.AraTurka\\.com", "addition_date": "2018/06/27", "instances": [ "Bot.AraTurka.com/0.0.1" ], "url": "http://www.araturka.com", "description": "AraTurka web crawler for content discovery", "tags": [ "search-engine" ] }, { "pattern": "bot-pge\\.chlooe\\.com", "addition_date": "2018/06/27", "instances": [ "bot-pge.chlooe.com/1.0.0 (+http://www.chlooe.com/)" ], "description": "Chlooe web crawler for content analysis", "tags": [ "seo" ] }, { "pattern": "BoxcarBot", "addition_date": "2018/06/27", "instances": [ "Mozilla/5.0 (compatible; BoxcarBot/1.1; +awesome@boxcar.io)" ], "url": "https://boxcar.io/", "description": "Boxcar web crawler for notifications", "tags": [ "seo" ] }, { "pattern": "BTWebClient", "addition_date": "2018/06/27", "instances": [ "BTWebClient/180B(9704)" ], "url": "http://www.utorrent.com/", "description": "µTorrent BitTorrent Client", "tags": [ "http-library" ] }, { "pattern": "ContextAd Bot", "addition_date": "2018/06/27", "instances": [ "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.0;.NET CLR 1.0.3705; ContextAd Bot 1.0)", "ContextAd Bot 1.0" ], "description": "ContextAd web crawler for advertising", "tags": [ "advertising" ] }, { "pattern": "Digincore bot", "addition_date": "2018/06/27", "instances": [ "Mozilla/5.0 (compatible; Digincore bot; https://www.digincore.com/crawler.html for rules and instructions.)" ], "url": "http://www.digincore.com/crawler.html", "description": "Digincore web crawler for content analysis", "tags": [ "seo" ] }, { "pattern": "Disqus", "addition_date": "2018/06/27", "instances": [ "Disqus/1.0" ], "url": "https://disqus.com/", "description": "validate and quality check pages.", "tags": [ "seo" ] }, { "pattern": "Feedly", "addition_date": "2018/06/27", "instances": [ "Feedly/1.0 (+http://www.feedly.com/fetcher.html; like FeedFetcher-Google)", "FeedlyBot/1.0 (http://feedly.com)" ], "url": "https://www.feedly.com/fetcher.html", "description": "Feedly Fetcher is how Feedly grabs RSS or Atom feeds when users choose to add them to their Feedly or any of the other applications built on top of the feedly cloud.", "tags": [ "feed-reader" ] }, { "pattern": "Fetch\\/", "addition_date": "2018/06/27", "instances": [ "Fetch/2.0a (CMS Detection/Web/SEO analysis tool, see http://guess.scritch.org)" ], "description": "Fetch web crawler for CMS detection", "tags": [ "seo" ] }, { "pattern": "Fever", "addition_date": "2018/06/27", "instances": [ "Fever/1.38 (Feed Parser; http://feedafever.com; Allow like Gecko)" ], "url": "http://feedafever.com", "description": "Fever feed reader web crawler bot", "tags": [ "feed-reader" ] }, { "pattern": "Flamingo_SearchEngine", "addition_date": "2018/06/27", "instances": [ "Flamingo_SearchEngine (+http://www.flamingosearch.com/bot)" ], "description": "Flamingo search engine web crawler bot", "tags": [ "search-engine" ] }, { "pattern": "FlipboardProxy", "addition_date": "2018/06/27", "instances": [ "Mozilla/5.0 (compatible; FlipboardProxy/1.1; +http://flipboard.com/browserproxy)", "Mozilla/5.0 (compatible; FlipboardProxy/1.2; +http://flipboard.com/browserproxy)", "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10.6; en-US; rv:1.9.2) Gecko/20100115 Firefox/3.6 (FlipboardProxy/1.1; +http://flipboard.com/browserproxy)", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.9; rv:28.0) Gecko/20100101 Firefox/28.0 (FlipboardProxy/1.1; +http://flipboard.com/browserproxy)", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.11; rv:49.0) Gecko/20100101 Firefox/49.0 (FlipboardProxy/1.2; +http://flipboard.com/browserproxy)" ], "url": "https://about.flipboard.com/browserproxy/", "description": "a proxy service to fetch, validate, and prepare certain elements of websites for presentation through the Flipboard Application", "tags": [ "social-preview" ] }, { "pattern": "g2reader-bot", "addition_date": "2018/06/27", "instances": [ "g2reader-bot/1.0 (+http://www.g2reader.com/)" ], "url": "http://www.g2reader.com/", "description": "G2Reader web crawler for content discovery", "tags": [ "feed-reader" ] }, { "pattern": "G2 Web Services", "addition_date": "2019/03/01", "instances": [ "G2 Web Services/1.0 (built with StormCrawler Archetype 1.8; https://www.g2webservices.com/; developers@g2llc.com)" ], "url": "https://www.g2webservices.com/", "description": "G2 web services web crawler bot", "tags": [ "seo" ] }, { "pattern": "imrbot", "addition_date": "2018/06/27", "instances": [ "Mozilla/5.0 (compatible; imrbot/1.10.8 +http://www.mignify.com)" ], "url": "http://www.mignify.com", "description": "Mignify imrbot web crawler bot", "tags": [ "seo" ] }, { "pattern": "K7MLWCBot", "addition_date": "2018/06/27", "instances": [ "K7MLWCBot/1.0 (+http://www.k7computing.com)" ], "url": "http://www.k7computing.com", "description": "Virus scanner", "tags": [ "scanner" ] }, { "pattern": "Kemvibot", "addition_date": "2018/06/27", "instances": [ "Kemvibot/1.0 (http://kemvi.com, marco@kemvi.com)" ], "url": "http://kemvi.com", "description": "Kemvi web crawler for content discovery", "tags": [ "seo" ] }, { "pattern": "Landau-Media-Spider", "addition_date": "2018/06/27", "instances": [ "Landau-Media-Spider/1.0(http://bots.landaumedia.de/bot.html)" ], "url": "http://bots.landaumedia.de/bot.html", "description": "Landau Media web crawler bot", "tags": [ "seo" ] }, { "pattern": "linkapediabot", "addition_date": "2018/06/27", "instances": [ "linkapediabot (+http://www.linkapedia.com)" ], "url": "http://www.linkapedia.com", "description": "Linkapedia web crawler for link discovery", "tags": [ "seo" ] }, { "pattern": "vkShare", "addition_date": "2018/07/02", "instances": [ "Mozilla/5.0 (compatible; vkShare; +http://vk.com/dev/Share)" ], "url": "http://vk.com/dev/Share", "description": "VK Share web crawler for social sharing", "tags": [ "social-preview" ] }, { "pattern": "Siteimprove\\.com", "addition_date": "2018/06/22", "instances": [ "Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; Trident/6.0) LinkCheck by Siteimprove.com", "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.0) Match by Siteimprove.com", "Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; Trident/6.0) SiteCheck-sitecrawl by Siteimprove.com", "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.0) LinkCheck by Siteimprove.com" ], "description": "Siteimprove web crawler for site analysis", "tags": [ "monitoring" ] }, { "pattern": "BLEXBot\\/", "addition_date": "2018/07/07", "instances": [ "Mozilla/5.0 (compatible; BLEXBot/1.0; +http://webmeup-crawler.com/)" ], "url": "http://webmeup-crawler.com", "description": "WebMeUp BLEX web crawler bot", "tags": [ "seo" ] }, { "pattern": "DareBoost", "addition_date": "2018/07/07", "instances": [ "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/62.0.3202.75 Safari/537.36 DareBoost" ], "url": "https://www.dareboost.com/", "description": "Bot to test, Analyze and Optimize website", "tags": [ "monitoring" ] }, { "pattern": "ZuperlistBot\\/", "addition_date": "2018/07/07", "instances": [ "Mozilla/5.0 (compatible; ZuperlistBot/1.0)" ], "description": "Zuperlist web crawler for content discovery", "tags": [ "seo" ] }, { "pattern": "Miniflux\\/", "addition_date": "2018/07/07", "instances": [ "Mozilla/5.0 (compatible; Miniflux/2.0.x-dev; +https://miniflux.net)", "Mozilla/5.0 (compatible; Miniflux/2.0.3; +https://miniflux.net)", "Mozilla/5.0 (compatible; Miniflux/2.0.7; +https://miniflux.net)", "Mozilla/5.0 (compatible; Miniflux/2.0.10; +https://miniflux.net)", "Mozilla/5.0 (compatibl$; Miniflux/2.0.x-dev; +https://miniflux.app)", "Mozilla/5.0 (compatible; Miniflux/2.0.11; +https://miniflux.app)", "Mozilla/5.0 (compatible; Miniflux/2.0.12; +https://miniflux.app)", "Mozilla/5.0 (compatible; Miniflux/ae1dc1a; +https://miniflux.app)", "Mozilla/5.0 (compatible; Miniflux/3b6e44c; +https://miniflux.app)" ], "url": "https://miniflux.net", "description": "Miniflux is a minimalist and opinionated feed reader.", "tags": [ "feed-reader" ] }, { "pattern": "Feedspot", "addition_date": "2018/07/07", "instances": [ "Mozilla/5.0 (compatible; Feedspotbot/1.0; +http://www.feedspot.com/fs/bot)", "Mozilla/5.0 (compatible; Feedspot/1.0 (+https://www.feedspot.com/fs/fetcher; like FeedFetcher-Google)" ], "url": "http://www.feedspot.com/fs/bot", "description": "Feedspot web crawler for feed discovery", "tags": [ "feed-reader" ] }, { "pattern": "Diffbot\\/", "addition_date": "2018/07/07", "instances": [ "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.1.2) Gecko/20090729 Firefox/3.5.2 (.NET CLR 3.5.30729; Diffbot/0.1; +http://www.diffbot.com)" ], "url": "http://www.diffbot.com", "description": "Diffbot web crawler for content extraction", "tags": [ "seo" ] }, { "pattern": "SEOkicks", "addition_date": "2018/08/22", "instances": [ "Mozilla/5.0 (compatible; SEOkicks; +https://www.seokicks.de/robot.html)" ], "url": "https://www.seokicks.de/robot.html", "description": "SEOkicks web crawler for SEO analysis", "tags": [ "seo" ] }, { "pattern": "tracemyfile", "addition_date": "2018/08/23", "instances": [ "Mozilla/5.0 (compatible; tracemyfile/1.0; +bot@tracemyfile.com)" ], "description": "TraceMyFile web crawler for file tracking", "tags": [ "seo" ] }, { "pattern": "Nimbostratus-Bot", "addition_date": "2018/08/29", "instances": [ "Mozilla/5.0 (compatible; Nimbostratus-Bot/v1.3.2; http://cloudsystemnetworks.com)" ], "description": "Nimbostratus web crawler for cloud analysis", "tags": [ "seo" ] }, { "pattern": "zgrab", "addition_date": "2018/08/30", "instances": [ "Mozilla/5.0 zgrab/0.x" ], "url": "https://github.com/zmap/zgrab2", "description": "Zgrab web crawler for security scanning", "tags": [ "scanner" ] }, { "pattern": "PR-CY\\.RU", "addition_date": "2018/08/30", "instances": [ "Mozilla/5.0 (compatible; PR-CY.RU; + https://a.pr-cy.ru)" ], "url": "https://a.pr-cy.ru/", "description": "PR-CY web crawler for SEO analysis", "tags": [ "seo" ] }, { "pattern": "AdsTxtCrawler", "addition_date": "2018/08/30", "instances": [ "AdsTxtCrawler/1.0" ], "description": "AdsTxt web crawler for ads.txt validation", "tags": [ "advertising" ] }, { "pattern": "Datafeedwatch", "addition_date": "2018/09/05", "instances": [ "Datafeedwatch/2.1.x" ], "url": "https://www.datafeedwatch.com/", "description": "Datafeedwatch web crawler for feed analysis", "tags": [ "seo" ] }, { "pattern": "Zabbix", "addition_date": "2018/09/05", "instances": [ "Zabbix" ], "url": "https://www.zabbix.com/documentation/3.4/manual/web_monitoring", "description": "Zabbix web crawler for monitoring", "tags": [ "monitoring" ] }, { "pattern": "TangibleeBot", "addition_date": "2018/09/05", "instances": [ "TangibleeBot/1.0.0.0 (http://tangiblee.com/bot)" ], "url": "http://tangiblee.com/bot", "description": "Tangiblee web crawler for visual search", "tags": [ "seo" ] }, { "pattern": "google-xrawler", "addition_date": "2018/09/05", "instances": [ "google-xrawler" ], "url": "https://webmasters.stackexchange.com/questions/105560/what-is-the-google-xrawler-user-agent-used-for", "description": "Google xrawler web crawler bot", "tags": [ "search-engine" ] }, { "pattern": "axios", "addition_date": "2018/09/06", "instances": [ "axios/0.18.0", "axios/0.19.0" ], "url": "https://github.com/axios/axios", "description": "Axios HTTP client library for requests", "tags": [ "http-library" ] }, { "pattern": "Amazon CloudFront", "addition_date": "2018/09/07", "instances": [ "Amazon CloudFront" ], "url": "https://aws.amazon.com/cloudfront/", "description": "Amazon CloudFront web crawler bot", "tags": [ "http-library" ] }, { "pattern": "Pulsepoint ", "addition_date": "2018/09/24", "instances": [ "Pulsepoint XT3 web scraper" ], "description": "Pulsepoint web crawler for content discovery", "tags": [ "advertising" ] }, { "pattern": "CloudFlare-AlwaysOnline", "addition_date": "2018/09/27", "instances": [ "Mozilla/5.0 (compatible; CloudFlare-AlwaysOnline/1.0; +http://www.cloudflare.com/always-online) AppleWebKit/534.34", "Mozilla/5.0 (compatible; CloudFlare-AlwaysOnline/1.0; +https://www.cloudflare.com/always-online) AppleWebKit/534.34" ], "url": "https://www.cloudflare.com/always-online/", "description": "CloudFlare always online web crawler", "tags": [ "monitoring" ] }, { "pattern": "Cloudflare-Healthchecks", "addition_date": "2024/12/17", "instances": [ "Mozilla/5.0 (compatible; Cloudflare-Healthchecks/1.0; +https://www.cloudflare.com/; healthcheck-id: AAAAAAAAAAAAAAAA)" ], "url": "https://developers.cloudflare.com/health-checks/", "description": "CloudFlare health checks web crawler", "tags": [ "monitoring" ] }, { "pattern": "Cloudflare-Traffic-Manager", "addition_date": "2024/12/17", "instances": [ "Mozilla/5.0 (compatible; Cloudflare-Traffic-Manager/1.0; +https://www.cloudflare.com/traffic-manager/; pool-id: AAAAAAAAAAAAAAAA)" ], "url": "https://developers.cloudflare.com/load-balancing/monitors/", "description": "CloudFlare traffic manager web crawler", "tags": [ "monitoring" ] }, { "pattern": "CloudFlare-Prefetch", "addition_date": "2024/12/17", "instances": [ "Mozilla/5.0 (compatible; CloudFlare-Prefetch/0.1; +http://www.cloudflare.com/)" ], "url": "https://developers.cloudflare.com/speed/optimization/content/prefetch-urls/", "description": "CloudFlare prefetch web crawler bot", "tags": [ "seo" ] }, { "pattern": "Cloudflare-SSLDetector", "addition_date": "2024/12/17", "instances": [ "Cloudflare-SSLDetector" ], "url": "https://developers.cloudflare.com/ssl/origin-configuration/ssl-tls-recommender/", "description": "CloudFlare SSL detector web crawler", "tags": [ "scanner" ] }, { "pattern": "https:\\/\\/developers\\.cloudflare\\.com\\/security-center\\/", "addition_date": "2024/12/17", "instances": [ "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/92.0.4515.107 Safari/537.36 (compatible; +https://developers.cloudflare.com/security-center/)" ], "url": "https://developers.cloudflare.com/ssl/origin-configuration/ssl-tls-recommender/", "description": "CloudFlare security center web crawler", "tags": [ "scanner" ] }, { "pattern": "Google-Structured-Data-Testing-Tool", "addition_date": "2018/10/02", "instances": [ "Mozilla/5.0 (compatible; Google-Structured-Data-Testing-Tool +https://search.google.com/structured-data/testing-tool)", "Mozilla/5.0 (compatible; Google-Structured-Data-Testing-Tool +http://developers.google.com/structured-data/testing-tool/)" ], "url": "https://search.google.com/structured-data/testing-tool", "description": "Google structured data testing web crawler", "tags": [ "search-engine" ] }, { "pattern": "WordupInfoSearch", "addition_date": "2018/10/07", "instances": [ "WordupInfoSearch/1.0" ], "description": "Wordup info search web crawler bot", "tags": [ "search-engine" ] }, { "pattern": "WebDataStats", "addition_date": "2018/10/08", "instances": [ "Mozilla/5.0 (compatible; WebDataStats/1.0 ; +https://webdatastats.com/policy.html)" ], "url": "https://webdatastats.com/", "description": "WebDataStats web crawler for data analysis", "tags": [ "seo" ] }, { "pattern": "HttpUrlConnection", "addition_date": "2018/10/08", "instances": [ "Jersey/2.25.1 (HttpUrlConnection 1.8.0_141)" ], "description": "Java HttpUrlConnection HTTP client", "tags": [ "http-library" ] }, { "pattern": "ZoomBot", "addition_date": "2018/10/10", "instances": [ "ZoomBot (Linkbot 1.0 http://suite.seozoom.it/bot.html)" ], "url": "http://suite.seozoom.it/bot.html", "description": "SEOZoom web crawler for SEO analysis", "tags": [ "seo" ] }, { "pattern": "VelenPublicWebCrawler", "addition_date": "2018/10/09", "url": "https://velen.io/", "instances": [ "VelenPublicWebCrawler (velen.io)" ], "description": "Velen web crawler for content discovery", "tags": [ "seo" ] }, { "pattern": "MoodleBot", "addition_date": "2018/10/10", "instances": [ "MoodleBot/1.0" ], "description": "Moodle web crawler for learning platforms", "tags": [ "seo" ] }, { "pattern": "jpg-newsbot", "addition_date": "2018/10/10", "instances": [ "jpg-newsbot/2.0; (+https://vipnytt.no/bots/)" ], "url": "https://vipnytt.no/bots/", "description": "JPG news web crawler bot", "tags": [ "seo" ] }, { "pattern": "outbrain", "addition_date": "2018/10/14", "instances": [ "Mozilla/5.0 (Java) outbrain" ], "url": "https://www.outbrain.com/help/advertisers/invalid-url/", "description": "Outbrain web crawler for content discovery", "tags": [ "advertising" ] }, { "pattern": "W3C_Validator", "addition_date": "2018/10/14", "instances": [ "W3C_Validator/1.3" ], "url": "https://validator.w3.org/services", "description": "W3C HTML validator web crawler bot", "tags": [ "monitoring" ] }, { "pattern": "Validator\\.nu", "addition_date": "2018/10/14", "instances": [ "Validator.nu/LV" ], "url": "https://validator.w3.org/services", "description": "Validator.nu HTML validator web crawler", "tags": [ "monitoring" ] }, { "pattern": "W3C-checklink", "addition_date": "2018/10/14", "depends_on": [ "libwww-perl" ], "instances": [ "W3C-checklink/2.90 libwww-perl/5.64", "W3C-checklink/3.6.2.3 libwww-perl/5.64", "W3C-checklink/4.2 [4.20] libwww-perl/5.803", "W3C-checklink/4.2.1 [4.21] libwww-perl/5.803", "W3C-checklink/4.3 [4.42] libwww-perl/5.805", "W3C-checklink/4.3 [4.42] libwww-perl/5.808", "W3C-checklink/4.3 [4.42] libwww-perl/5.820", "W3C-checklink/4.5 [4.154] libwww-perl/5.823", "W3C-checklink/4.5 [4.160] libwww-perl/5.823" ], "url": "https://validator.w3.org/services", "description": "W3C checklink web crawler for link validation", "tags": [ "monitoring" ] }, { "pattern": "W3C-mobileOK", "addition_date": "2018/10/14", "instances": [ "W3C-mobileOK/DDC-1.0" ], "url": "https://validator.w3.org/services", "description": "W3C mobile OK web crawler bot", "tags": [ "monitoring" ] }, { "pattern": "W3C_I18n-Checker", "addition_date": "2018/10/14", "instances": [ "W3C_I18n-Checker/1.0" ], "url": "https://validator.w3.org/services", "description": "W3C internationalization web crawler", "tags": [ "monitoring" ] }, { "pattern": "FeedValidator", "addition_date": "2018/10/14", "instances": [ "FeedValidator/1.3" ], "url": "https://validator.w3.org/services", "description": "Feed validator web crawler for RSS validation", "tags": [ "monitoring" ] }, { "pattern": "W3C_CSS_Validator", "addition_date": "2018/10/14", "instances": [ "Jigsaw/2.3.0 W3C_CSS_Validator_JFouffa/2.0" ], "url": "https://validator.w3.org/services", "description": "W3C CSS validator web crawler bot", "tags": [ "monitoring" ] }, { "pattern": "W3C_Unicorn", "addition_date": "2018/10/14", "instances": [ "W3C_Unicorn/1.0" ], "url": "https://validator.w3.org/services", "description": "W3C Unicorn validator web crawler bot", "tags": [ "monitoring" ] }, { "pattern": "Google-PhysicalWeb", "addition_date": "2018/10/21", "instances": [ "Mozilla/5.0 (Google-PhysicalWeb)" ], "description": "Google physical web web crawler bot", "tags": [ "search-engine" ] }, { "pattern": "Blackboard", "addition_date": "2018/10/28", "instances": [ "Blackboard Safeassign" ], "url": "https://help.blackboard.com/Learn/Administrator/Hosting/Tools_Management/SafeAssign", "description": "Blackboard SafeAssign web crawler bot", "tags": [ "academic" ] }, { "pattern": "ICBot\\/", "addition_date": "2018/10/23", "instances": [ "Mozilla/5.0 (compatible; ICBot/0.1; +https://ideasandcode.xyz" ], "url": "https://ideasandcode.xyz", "description": "Ideas and Code web crawler bot", "tags": [ "seo" ] }, { "pattern": "BazQux", "addition_date": "2018/10/23", "instances": [ "Mozilla/5.0 (compatible; BazQux/2.4; +https://bazqux.com/fetcher; 1 subscribers)" ], "url": "https://bazqux.com/fetcher", "description": "BazQux RSS reader web crawler bot", "tags": [ "feed-reader" ] }, { "pattern": "Twingly", "addition_date": "2018/10/23", "instances": [ "Mozilla/5.0 (compatible; Twingly Recon; twingly.com)" ], "url": "https://twingly.com", "description": "Twingly blog search web crawler bot", "tags": [ "seo" ] }, { "pattern": "Rivva", "addition_date": "2018/10/23", "instances": [ "Mozilla/5.0 (compatible; Rivva; http://rivva.de)" ], "url": "http://rivva.de", "description": "Rivva blog search web crawler bot", "tags": [ "seo" ] }, { "pattern": "Experibot", "addition_date": "2018/11/03", "instances": [ "Experibot-v2 http://goo.gl/ZAr8wX", "Experibot-v3 http://goo.gl/ZAr8wX" ], "url": "https://amirkr.wixsite.com/experibot", "description": "Experibot web crawler for testing", "tags": [ "monitoring" ] }, { "pattern": "awesomecrawler", "addition_date": "2018/11/24", "instances": [ "Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.22 (KHTML, like Gecko) Chrome/25.0.1364.5 Safari/537.22 +awesomecrawler" ], "description": "Awesome web crawler for content discovery", "tags": [ "seo" ] }, { "pattern": "Dataprovider\\.com", "addition_date": "2018/11/24", "instances": [ "Mozilla/5.0 (compatible; Dataprovider.com)" ], "url": "https://www.dataprovider.com/", "description": "Dataprovider web crawler for data collection", "tags": [ "seo" ] }, { "pattern": "GroupHigh\\/", "addition_date": "2018/11/24", "instances": [ "Mozilla/5.0 (compatible; GroupHigh/1.0; +http://www.grouphigh.com/" ], "url": "http://www.grouphigh.com/", "description": "GroupHigh web crawler for influencer marketing", "tags": [ "seo" ] }, { "pattern": "theoldreader\\.com", "addition_date": "2018/12/02", "instances": [ "Mozilla/5.0 (compatible; theoldreader.com)" ], "url": "https://www.theoldreader.com/", "description": "The Old Reader web crawler bot", "tags": [ "feed-reader" ] }, { "pattern": "AnyEvent", "addition_date": "2018/12/07", "instances": [ "Mozilla/5.0 (compatible; U; AnyEvent-HTTP/2.24; +http://software.schmorp.de/pkg/AnyEvent)" ], "url": "http://software.schmorp.de/pkg/AnyEvent.html", "description": "AnyEvent Perl HTTP web crawler library", "tags": [ "http-library" ] }, { "pattern": "Uptimebot\\.org", "addition_date": "2019/01/17", "instances": [ "Uptimebot.org - Free website monitoring" ], "url": "http://uptimebot.org/", "description": "Uptimebot website monitoring web crawler", "tags": [ "monitoring" ] }, { "pattern": "Nmap Scripting Engine", "addition_date": "2019/02/04", "instances": [ "Mozilla/5.0 (compatible; Nmap Scripting Engine; https://nmap.org/book/nse.html)" ], "url": "https://nmap.org/book/nse.html", "description": "Nmap NSE web crawler for security scanning", "tags": [ "scanner" ] }, { "pattern": "2ip\\.ru", "addition_date": "2019/02/12", "instances": [ "2ip.ru CMS Detector (https://2ip.ru/cms/)" ], "url": "https://2ip.ru/cms/", "description": "2IP CMS detector web crawler bot", "tags": [ "seo" ] }, { "pattern": "Clickagy", "addition_date": "2019/02/19", "instances": [ "Clickagy Intelligence Bot v2" ], "url": "https://www.clickagy.com", "description": "Clickagy intelligence web crawler bot", "tags": [ "advertising" ] }, { "pattern": "Caliperbot", "addition_date": "2019/03/02", "instances": [ "Caliperbot/1.0 (+http://www.conductor.com/caliperbot)" ], "url": "http://www.conductor.com/caliperbot", "description": "Conductor Caliperbot web crawler for analysis", "tags": [ "seo" ] }, { "pattern": "MBCrawler", "addition_date": "2019/03/02", "instances": [ "MBCrawler/1.0 (https://monitorbacklinks.com)" ], "url": "https://monitorbacklinks.com", "description": "Monitor Backlinks web crawler bot", "tags": [ "seo" ] }, { "pattern": "online-webceo-bot", "addition_date": "2019/03/02", "instances": [ "Mozilla/5.0 (compatible; online-webceo-bot/1.0; +http://online.webceo.com)" ], "url": "http://online.webceo.com", "description": "Online WebCEO web crawler for analysis", "tags": [ "seo" ] }, { "pattern": "B2B Bot", "addition_date": "2019/03/02", "instances": [ "B2B Bot" ], "description": "B2B web crawler for business discovery", "tags": [ "seo" ] }, { "pattern": "AddSearchBot", "addition_date": "2019/03/02", "instances": [ "Mozilla/5.0 (compatible; AddSearchBot/0.9; +http://www.addsearch.com/bot; info@addsearch.com)" ], "url": "http://www.addsearch.com/bot", "description": "AddSearch web crawler for site search", "tags": [ "search-engine" ] }, { "pattern": "Google Favicon", "addition_date": "2019/03/14", "instances": [ "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/49.0.2623.75 Safari/537.36 Google Favicon" ], "description": "Google favicon web crawler bot", "tags": [ "search-engine" ] }, { "pattern": "HubSpot", "addition_date": "2019/04/15", "instances": [ "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/27.0.1453.116 Safari/537.36 HubSpot Webcrawler - web-crawlers@hubspot.com", "Mozilla/5.0 (X11; Linux x86_64; HubSpot Single Page link check; web-crawlers+links@hubspot.com) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.103 Safari/537.36", "Mozilla/5.0 (compatible; HubSpot Crawler; web-crawlers@hubspot.com)", "HubSpot Connect 2.0 (http://dev.hubspot.com/) - BizOpsCompanies-Tq2-BizCoDomainValidationAudit" ], "description": "HubSpot web crawler for marketing automation", "tags": [ "seo" ] }, { "pattern": "Chrome-Lighthouse", "addition_date": "2019/03/15", "instances": [ "Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5 Build/MRA58N) AppleWebKit/537.36(KHTML, like Gecko) Chrome/69.0.3464.0 Mobile Safari/537.36 Chrome-Lighthouse", "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36(KHTML, like Gecko) Chrome/69.0.3464.0 Safari/537.36 Chrome-Lighthouse", "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3694.0 Safari/537.36 Chrome-Lighthouse", "Mozilla/5.0 (Linux; Android 7.0; Moto G (4)) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/98.0.4695.0 Mobile Safari/537.36 Chrome-Lighthouse", "Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5 Build/MRA58N) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3694.0 Mobile Safari/537.36 Chrome-Lighthouse" ], "url": "https://developers.google.com/speed/pagespeed/insights", "description": "Chrome Lighthouse web crawler for audits", "tags": [ "monitoring" ] }, { "pattern": "HeadlessChrome", "url": "https://developers.google.com/web/updates/2017/04/headless-chrome", "addition_date": "2019/06/17", "instances": [ "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) HeadlessChrome/74.0.3729.169 Safari/537.36", "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) HeadlessChrome/69.0.3494.0 Safari/537.36", "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) HeadlessChrome/76.0.3803.0 Safari/537.36" ], "description": "Headless Chrome web crawler for testing", "tags": [ "browser-automation" ] }, { "pattern": "CheckMarkNetwork\\/", "addition_date": "2019/06/30", "instances": [ "CheckMarkNetwork/1.0 (+http://www.checkmarknetwork.com/spider.html)" ], "url": "https://www.checkmarknetwork.com/", "description": "CheckMark Network web crawler bot", "tags": [ "monitoring" ] }, { "pattern": "www\\.uptime\\.com", "addition_date": "2019/07/21", "instances": [ "Mozilla/5.0 (compatible; Uptimebot/1.0; +http://www.uptime.com/uptimebot)" ], "url": "http://www.uptime.com/uptimebot", "description": "Uptime.com website monitoring web crawler", "tags": [ "monitoring" ] }, { "pattern": "Streamline3Bot\\/", "addition_date": "2019/07/21", "instances": [ "Mozilla/5.0 (compatible; MSIE 8.0; Windows NT 5.1) Streamline3Bot/1.0", "Mozilla/5.0 (Windows NT 6.1; Win64; x64; +https://www.ubtsupport.com/legal/Streamline3Bot.php) Streamline3Bot/1.0" ], "url": "https://www.ubtsupport.com/legal/Streamline3Bot.php", "description": "Streamline3 web crawler for monitoring", "tags": [ "monitoring" ] }, { "pattern": "serpstatbot\\/", "addition_date": "2019/07/25", "instances": [ "serpstatbot/1.0 (advanced backlink tracking bot; http://serpstatbot.com/; abuse@serpstatbot.com)", "serpstatbot/1.0 (advanced backlink tracking bot; curl/7.58.0; http://serpstatbot.com/; abuse@serpstatbot.com)" ], "url": "http://serpstatbot.com", "description": "Serpstat web crawler for backlink tracking", "tags": [ "seo" ] }, { "pattern": "MixnodeCache\\/", "addition_date": "2019/08/04", "instances": [ "MixnodeCache/1.8(+https://cache.mixnode.com/)" ], "url": "https://cache.mixnode.com/", "description": "Mixnode cache web crawler bot", "tags": [ "seo" ] }, { "pattern": "^curl", "addition_date": "2019/08/15", "instances": [ "curl", "curl/7.29.0", "curl/7.47.0", "curl/7.54.0", "curl/7.55.1", "curl/7.64.0", "curl/7.64.1", "curl/7.65.3" ], "url": "https://curl.haxx.se/", "description": "cURL command-line tool for HTTP requests", "tags": [ "http-library" ] }, { "pattern": "SimpleScraper", "addition_date": "2019/08/16", "instances": [ "Mozilla/5.0 (compatible; SimpleScraper)" ], "url": "https://github.com/ramonkcom/simple-scraper/", "description": "Simple Scraper web scraping tool", "tags": [ "seo" ] }, { "pattern": "RSSingBot", "addition_date": "2019/09/15", "instances": [ "RSSingBot (http://www.rssing.com)" ], "url": "http://www.rssing.com", "description": "RSSing RSS reader web crawler bot", "tags": [ "feed-reader" ] }, { "pattern": "Jooblebot", "addition_date": "2019/09/25", "instances": [ "Mozilla/5.0 (compatible; Jooblebot/2.0; Windows NT 6.1; WOW64; +http://jooble.org/jooble-bot) AppleWebKit/537.36 (KHTML, like Gecko) Safari/537.36" ], "url": "http://jooble.org/jooble-bot", "description": "Jooble job search web crawler bot", "tags": [ "search-engine" ] }, { "pattern": "fedoraplanet", "addition_date": "2019/09/28", "instances": [ "venus/fedoraplanet" ], "url": "http://fedoraplanet.org/", "description": "Fedora Planet web crawler for aggregation", "tags": [ "feed-reader" ] }, { "pattern": "Friendica", "addition_date": "2019/09/28", "instances": [ "Friendica 'The Tazmans Flax-lily' 2019.01-1293; https://hoyer.xyz" ], "url": "https://hoyer.xyz", "description": "Friendica social network web crawler bot", "tags": [ "social-preview" ] }, { "pattern": "NextCloud", "addition_date": "2019/09/30", "instances": [ "NextCloud-News/1.0" ], "url": "https://nextcloud.com/", "description": "NextCloud news reader web crawler bot", "tags": [ "feed-reader" ] }, { "pattern": "Tiny Tiny RSS", "addition_date": "2019/10/04", "instances": [ "Tiny Tiny RSS/1.15.3 (http://tt-rss.org/)", "Tiny Tiny RSS/17.12 (a2d1fa5) (http://tt-rss.org/)", "Tiny Tiny RSS/19.2 (b68db2d) (http://tt-rss.org/)", "Tiny Tiny RSS/19.8 (http://tt-rss.org/)" ], "url": "http://tt-rss.org/", "description": "Tiny Tiny RSS reader web crawler bot", "tags": [ "feed-reader" ] }, { "pattern": "RegionStuttgartBot", "addition_date": "2019/10/17", "instances": [ "Mozilla/5.0 (compatible; RegionStuttgartBot/1.0; +http://it.region-stuttgart.de/competenzatlas/unternehmen-suchen/)" ], "url": "http://it.region-stuttgart.de/competenzatlas/unternehmen-suchen/", "description": "Region Stuttgart web crawler bot", "tags": [ "seo" ] }, { "pattern": "Bytespider", "addition_date": "2019/11/11", "instances": [ "Mozilla/5.0 (iPhone; CPU iPhone OS 11_0 like Mac OS X) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/40.0.3754.1902 Mobile Safari/537.36; Bytespider", "Mozilla/5.0 (iPhone; CPU iPhone OS 11_0 like Mac OS X) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.4454.1745 Mobile Safari/537.36; Bytespider", "Mozilla/5.0 (iPhone; CPU iPhone OS 11_0 like Mac OS X) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/43.0.7597.1164 Mobile Safari/537.36; Bytespider;bytespider@bytedance.com", "Mozilla/5.0 (iPhone; CPU iPhone OS 11_0 like Mac OS X) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/46.0.2988.1545 Mobile Safari/537.36; Bytespider", "Mozilla/5.0 (iPhone; CPU iPhone OS 11_0 like Mac OS X) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.4141.1682 Mobile Safari/537.36; Bytespider", "Mozilla/5.0 (iPhone; CPU iPhone OS 11_0 like Mac OS X) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.3478.1649 Mobile Safari/537.36; Bytespider", "Mozilla/5.0 (Linux; Android 5.0; SM-G900P Build/LRX21T) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/47.0.5267.1259 Mobile Safari/537.36; Bytespider", "Mozilla/5.0 (Linux; Android 5.0; SM-G900P Build/LRX21T) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.7990.1979 Mobile Safari/537.36; Bytespider", "Mozilla/5.0 (Linux; Android 5.0; SM-G900P Build/LRX21T) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.2268.1523 Mobile Safari/537.36; Bytespider", "Mozilla/5.0 (Linux; Android 6.0; Nexus 5 Build/MRA58N) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/40.0.2576.1836 Mobile Safari/537.36; Bytespider", "Mozilla/5.0 (Linux; Android 6.0; Nexus 5 Build/MRA58N) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/43.0.9681.1227 Mobile Safari/537.36; Bytespider", "Mozilla/5.0 (Linux; Android 6.0; Nexus 5 Build/MRA58N) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/44.0.6023.1635 Mobile Safari/537.36; Bytespider", "Mozilla/5.0 (Linux; Android 6.0; Nexus 5 Build/MRA58N) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/50.0.4944.1981 Mobile Safari/537.36; Bytespider", "Mozilla/5.0 (Linux; Android 6.0; Nexus 5 Build/MRA58N) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.3613.1739 Mobile Safari/537.36; Bytespider", "Mozilla/5.0 (Linux; Android 6.0; Nexus 5 Build/MRA58N) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.4022.1033 Mobile Safari/537.36; Bytespider", "Mozilla/5.0 (Linux; Android 6.0; Nexus 5 Build/MRA58N) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/52.0.3248.1547 Mobile Safari/537.36; Bytespider", "Mozilla/5.0 (Linux; Android 6.0; Nexus 5 Build/MRA58N) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/57.0.5527.1507 Mobile Safari/537.36; Bytespider", "Mozilla/5.0 (Linux; Android 8.0; Pixel 2 Build/OPD3.170816.012) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.5216.1326 Mobile Safari/537.36; Bytespider", "Mozilla/5.0 (Linux; Android 8.0; Pixel 2 Build/OPD3.170816.012) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.9038.1080 Mobile Safari/537.36; Bytespider" ], "url": "https://stackoverflow.com/questions/57908900/what-is-the-bytespider-user-agent", "description": "ByteDance Bytespider web crawler bot", "tags": [ "ai-crawler" ] }, { "pattern": "Datanyze", "addition_date": "2019/11/17", "instances": [ "Mozilla/5.0 (X11; Datanyze; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.181 Safari/537.36" ], "url": "https://www.datanyze.com/dnyzbot/", "description": "Datanyze web crawler for technology detection", "tags": [ "seo" ] }, { "pattern": "Google-Site-Verification", "addition_date": "2019/12/11", "instances": [ "Mozilla/5.0 (compatible; Google-Site-Verification/1.0)" ], "url": "https://support.google.com/webmasters/answer/9008080", "description": "Google site verification web crawler bot", "tags": [ "search-engine" ] }, { "pattern": "TrendsmapResolver", "addition_date": "2020/02/24", "instances": [ "Mozilla/5.0 (compatible; TrendsmapResolver/0.1)" ], "url": "https://www.trendsmap.com/", "description": "Trendsmap web crawler for trend analysis", "tags": [ "seo" ] }, { "pattern": "tweetedtimes", "addition_date": "2020/02/24", "instances": [ "Mozilla/5.0 (compatible; +http://tweetedtimes.com)" ], "url": "https://tweetedtimes.com/", "description": "Tweeted Times web crawler for news", "tags": [ "seo" ] }, { "pattern": "NTENTbot", "addition_date": "2020/02/24", "instances": [ "Mozilla/5.0 (compatible; NTENTbot; +http://www.ntent.com/ntentbot)" ], "url": "https://ntent.com/ntentbot/", "description": "NTENT web crawler for search results", "tags": [ "search-engine" ] }, { "pattern": "Gwene", "addition_date": "2020/02/24", "instances": [ "Gwene/1.0 (The gwene.org rss-to-news gateway) Googlebot" ], "url": "https://gwene.org", "description": "Gwene RSS to news gateway web crawler", "tags": [ "feed-reader" ] }, { "pattern": "SimplePie", "addition_date": "2020/02/24", "instances": [ "SimplePie/1.3-dev (Feed Parser; http://simplepie.org; Allow like Gecko)" ], "url": "http://simplepie.org", "description": "SimplePie PHP feed parser web crawler", "tags": [ "feed-reader" ] }, { "pattern": "SearchAtlas", "addition_date": "2020/03/02", "instances": [ "SearchAtlas.com SEO Crawler" ], "url": "http://SearchAtlas.com", "description": "SearchAtlas SEO web crawler for analysis", "tags": [ "seo" ] }, { "pattern": "Superfeedr", "addition_date": "2020/03/02", "instances": [ "Superfeedr bot/2.0 http://superfeedr.com - Make your feeds realtime: get in touch - feed-id:1162088860" ], "url": "http://superfeedr.com", "description": "Superfeedr feed reader web crawler bot", "tags": [ "feed-reader" ] }, { "pattern": "feedbot", "addition_date": "2020/03/02", "instances": [ "wp.com feedbot/1.0 (+https://wp.com)" ], "url": "http://wp.com", "description": "WordPress.com feed web crawler bot", "tags": [ "feed-reader" ] }, { "pattern": "UT-Dorkbot", "addition_date": "2020/03/02", "instances": [ "UT-Dorkbot/1.0" ], "url": "https://security.utexas.edu/dorkbot", "description": "University of Texas security web crawler", "tags": [ "scanner" ] }, { "pattern": "Amazonbot", "addition_date": "2020/03/02", "instances": [ "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/600.2.5 (KHTML, like Gecko) Version/8.0.2 Safari/600.2.5 (Amazonbot/0.1; +https://developer.amazon.com/support/amazonbot)" ], "url": "https://developer.amazon.com/support/amazonbot", "description": "Amazon web crawler for product discovery", "tags": [ "search-engine" ] }, { "pattern": "AmazonProductDiscovery", "addition_date": "2025/12/22", "instances": [ "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/110.0.0.0 Safari/537.36 (compatible; AmazonProductDiscovery/1.0; https://vendorcentral.amazon.com/support/amazonproductbot)", "Mozilla/5.0 (X11; Linux x86_64; rv:38.0) Gecko/20100101 Firefox/38.0 (compatible; AmazonProductDiscovery/1.0; https://vendorcentral.amazon.com/support/amazonproductbot)" ], "url": "https://vendorcentral.amazon.com/support/amazonproductbot", "description": "Amazon product discovery web crawler", "tags": [ "search-engine" ] }, { "pattern": "AmazonSellerInitiatedListing", "addition_date": "2025/12/22", "instances": [ "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36 (compatible; AmazonSellerInitiatedListing/1.0; https://vendorcentral.amazon.com/support/amazonproductbot)" ], "url": "https://vendorcentral.amazon.com/support/amazonproductbot", "description": "Amazon seller listing web crawler", "tags": [ "search-engine" ] }, { "pattern": "SerendeputyBot", "addition_date": "2020/03/02", "instances": [ "SerendeputyBot/0.8.6 (http://serendeputy.com/about/serendeputy-bot)" ], "url": "http://serendeputy.com/about/serendeputy-bot", "description": "Serendeputy web crawler for content discovery", "tags": [ "seo" ] }, { "pattern": "Eyeotabot", "addition_date": "2020/03/02", "instances": [ "Mozilla/5.0 (compatible; Eyeotabot/1.0; +http://www.eyeota.com)" ], "url": "http://www.eyeota.com", "description": "Eyeota web crawler for audience data", "tags": [ "advertising" ] }, { "pattern": "officestorebot", "addition_date": "2020/03/02", "instances": [ "Mozilla/5.0 (compatible; officestorebot/1.0; +https://aka.ms/officestorebot)" ], "url": "https://aka.ms/officestorebot", "description": "Microsoft Office Store web crawler bot", "tags": [ "search-engine" ] }, { "pattern": "Neticle Crawler", "addition_date": "2020/03/02", "instances": [ "Neticle Crawler v1.0 ( https://neticle.com/bot/en/ )" ], "url": "https://neticle.com/bot/en/", "description": "Neticle web crawler for content analysis", "tags": [ "seo" ] }, { "pattern": "SurdotlyBot", "addition_date": "2020/03/02", "instances": [ "Mozilla/5.0 (compatible; SurdotlyBot/1.0; +http://sur.ly/bot.html; Linux; Android 4; iPhone; CPU iPhone OS 6_0_1 like Mac OS X)" ], "url": "http://sur.ly/bot.html", "description": "Surly web crawler for link shortening", "tags": [ "seo" ] }, { "pattern": "LinkisBot", "addition_date": "2020/03/02", "instances": [ "Mozilla/5.0 (compatible; LinkisBot/1.0; bot@linkis.com) (iPhone; CPU iPhone OS 8_4_1 like Mac OS X) Mobile/12H321" ], "description": "Linkis web crawler for link sharing", "tags": [ "seo" ] }, { "pattern": "AwarioSmartBot", "addition_date": "2020/03/02", "instances": [ "AwarioSmartBot/1.0 (+https://awario.com/bots.html; bots@awario.com)" ], "url": "https://awario.com/bots.html", "description": "Awario smart web crawler bot", "tags": [ "seo" ] }, { "pattern": "AwarioRssBot", "addition_date": "2020/03/02", "instances": [ "AwarioRssBot/1.0 (+https://awario.com/bots.html; bots@awario.com)" ], "url": "https://awario.com/bots.html", "description": "Awario RSS feed web crawler bot", "tags": [ "feed-reader", "seo" ] }, { "pattern": "RyteBot", "addition_date": "2020/03/02", "instances": [ "RyteBot/1.0.0 (+https://bot.ryte.com/)" ], "url": "https://bot.ryte.com/", "description": "Ryte web crawler for site analysis", "tags": [ "seo" ] }, { "pattern": "FreeWebMonitoring SiteChecker", "addition_date": "2020/03/02", "instances": [ "FreeWebMonitoring SiteChecker/0.2 (+https://www.freewebmonitoring.com/bot.html)" ], "url": "https://www.freewebmonitoring.com/bot.html", "description": "FreeWebMonitoring site checker web crawler", "tags": [ "monitoring" ] }, { "pattern": "AspiegelBot", "addition_date": "2020/03/16", "instances": [ "Mozilla/5.0 (Linux; Android 7.0;) AppleWebKit/537.36 (KHTML, like Gecko) Mobile Safari/537.36 (compatible; AspiegelBot)" ], "url": "https://aspiegel.com", "description": "Aspiegel web crawler for content discovery", "tags": [ "search-engine" ] }, { "pattern": "NAVER Blog Rssbot", "addition_date": "2020/03/16", "instances": [ "NAVER Blog Rssbot" ], "url": "http://www.naver.com", "description": "Naver blog RSS web crawler bot", "tags": [ "feed-reader" ] }, { "pattern": "zenback bot", "addition_date": "2020/03/16", "instances": [ "Mozilla/5.0 (compatible; zenback bot; powered by logly +http://corp.logly.co.jp/)" ], "url": "http://corp.logly.co.jp/", "description": "Zenback web crawler for content discovery", "tags": [ "seo" ] }, { "pattern": "SentiBot", "addition_date": "2020/03/16", "instances": [ "SentiBot www.sentibot.eu (compatible with Googlebot)" ], "url": "https://sites.google.com/senti1.com/sentibot-eu/home", "description": "SentiBot web crawler for sentiment analysis", "tags": [ "seo" ] }, { "pattern": "Domains Project\\/", "addition_date": "2020/03/16", "instances": [ "Mozilla/5.0 (compatible; Domains Project/1.0.3; +https://github.com/tb0hdan/domains)" ], "url": "https://github.com/tb0hdan/domains", "description": "Domains Project web crawler bot", "tags": [ "seo" ] }, { "pattern": "Pandalytics", "addition_date": "2020/03/16", "instances": [ "Pandalytics/1.0 (https://domainsbot.com/pandalytics/)" ], "url": "https://domainsbot.com/pandalytics/", "description": "Pandalytics web crawler for domain analysis", "tags": [ "seo" ] }, { "pattern": "VKRobot", "addition_date": "2020/03/16", "instances": [ "Mozilla/5.0 (compatible; VKRobot/1.0)" ], "description": "VK social network web crawler bot", "tags": [ "social-preview" ] }, { "pattern": "bidswitchbot", "addition_date": "2020/03/16", "instances": [ "bidswitchbot/1.0" ], "url": "https://www.bidswitch.com/about-us/", "description": "BidSwitch web crawler for advertising", "tags": [ "advertising" ] }, { "pattern": "tigerbot", "addition_date": "2020/03/16", "instances": [ "tigerbot" ], "description": "Tiger web crawler for content discovery", "tags": [ "seo" ] }, { "pattern": "NIXStatsbot", "addition_date": "2020/03/16", "instances": [ "Mozilla/5.0 (compatible; NIXStatsbot/1.1; +http://www.nixstats.com/bot.html)" ], "url": "http://www.nixstats.com/bot.html", "description": "NIXStats web crawler for monitoring", "tags": [ "monitoring" ] }, { "pattern": "Atom Feed Robot", "addition_date": "2020/03/16", "instances": [ "RSSMicro.com RSS/Atom Feed Robot" ], "url": "https://rssmicro.com", "description": "RSSMicro atom feed web crawler bot", "tags": [ "feed-reader" ] }, { "pattern": "[Cc]urebot", "addition_date": "2020/03/16", "instances": [ "Curebot/1.0", "curebot-feed-fetcher" ], "description": "Cure web crawler for content discovery", "tags": [ "monitoring" ] }, { "pattern": "PagePeeker\\/", "addition_date": "2020/03/16", "instances": [ "Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.120 Safari/537.36 (compatible; PagePeeker/3.0; +https://pagepeeker.com/robots/)" ], "url": "https://pagepeeker.com/robots/", "description": "PagePeeker web crawler for screenshots", "tags": [ "seo" ] }, { "pattern": "Vigil\\/", "addition_date": "2020/03/16", "instances": [ "Mozilla/5.0 (compatible; Vigil/1.0; +http://vigil-app.com/bot.html)" ], "url": "http://vigil-app.com/bot.html", "description": "Vigil web crawler for monitoring", "tags": [ "monitoring" ] }, { "pattern": "rssbot\\/", "addition_date": "2020/03/16", "instances": [ "rssbot/1.4.3 (+https://t.me/RustRssBot)" ], "url": "https://github.com/iovxw/rssbot", "description": "RSS bot web crawler for feeds", "tags": [ "feed-reader" ] }, { "pattern": "startmebot\\/", "addition_date": "2020/03/16", "instances": [ "Mozilla/5.0 (compatible; startmebot/1.0; +https://start.me/bot)" ], "url": "https://start.me/bot", "description": "Start.me web crawler for content discovery", "tags": [ "seo" ] }, { "pattern": "JobboerseBot", "addition_date": "2020/03/16", "instances": [ "Mozilla/5.0 (X11; U; Linux Core i7-4980HQ; de; rv:32.0; compatible; JobboerseBot; http://www.jobboerse.com/bot.htm) Gecko/20100101 Firefox/38.0" ], "url": "http://www.jobboerse.com/bot.htm", "description": "Jobboerse web crawler for job discovery", "tags": [ "search-engine" ] }, { "pattern": "seewithkids", "addition_date": "2020/03/16", "instances": [ "http://seewithkids.com/bot" ], "url": "http://seewithkids.com/bot", "description": "SeeWithKids web crawler for content discovery", "tags": [ "seo" ] }, { "pattern": "NINJA bot", "addition_date": "2020/03/16", "instances": [ "NINJA bot" ], "description": "NINJA web crawler for content discovery", "tags": [ "scanner" ] }, { "pattern": "Cutbot", "addition_date": "2020/03/16", "instances": [ "Cutbot; 1.5; http://cutbot.net/" ], "url": "http://cutbot.net/", "description": "Cutbot web crawler for content discovery", "tags": [ "seo" ] }, { "pattern": "BublupBot", "addition_date": "2020/03/16", "instances": [ "BublupBot (+https://www.bublup.com/bublup-bot.html)" ], "url": "https://www.bublup.com/bublup-bot.html", "description": "Bublup web crawler for content discovery", "tags": [ "seo" ] }, { "pattern": "BrandONbot", "addition_date": "2020/03/16", "instances": [ "BrandONbot (http://brandonmedia.net)" ], "url": "http://brandonmedia.net", "description": "BrandON web crawler for brand monitoring", "tags": [ "seo" ] }, { "pattern": "RidderBot", "addition_date": "2020/03/16", "instances": [ "Mozilla/5.0 (compatible; RidderBot/1.0; bot@ridder.co)", "Mozilla/5.0 (compatible; RidderBot/1.0; bot@ridder.co) (iPhone; CPU iPhone OS 8_4_1 like Mac OS X) Mobile/12H321" ], "url": "https://ridder.co/", "description": "Ridder web crawler for content discovery", "tags": [ "seo" ] }, { "pattern": "Taboolabot", "addition_date": "2020/03/16", "instances": [ "Mozilla/5.0 (compatible; Taboolabot/3.7; +http://www.taboola.com)" ], "url": "http://www.taboola.com", "description": "Taboola web crawler for content discovery", "tags": [ "advertising" ] }, { "pattern": "Dubbotbot", "addition_date": "2020/03/16", "instances": [ "Mozilla/5.0 (compatible; Dubbotbot/0.2; +http://dubbot.com)" ], "url": "http://dubbot.com", "description": "Dubbot web crawler for content discovery", "tags": [ "monitoring" ] }, { "pattern": "FindITAnswersbot", "addition_date": "2020/03/16", "instances": [ "Mozilla/5.0 (compatible;FindITAnswersbot/1.0;+http://search.it-influentials.com/bot.htm)" ], "url": "http://search.it-influentials.com/bot.htm", "description": "FindITAnswers web crawler for discovery", "tags": [ "seo" ] }, { "pattern": "infoobot", "addition_date": "2020/03/16", "instances": [ "infoobot/0.1 (https://www.infoo.nl/bot.html)" ], "url": "https://www.infoo.nl/bot.html", "description": "Infoo web crawler for content discovery", "tags": [ "seo" ] }, { "pattern": "Refindbot", "addition_date": "2020/03/16", "instances": [ "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.103 Safari/537.36 (Refindbot/1.0)" ], "url": "https://refind.com/about", "description": "Refind web crawler for content discovery", "tags": [ "seo" ] }, { "pattern": "BlogTraffic\\/\\d\\.\\d+ Feed-Fetcher", "addition_date": "2020/03/16", "instances": [ "Mozilla/5.0 (compatible; BlogTraffic/1.4 Feed-Fetcher; +http://www.blogtraffic.de/rss-bot.html)" ], "url": "http://www.blogtraffic.de/rss-bot.html", "description": "BlogTraffic feed fetcher web crawler", "tags": [ "feed-reader" ] }, { "pattern": "SeobilityBot", "addition_date": "2020/03/16", "instances": [ "SeobilityBot (SEO Tool; https://www.seobility.net/sites/bot.html)" ], "url": "https://www.seobility.net/sites/bot.html", "description": "Seobility web crawler for SEO analysis", "tags": [ "seo" ] }, { "pattern": "Cincraw", "addition_date": "2020/03/16", "instances": [ "Mozilla/5.0 (compatible; Cincraw/1.0; +http://cincrawdata.net/bot/)" ], "url": "http://cincrawdata.net/bot/", "description": "Cincraw web crawler for data collection", "tags": [ "seo" ] }, { "pattern": "Dragonbot", "addition_date": "2020/03/16", "instances": [ "Mozilla/5.0 (Windows NT 6.1; rv:34.0) Gecko/20100101 Firefox/34.0; Dragonbot; http://www.dragonmetrics.com" ], "url": "http://www.dragonmetrics.com", "description": "DragonMetrics web crawler for analysis", "tags": [ "seo" ] }, { "pattern": "VoluumDSP-content-bot", "addition_date": "2020/03/16", "instances": [ "Mozilla/5.0 (compatible; VoluumDSP-content-bot/2.0; +dsp-dev@codewise.com)" ], "url": "https://codewise.com", "description": "Voluum DSP web crawler for advertising", "tags": [ "advertising" ] }, { "pattern": "FreshRSS", "addition_date": "2020/03/16", "instances": [ "FreshRSS/1.11.2 (Linux; https://freshrss.org) like Googlebot" ], "url": "https://freshrss.org", "description": "FreshRSS feed reader web crawler bot", "tags": [ "feed-reader" ] }, { "pattern": "BitBot", "addition_date": "2020/03/16", "instances": [ "Mozilla/5.0 (compatible; BitBot/v1.19.0; +https://bitbot.dev)" ], "url": "https://bitbot.dev", "description": "BitBot web crawler for content discovery", "tags": [ "seo" ] }, { "pattern": "^PHP-Curl-Class", "addition_date": "2020/12/10", "instances": [ "PHP-Curl-Class/4.13.0 (+https://github.com/php-curl-class/php-curl-class) PHP/7.2.24 curl/7.61.1", "PHP-Curl-Class/4.13.0 (+https://github.com/php-curl-class/php-curl-class) PHP/7.3.19 curl/7.66.0", "PHP-Curl-Class/4.13.0 (+https://github.com/php-curl-class/php-curl-class) PHP/7.3.23 curl/7.66.0", "PHP-Curl-Class/4.13.0 (+https://github.com/php-curl-class/php-curl-class) PHP/7.4.7 curl/7.69.1", "PHP-Curl-Class/4.13.0 (+https://github.com/php-curl-class/php-curl-class) PHP/7.4.9 curl/7.69.1", "PHP-Curl-Class/4.13.0 (+https://github.com/php-curl-class/php-curl-class) PHP/7.4.10 curl/7.69.1", "PHP-Curl-Class/4.13.0 (+https://github.com/php-curl-class/php-curl-class) PHP/7.4.11 curl/7.69.1" ], "url": "https://github.com/php-curl-class/php-curl-class", "description": "PHP Curl Class HTTP client library", "tags": [ "http-library" ] }, { "pattern": "Google-Certificates-Bridge", "addition_date": "2020/12/23", "instances": [ "Google-Certificates-Bridge" ], "description": "Google certificates bridge web crawler", "tags": [ "search-engine" ] }, { "pattern": "centurybot", "addition_date": "2022/04/26", "instances": [ "Mozilla/5.0 (compatible; Go-http-client/1.1; +centurybot9@gmail.com)" ], "description": "Century web crawler for content discovery", "tags": [ "seo" ] }, { "pattern": "Viber", "addition_date": "2021/04/27", "instances": [ "Viber" ], "url": "https://www.viber.com/", "description": "Viber messaging web crawler bot", "tags": [ "social-preview" ] }, { "pattern": "e\\.ventures Investment Crawler", "addition_date": "2021/06/05", "url": "https://www.eventures.vc/", "instances": [ "e.ventures Investment Crawler (eventures.vc)" ], "description": "E.ventures investment web crawler bot", "tags": [ "seo" ] }, { "pattern": "evc-batch", "addition_date": "2021/06/07", "url": "https://www.eventures.vc/", "instances": [ "Mozilla/5.0 (compatible; evc-batch/2.0)" ], "description": "E.ventures batch web crawler bot", "tags": [ "seo" ] }, { "pattern": "PetalBot", "addition_date": "2021/06/07", "instances": [ "Mozilla/5.0 (compatible;PetalBot;+https://webmaster.petalsearch.com/site/petalbot)", "Mozilla/5.0 (Linux; Android 7.0;) AppleWebKit/537.36 (KHTML, like Gecko) Mobile Safari/537.36 (compatible; PetalBot;+https://webmaster.petalsearch.com/site/petalbot)" ], "url": "https://webmaster.petalsearch.com/site/petalbot", "description": "Petal search engine web crawler bot", "tags": [ "search-engine" ] }, { "pattern": "virustotal", "addition_date": "2021/09/22", "instances": [ "Mozilla/5.0 (Windows; U; MSIE 9.0; Windows NT 9.0; en-US) AppEngine-Google; (+http://code.google.com/appengine; appid: s~virustotalcloud)", "AppEngine-Google; (+http://code.google.com/appengine; appid: s~virustotalcloud)" ], "url": "https://www.virustotal.com/gui/home/url", "description": "VirusTotal web crawler for security", "tags": [ "scanner" ] }, { "pattern": "(^| )PTST\\/", "addition_date": "2021/12/05", "instances": [ "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/96.0.4664.45 Safari/537.36 PTST/211202.211915", "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:94.0) Gecko/20100101 Firefox/94.0 PTST/211202.211915" ], "url": "https://www.webpagetest.org", "description": "WebPageTest web crawler for testing", "tags": [ "monitoring" ] }, { "pattern": "minicrawler", "addition_date": "2022/01/12", "instances": [ "Testomatobot/1.0 (Linux x86_64; +https://www.testomato.com/testomatobot) minicrawler/5.2.2" ], "url": "https://www.testomato.com/bot", "description": "Testomato web crawler for testing", "tags": [ "monitoring" ] }, { "pattern": "Cookiebot", "addition_date": "2022/01/23", "url": "https://www.cookiebot.com/", "instances": [ "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko; compatible; Cookiebot/1.0; +http://cookiebot.com/) Chrome/97.0.4692.71 Safari/537.36" ], "description": "Cookiebot web crawler for cookie scanning", "tags": [ "scanner" ] }, { "pattern": "trovitBot", "addition_date": "2022/06/08", "url": "http://www.trovit.com/bot.html", "instances": [ "Mozilla/5.0 (compatible; trovitBot 1.0; +http://www.trovit.com/bot.html)" ], "description": "Trovit web crawler for content discovery", "tags": [ "search-engine" ] }, { "pattern": "seostar\\.co", "addition_date": "2022/08/04", "url": "https://seostar.co/robot/", "instances": [ "Mozilla/5.0 (compatible; Adsbot/3.1; +https://seostar.co/robot/)" ], "description": "SEOstar web crawler for SEO analysis", "tags": [ "seo" ] }, { "pattern": "IonCrawl", "addition_date": "2022/08/04", "url": "https://www.ionos.de/terms-gtc/faq-crawler-en", "instances": [ "IonCrawl (https://www.ionos.de/terms-gtc/faq-crawler-en/)" ], "description": "IONOS IonCrawl web crawler bot", "tags": [ "seo" ] }, { "pattern": "Uptime-Kuma", "addition_date": "2022/10/17", "url": "https://uptime.kuma.pet/", "instances": [ "Uptime-Kuma/1.23.16", "Uptime-Kuma/1.23.15", "Uptime-Kuma/1.23.14", "Uptime-Kuma/1.23.13", "Uptime-Kuma/1.23.12", "Uptime-Kuma/1.23.11", "Uptime-Kuma/1.23.10", "Uptime-Kuma/1.18.0" ], "description": "Uptime Kuma web crawler for monitoring", "tags": [ "monitoring" ] }, { "pattern": "Seekport", "addition_date": "2022/10/17", "url": "https://bot.seekport.com", "instances": [ "Mozilla/5.0 (compatible; SeekportBot; +https://bot.seekport.com)", "Mozilla/5.0 (compatible; Seekport Crawler; http://seekport.com/)" ], "description": "Seekport web crawler for content discovery", "tags": [ "search-engine" ] }, { "pattern": "FreshpingBot", "addition_date": "2022/10/17", "url": "https://www.freshworks.com/website-monitoring/", "instances": [ "FreshpingBot/1.0 (+https://freshping.io/)" ], "description": "Freshping web crawler for monitoring", "tags": [ "monitoring" ] }, { "pattern": "Feedbin", "addition_date": "2022/11/05", "url": "https://feedbin.com/", "instances": [ "Feedbin feed-id:2005098 - 2 subscribers" ], "description": "Feedbin feed reader web crawler bot", "tags": [ "feed-reader" ] }, { "pattern": "CriteoBot", "addition_date": "2022/11/13", "url": "https://www.criteo.com/", "instances": [ "CriteoBot/0.1 (+https://www.criteo.com/criteo-crawler/)" ], "description": "Criteo web crawler for advertising", "tags": [ "advertising" ] }, { "pattern": "Snap URL Preview Service", "addition_date": "2022/11/13", "url": "https://snap.com/", "instances": [ "Snap URL Preview Service; bot; snapchat; https://developers.snap.com/robots" ], "description": "Snapchat URL preview web crawler", "tags": [ "social-preview" ] }, { "pattern": "Better Uptime Bot", "addition_date": "2022/11/13", "url": "https://betteruptime.com/", "instances": [ "Better Uptime Bot Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.169 Safari/537.36" ], "description": "Better Uptime web crawler for monitoring", "tags": [ "monitoring" ] }, { "pattern": "RuxitSynthetic", "addition_date": "2023/02/16", "url": "https://www.dynatrace.com/support/help/platform-modules/digital-experience/synthetic-monitoring/browser-monitors/configure-browser-monitors#expand--default-user-agent", "instances": [ "RuxitSynthetic/1.0" ], "description": "Dynatrace Ruxit synthetic web crawler", "tags": [ "monitoring" ] }, { "pattern": "Google-Read-Aloud", "addition_date": "2023/02/16", "url": "https://developers.google.com/search/docs/crawling-indexing/overview-google-crawlers", "instances": [ "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2272.118 Safari/537.36 (compatible; Google-Read-Aloud; +https://developers.google.com/search/docs/crawling-indexing/overview-google-crawlers)", "Mozilla/5.0 (Linux; Android 7.0; SM-G930V Build/NRD90M) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/59.0.3071.125 Mobile Safari/537.36 (compatible; Google-Read-Aloud; +https://developers.google.com/search/docs/crawling-indexing/overview-google-crawlers)" ], "description": "Google read aloud web crawler bot", "tags": [ "search-engine" ] }, { "pattern": "Valve\\/Steam", "addition_date": "2023/05/24", "instances": [ "Valve/Steam HTTP Client 1.0 (SteamChatURLLookup)" ], "description": "Steam web crawler for link previews", "tags": [ "social-preview" ] }, { "pattern": "OdklBot\\/", "addition_date": "2023/05/24", "instances": [ "OdklBot/1.0 (share@odnoklassniki.ru)", "Mozilla/5.0 (compatible; OdklBot/1.0 like Linux; klass@odnoklassniki.ru)" ], "url": "https://odnoklassniki.ru/", "description": "Odnoklassniki web crawler for sharing", "tags": [ "social-preview" ] }, { "pattern": "GPTBot", "addition_date": "2023/08/09", "instances": [ "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; GPTBot/1.0; +https://openai.com/gptbot)" ], "url": "https://platform.openai.com/docs/gptbot", "description": "OpenAI GPT web crawler bot", "tags": [ "ai-crawler" ] }, { "pattern": "ChatGPT-User", "addition_date": "2024/04/19", "instances": [ "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko); compatible; ChatGPT-User/1.0; +https://openai.com/bot" ], "url": "https://openai.com/bot", "description": "ChatGPT user web crawler bot", "tags": [ "ai-crawler" ] }, { "pattern": "OAI-SearchBot", "addition_date": "2024/09/24", "instances": [ "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko); compatible; OAI-SearchBot/1.0; +https://openai.com/searchbot" ], "url": "https://platform.openai.com/docs/bots", "description": "OpenAI search web crawler bot", "tags": [ "ai-crawler", "search-engine" ] }, { "pattern": "YandexRenderResourcesBot\\/", "addition_date": "2023/08/16", "instances": [ "Mozilla/5.0 (compatible; YandexRenderResourcesBot/1.0; +http://yandex.com/bots) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/108.0.0.0" ], "url": "http://yandex.com/bots", "description": "Yandex render resources web crawler", "tags": [ "search-engine" ] }, { "pattern": "LightspeedSystemsCrawler", "addition_date": "2023/08/16", "instances": [ "LightspeedSystemsCrawler Mozilla/5.0 (Windows; U; MSIE 9.0; Windows NT 9.0; en-US" ], "description": "Lightspeed Systems web crawler bot", "tags": [ "scanner" ] }, { "pattern": "ev-crawler\\/", "addition_date": "2023/08/16", "instances": [ "Mozilla/5.0 (compatible; ev-crawler/1.0; +https://headline.com/legal/crawler)" ], "url": "https://headline.com/legal/crawler", "description": "Headline web crawler for content discovery", "tags": [ "seo" ] }, { "pattern": "BitSightBot\\/", "addition_date": "2023/08/16", "instances": [ "Mozilla/5.0 (compatible; BitSightBot/1.0)" ], "url": "https://www.bitsight.com", "description": "BitSight web crawler for security", "tags": [ "scanner" ] }, { "pattern": "woorankreview\\/", "addition_date": "2023/08/16", "instances": [ "Mozilla/5.0 (iPhone; CPU iPhone OS 11_0 like Mac OS X) AppleWebKit/604.1.38 (KHTML, like Gecko) Version/11.0 Mobile/15A372 Safari/604.1 (compatible; woorankreview/2.0; +https://www.woorank.com/)", "Mozilla/5.0 (compatible; woorankreview/2.0; +https://www.woorank.com/)" ], "url": "https://www.woorank.com/", "description": "WooRank web crawler for SEO analysis", "tags": [ "seo" ] }, { "pattern": "Google-Safety", "addition_date": "2023/08/17", "instances": [ "Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.5735.179 Mobile Safari/537.36 (compatible; Google-Safety; +http://www.google.com/bot.html)", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.5735.179 Safari/537.36 (compatible; Google-Safety; +http://www.google.com/bot.html)", "Google-Safety" ], "url": "https://developers.google.com/search/docs/crawling-indexing/overview-google-crawlers", "description": "Google safety web crawler bot", "tags": [ "search-engine" ] }, { "pattern": "AwarioBot", "addition_date": "2023/08/23", "instances": [ "Mozilla/5.0 (compatible; AwarioBot/1.0; +https://awario.com/bots.html)" ], "url": "https://awario.com/bots.html", "description": "Awario web crawler for monitoring", "tags": [ "seo" ] }, { "pattern": "DataForSeoBot", "addition_date": "2023/08/23", "instances": [ "Mozilla/5.0 (compatible; DataForSeoBot/1.0; +https://dataforseo.com/dataforseo-bot)" ], "url": "https://dataforseo.com/dataforseo-bot", "description": "DataForSEO web crawler for analysis", "tags": [ "seo" ] }, { "pattern": "Linespider", "addition_date": "2023/08/24", "instances": [ "Mozilla/5.0 (compatible; Linespider/1.1; +https://lin.ee/4dwXkTH)", "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Linespider/1.1; +https://lin.ee/4dwXkTH) Chrome/W.X.Y.Z Safari/537.36" ], "url": "https://help2.line.me/linesearchbot/web/?contentId=50006055&lang=en", "description": "LINE Linespider web crawler bot", "tags": [ "social-preview" ] }, { "pattern": "WellKnownBot", "addition_date": "2023/08/29", "instances": [ "Mozilla/5.0 (compatible; WellKnownBot/0.1; +https://well-known.dev/about/#bot)" ], "url": "https://well-known.dev/about/#bot)", "description": "WellKnown web crawler for discovery", "tags": [ "seo" ] }, { "pattern": "A Patent Crawler", "addition_date": "2023/08/29", "instances": [ "E. Orliac, G. Fourestey/2.3 (A Patent Crawler; http://scitas.epfl.ch/; etienne.orliac@epfl.ch, gilles.fourestey@epfl.ch)" ], "url": "http://scitas.epfl.ch/", "description": "EPFL patent web crawler bot", "tags": [ "academic" ] }, { "pattern": "StractBot", "addition_date": "2023/09/06", "instances": [ "Mozilla/5.0 (compatible; StractBot/0.1; open source search engine; +https://trystract.com/webmasters)" ], "url": "https://trystract.com/webmasters", "description": "Stract search engine web crawler bot", "tags": [ "search-engine" ] }, { "pattern": "search\\.marginalia\\.nu", "addition_date": "2023/09/08", "instances": [ "search.marginalia.nu" ], "url": "https://search.marginalia.nu", "description": "Marginalia search web crawler bot", "tags": [ "search-engine" ] }, { "pattern": "YouBot", "addition_date": "2023/09/08", "instances": [ "YouBot (+http://www.you.com)" ], "url": "https://you.com/", "description": "You.com search engine web crawler bot", "tags": [ "search-engine" ] }, { "pattern": "Nicecrawler", "addition_date": "2023/09/08", "instances": [ "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Nicecrawler/1.1; +http://www.nicecrawler.com/) Chrome/90.0.4430.97 Safari/537.36" ], "url": "http://www.nicecrawler.com/", "description": "Nicecrawler web crawler for discovery", "tags": [ "search-engine" ] }, { "pattern": "Neevabot", "addition_date": "2023/09/08", "instances": [ "Mozilla/5.0 (compatible; Neevabot/1.0; +https://neeva.com/neevabot)" ], "url": "https://neeva.com/neevabot", "description": "Neeva search engine web crawler bot", "tags": [ "search-engine" ] }, { "pattern": "BrightEdge Crawler", "addition_date": "2023/09/08", "instances": [ "BrightEdge Crawler/1.0 (crawler@brightedge.com)" ], "url": "https://www.brightedge.com/", "description": "BrightEdge web crawler for SEO", "tags": [ "seo" ] }, { "pattern": "SiteCheckerBotCrawler", "addition_date": "2023/09/08", "instances": [ "SiteCheckerBotCrawler/1.0 (+http://sitechecker.pro)" ], "url": "http://sitechecker.pro", "description": "SiteChecker web crawler for analysis", "tags": [ "seo" ] }, { "pattern": "TombaPublicWebCrawler", "addition_date": "2023/09/08", "instances": [ "Mozilla/5.0 (compatible; TombaPublicWebCrawler/1.0; +https://tombascraper.com)" ], "url": "https://tombascraper.com", "description": "Tomba web crawler for email discovery", "tags": [ "seo" ] }, { "pattern": "CrawlyProjectCrawler", "addition_date": "2023/09/08", "instances": [ "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36 (compatible; CrawlyProjectCrawler/0.1.3; crawlyproject@digitaldragon.dev +https://crawlyproject.digitaldragon.dev/)" ], "url": "https://crawlyproject.digitaldragon.dev/", "description": "Crawly Project web crawler framework", "tags": [ "seo" ] }, { "pattern": "KomodiaBot", "addition_date": "2023/09/08", "instances": [ "Mozilla/5.0 (Windows NT 6.1; Win64; x64; +http://www.komodia.com/newwiki/index.php/URL_server_crawler) KomodiaBot/1.0" ], "url": "http://www.komodia.com/newwiki/index.php/URL_server_crawler", "description": "Komodia web crawler for classification", "tags": [ "seo" ] }, { "pattern": "KStandBot", "addition_date": "2023/09/08", "instances": [ "Mozilla/5.0 (Windows NT 6.1; Win64; x64; +http://url-classification.io/wiki/index.php?title=URL_server_crawler) KStandBot/1.0" ], "url": "http://url-classification.io", "description": "KStand web crawler for classification", "tags": [ "seo" ] }, { "pattern": "CISPA Webcrawler", "addition_date": "2023/09/08", "instances": [ "CISPA Webcrawler (https://vuln-notify-checker.cispa.saarland)" ], "url": "https://vuln-notify-checker.cispa.saarland", "description": "CISPA web crawler for vulnerability", "tags": [ "academic" ] }, { "pattern": "MTRobot", "addition_date": "2023/09/08", "instances": [ "MTRobot/0.2 (Metrics Tools Analytics Crawler; https://metrics-tools.de/robot.html; crawler@metrics-tools.de)" ], "url": "https://metrics-tools.de/robot.html", "description": "Metrics Tools web crawler for analysis", "tags": [ "seo" ] }, { "pattern": "hyscore\\.io", "addition_date": "2023/09/08", "instances": [ "Mozilla/5.0 (iPhone; CPU iPhone OS 8_3 like Mac OS X) AppleWebKit/600.1.4 (KHTML, like Gecko) Version/8.0 Mobile/12F70 Safari/600.1. 4 (compatible; HyScore/1.0; +https://hyscore.io/crawler/)" ], "url": "https://hyscore.io/crawler/", "description": "HyScore web crawler for analysis", "tags": [ "seo" ] }, { "pattern": "AlexandriaOrgBot", "addition_date": "2023/09/08", "instances": [ "Mozilla/5.0 (Linux) (compatible; AlexandriaOrgBot/1.0; +https://www.alexandria.org/bot.html)" ], "url": "https://www.alexandria.org/bot.html", "description": "Alexandria web crawler for discovery", "tags": [ "search-engine" ] }, { "pattern": "2ip bot", "addition_date": "2023/09/08", "instances": [ "2ip bot/1.1 (+http://2ip.io)" ], "url": "http://2ip.io", "description": "2IP web crawler for analysis", "tags": [ "seo" ] }, { "pattern": "Yellowbrandprotectionbot", "addition_date": "2023/09/08", "instances": [ "Mozilla/5.0 (compatible; Yellowbrandprotectionbot/1.0; +https://www.yellowbp.com/bot.html)" ], "url": "https://www.yellowbp.com/bot.html", "description": "Yellow brand protection web crawler", "tags": [ "seo" ] }, { "pattern": "SEOlizer", "addition_date": "2023/09/08", "instances": [ "SEOlizer/1.1 (Windows; U; Windows NT 5.1; en-US; rv:1.8.1.13) Gecko/20080311 Firefox/2.0.0.13 (+https://www.seolizer.de/bot.html)" ], "url": "https://www.seolizer.de/bot.html", "description": "SEOlizer web crawler for SEO analysis", "tags": [ "seo" ] }, { "pattern": "vuhuvBot", "addition_date": "2023/09/08", "instances": [ "Mozilla/5.0 (compatible; vuhuvBot/1.0; +http://vuhuv.com/bot.html)" ], "url": "http://vuhuv.com/bot.html", "description": "Vuhuv web crawler for content discovery", "tags": [ "seo" ] }, { "pattern": "INETDEX-BOT", "addition_date": "2023/09/08", "instances": [ "INETDEX-BOT/1.5 (Mozilla/5.0; https://inetdex.com/bot.html)" ], "url": "https://inetdex.com/bot.html", "description": "INETDEX web crawler for indexing", "tags": [ "search-engine" ] }, { "pattern": "Synapse", "addition_date": "2023/09/08", "instances": [ "Synapse (bot; +https://github.com/matrix-org/synapse)" ], "url": "https://github.com/matrix-org/synapse", "description": "Matrix Synapse web crawler bot", "tags": [ "social-preview" ] }, { "pattern": "t3versionsBot", "addition_date": "2023/09/08", "instances": [ "Mozilla/5.0 (compatible; t3versionsBot/1.0; +https://www.t3versions.com/bot)" ], "url": "https://www.t3versions.com/bot", "description": "T3versions web crawler for discovery", "tags": [ "seo" ] }, { "pattern": "deepnoc", "addition_date": "2023/09/08", "instances": [ "deepnoc - https://deepnoc.com/bot" ], "url": "https://deepnoc.com/bot", "description": "DeepNOC web crawler for monitoring", "tags": [ "monitoring" ] }, { "pattern": "Cocolyzebot", "addition_date": "2023/09/08", "instances": [ "Mozilla/5.0 (compatible; Cocolyzebot/1.0; https://cocolyze.com/bot)" ], "url": "https://cocolyze.com/bot", "description": "Cocolyze web crawler for analysis", "tags": [ "seo" ] }, { "pattern": "hypestat", "addition_date": "2023/09/08", "instances": [ "Mozilla/5.0 (compatible; hypestat/1.0; +https://hypestat.com/bot)" ], "url": "https://hypestat.com/bot", "description": "Hypestat web crawler for analysis", "tags": [ "seo" ] }, { "pattern": "ReverseEngineeringBot", "addition_date": "2023/09/08", "instances": [ "Mozilla/5.0 (compatible; ReverseEngineeringBot/0.1; +https://torus.company/bot.html)" ], "url": "https://torus.company/bot.html", "description": "Torus reverse engineering web crawler", "tags": [ "scanner" ] }, { "pattern": "sempi\\.tech", "addition_date": "2023/09/08", "instances": [ "Mozilla/5.0 (compatible; Semanticbot/1.0; +http://sempi.tech/bot.html)" ], "url": "http://sempi.tech/bot.html", "description": "Sempi web crawler for analysis", "tags": [ "seo" ] }, { "pattern": "Iframely", "addition_date": "2023/09/08", "instances": [ "Iframely/1.3.1 (+https://iframely.com/docs/about) Atlassian" ], "url": "https://iframely.com/docs/about", "description": "Iframely web crawler for embeds", "tags": [ "social-preview" ] }, { "pattern": "MetaInspector", "addition_date": "2023/09/08", "instances": [ "MetaInspector/5.6.0 (+https://github.com/jaimeiniesta/metainspector)" ], "url": "https://github.com/jaimeiniesta/metainspector", "description": "MetaInspector web crawler for metadata", "tags": [ "seo" ] }, { "pattern": "node-fetch", "addition_date": "2023/09/08", "instances": [ "node-fetch/1.0 (+https://github.com/bitinn/node-fetch)" ], "url": "https://github.com/bitinn/node-fetch", "description": "Node-fetch HTTP client library", "tags": [ "http-library" ] }, { "pattern": "l9explore", "addition_date": "2023/09/08", "instances": [ "l9explore/1.2.2", "lkxscan/v0.1.0 (+https://leakix.net) l9explore/v1.0.0 (+https://github.com/LeakIX/l9explore)" ], "url": "https://github.com/LeakIX/l9explore", "description": "L9explore web crawler for discovery", "tags": [ "scanner" ] }, { "pattern": "python-opengraph", "addition_date": "2023/09/08", "instances": [ "python-opengraph-jaywink/0.2.0 (+https://github.com/jaywink/python-opengraph)" ], "url": "https://github.com/jaywink/python-opengraph", "description": "Python OpenGraph web crawler bot", "tags": [ "http-library" ] }, { "pattern": "OpenGraphCheck", "addition_date": "2023/09/08", "instances": [ "OpenGraphCheck/2.1 (+https://opengraphcheck.com)" ], "url": "https://opengraphcheck.com", "description": "OpenGraphCheck web crawler for metadata", "tags": [ "seo" ] }, { "pattern": "developers\\.google\\.com\\/\\+\\/web\\/snippet", "addition_date": "2023/09/08", "instances": [ "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.2924.87 Safari/537.36 Google-PageRenderer Google (+https://developers.google.com/+/web/snippet/)", "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.2924.87 Safari/537.36 Google (+https://developers.google.com/+/web/snippet/" ], "url": "https://developers.google.com/+/web/snippet", "description": "Google snippet web crawler bot", "tags": [ "search-engine" ] }, { "pattern": "SenutoBot", "addition_date": "2023/09/08", "instances": [ "SenutoBot/1.0 (compatible; SenutoBot/1.0; +https://www.senuto.com/)" ], "url": "https://www.senuto.com", "description": "Senuto web crawler for SEO analysis", "tags": [ "seo" ] }, { "pattern": "MaCoCu", "addition_date": "2023/09/08", "instances": [ "Mozilla/5.0 (compatible; MaCoCu; +https://www.clarin.si/info/macocu-massive-collection-and-curation-of-monolingual-and-bilingual-data/)" ], "url": "https://www.clarin.si/info/macocu-massive-collection-and-curation-of-monolingual-and-bilingual-data", "description": "MaCoCu web crawler for language research", "tags": [ "academic" ] }, { "pattern": "NewsBlur", "addition_date": "2023/09/08", "instances": [ "NewsBlur Feed Fetcher - 1 subscriber - http://www.newsblur.com/site/0000000/webpage (Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/14.0.1 Safari/605.1.15)" ], "url": "http://www.newsblur.com", "description": "NewsBlur feed reader web crawler bot", "tags": [ "feed-reader" ] }, { "pattern": "inoreader", "addition_date": "2023/09/08", "instances": [ "Mozilla/5.0 (compatible; inoreader.com; 1 subscribers)" ], "url": "http://inoreader.com", "description": "Inoreader feed reader web crawler bot", "tags": [ "feed-reader" ] }, { "pattern": "NetSystemsResearch", "addition_date": "2023/09/08", "instances": [ "NetSystemsResearch studies the availability of various services across the internet. Our website is netsystemsresearch.com" ], "url": "http://netsystemsresearch.com", "description": "NetSystemsResearch web crawler bot", "tags": [ "scanner" ] }, { "pattern": "PageThing", "addition_date": "2023/09/08", "instances": [ "PageThing http://pagething.com curl www" ], "url": "http://pagething.com", "description": "PageThing web crawler for content discovery", "tags": [ "seo" ] }, { "pattern": "WordPress\\/", "addition_date": "2023/10/24", "instances": [ "WordPress/X.X.X; https://example.com" ], "url": "https://wordpress.org", "description": "WordPress web crawler for site discovery", "tags": [ "seo" ] }, { "pattern": "PhxBot", "addition_date": "2024/01/06", "instances": [ "PhxBot/0.1 (phxbot@protonmail.com)" ], "description": "Phoenix web crawler for content discovery", "tags": [ "search-engine" ] }, { "pattern": "ImagesiftBot", "addition_date": "2024/01/06", "instances": [ "Mozilla/5.0 (compatible; ImagesiftBot; +imagesift.com)" ], "url": "https://imagesift.com/about", "description": "Imagesift bot for image search and indexing", "tags": [ "search-engine" ] }, { "pattern": "Expanse", "addition_date": "2024/02/01", "instances": [ "Expanse, a Palo Alto Networks company, searches across the global IPv4 space multiple times per day to identify customers' presences on the Internet. If you would like to be excluded from our scans, please send IP addresses/domains to: scaninfo@paloaltonetworks.com" ], "url": "https://www.paloaltonetworks.com/cortex/cortex-xpanse", "description": "Palo Alto Networks Expanse bot for internet asset discovery", "tags": [ "scanner" ] }, { "pattern": "InternetMeasurement", "addition_date": "2024/02/01", "instances": [ "Mozilla/5.0 (compatible; InternetMeasurement/1.0; +https://internet-measurement.com/)" ], "url": "https://internet-measurement.com", "description": "Internet Measurement bot for network research and analysis", "tags": [ "academic" ] }, { "pattern": "^BW\\/", "addition_date": "2024/02/08", "instances": [ "BW/1.1; bit.ly/3eZNDnO", "BW/1.1; rb.gy/oupwis" ], "url": "https://builtwith.com/biup", "description": "BuiltWith web crawler for technology detection", "tags": [ "seo" ] }, { "pattern": "GeedoBot", "addition_date": "2024/02/11", "instances": [ "Mozilla/5.0 (compatible; GeedoBot; +http://www.geedo.com/bot.html)" ], "url": "http://www.geedo.com", "description": "Geedo web crawler for content discovery", "tags": [ "search-engine" ] }, { "pattern": "Audisto Crawler", "addition_date": "2024/03/14", "instances": [ "Audisto Crawler (mobile; +https://audisto.com/bot)", "Audisto Crawler (desktop; +https://audisto.com/bot)", "Audisto Crawler (mobile; essential; +https://audisto.com/bot)", "Audisto Crawler (desktop; essential; +https://audisto.com/bot)" ], "url": "https://audisto.com/help/crawler/bot/", "description": "Audisto SEO web crawler for analysis", "tags": [ "seo" ] }, { "pattern": "PerplexityBot\\/", "addition_date": "2024/03/14", "instances": [ "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; PerplexityBot/1.0; +https://perplexity.ai/perplexitybot)" ], "url": "https://docs.perplexity.ai/docs/perplexitybot", "description": "Perplexity AI web crawler for search", "tags": [ "ai-crawler", "search-engine" ] }, { "pattern": "[cC]laude[bB]ot", "addition_date": "2024/04/19", "instances": [ "claudebot", "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; ClaudeBot/1.0; +claudebot@anthropic.com)" ], "url": "https://www.anthropic.com/", "description": "Anthropic Claude web crawler bot", "tags": [ "ai-crawler" ] }, { "pattern": "Monsidobot", "addition_date": "2024/05/14", "instances": [ "Mozilla/5.0 (compatible; Monsidobot/2.2; +http://monsido.com/bot.html; info@monsido.com)" ], "url": "http://monsido.com/bot.html", "description": "Monsido web crawler for website monitoring", "tags": [ "monitoring" ] }, { "pattern": "GroupMeBot", "addition_date": "2024/05/19", "instances": [ "GroupMeBot/1.0" ], "url": "https://groupme.com/", "description": "GroupMe web crawler for messaging", "tags": [ "social-preview" ] }, { "pattern": "Vercelbot", "addition_date": "2024/08/30", "instances": [ "Vercelbot (+https://vercel.com)" ], "url": "https://github.com/vercel/vercel/discussions/5095#discussioncomment-58705", "description": "Vercel web crawler for deployment", "tags": [ "seo" ] }, { "pattern": "vercel-screenshot", "addition_date": "2024/08/30", "instances": [], "description": "Vercel screenshot web crawler bot", "tags": [ "seo" ] }, { "pattern": "facebookcatalog\\/", "addition_date": "2024/10/03", "instances": [ "facebookcatalog/1.0" ], "url": "https://developers.facebook.com/docs/sharing/webmasters/web-crawlers", "description": "Facebook catalog web crawler bot", "tags": [ "advertising" ] }, { "pattern": "meta-externalads\\/", "addition_date": "2025/08/08", "instances": [ "meta-externalads/1.1 (+https://developers.facebook.com/docs/sharing/webmasters/crawler)", "meta-externalads/1.1" ], "url": "https://developers.facebook.com/docs/sharing/webmasters/web-crawlers", "description": "Meta external ads web crawler bot", "tags": [ "advertising" ] }, { "pattern": "meta-externalagent\\/", "addition_date": "2024/10/03", "instances": [ "meta-externalagent/1.1 (+https://developers.facebook.com/docs/sharing/webmasters/crawler)", "meta-externalagent/1.1" ], "url": "https://developers.facebook.com/docs/sharing/webmasters/web-crawlers", "description": "Meta external agent web crawler bot", "tags": [ "ai-crawler", "social-preview" ] }, { "pattern": "meta-externalfetcher\\/", "addition_date": "2024/10/03", "instances": [ "meta-externalfetcher/1.1 (+https://developers.facebook.com/docs/sharing/webmasters/crawler)", "meta-externalfetcher/1.1" ], "url": "https://developers.facebook.com/docs/sharing/webmasters/web-crawlers", "description": "Meta external fetcher web crawler bot", "tags": [ "social-preview" ] }, { "pattern": "AcademicBotRTU", "addition_date": "2024/10/17", "instances": [ "AcademicBotRTU (https://academicbot.rtu.lv; mailto:caps@rtu.lv)" ], "url": "https://academicbot.rtu.lv", "description": "Academic Bot RTU web crawler bot", "tags": [ "academic" ] }, { "pattern": "KeybaseBot", "addition_date": "2024/10/21", "url": "https://book.keybase.io/docs/chat/link-previews", "instances": [ "Mozilla/5.0 (compatible; KeybaseBot; +https://keybase.io)" ], "description": "Keybase web crawler for link previews", "tags": [ "social-preview" ] }, { "pattern": "Lemmy", "addition_date": "2025/02/11", "instances": [ "Lemmy/0.19.8; +https://leminal.space" ], "url": "https://leminal.space", "description": "Lemmy social network web crawler bot", "tags": [ "social-preview" ] }, { "pattern": "CookieHubScan", "addition_date": "2024/11/29", "url": "https://www.cookiehub.com/", "instances": [ "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36 CookieHubScan/3.0" ], "description": "CookieHub web crawler for cookie scanning", "tags": [ "scanner" ] }, { "pattern": "Hydrozen\\.io", "addition_date": "2025/02/02", "instances": [ "Hydrozen.io/1.0" ], "url": "https://docs.hydrozen.io/overview/misc/user-agent-and-ip-list", "description": "Hydrozen web crawler for monitoring", "tags": [ "monitoring" ] }, { "pattern": "HTTP Banner Detection", "addition_date": "2025/02/10", "instances": [ "HTTP Banner Detection (https://security.ipip.net)" ], "url": "https://security.ipip.net", "description": "IPIP HTTP banner detection web crawler", "tags": [ "scanner" ] }, { "pattern": "SummalyBot", "addition_date": "2025/02/10", "instances": [ "SummalyBot/5.1.0" ], "url": "https://github.com/misskey-dev/summaly", "description": "Summaly web crawler for content summarization", "tags": [ "social-preview" ] }, { "pattern": "MicrosoftPreview\\/", "addition_date": "2025/02/11", "url": "https://www.bing.com/webmasters/help/which-crawlers-does-bing-use-8c184ec0", "instances": [ "MicrosoftPreview/2.0; +https://aka.ms/MicrosoftPreview" ], "description": "Microsoft preview web crawler bot", "tags": [ "social-preview" ] }, { "pattern": "GeedoProductSearch", "addition_date": "2025/03/15", "url": "http://www.geedo.com/product-search.html", "instances": [ "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; GeedoProductSearch; +http://www.geedo.com/product-search.html) Chrome/79.0.3945.88 Safari/537.36" ], "description": "Geedo product search web crawler bot", "tags": [ "search-engine" ] }, { "pattern": "TikTokSpider", "addition_date": "2025/03/16", "instances": [ "Mozilla/5.0 (Linux; Android 5.0) AppleWebKit/537.36 (KHTML, like Gecko) Mobile Safari/537.36 (compatible; TikTokSpider; ttspider-feedback@tiktok.com)" ], "description": "TikTok web crawler for content discovery", "tags": [ "ai-crawler", "social-preview" ] }, { "pattern": "OnCrawl\\/", "addition_date": "2025/03/27", "url": "http://www.oncrawl.com", "instances": [ "Mozilla/5.0 (compatible; OnCrawl/1.0; +http://www.oncrawl.com)", "Mozilla/5.0 (iPhone; CPU iPhone OS 8_3 like Mac OS X) AppleWebKit/600.1.4 (KHTML, like Gecko) Version/8.0 Mobile/12F70 Safari/600.1.4 (compatible; OnCrawl/1.0; +http://www.oncrawl.com)" ], "description": "OnCrawl SEO web crawler for analysis", "tags": [ "seo" ] }, { "pattern": "sindresorhus\\/got", "addition_date": "2025/04/22", "url": "https://github.com/sindresorhus/got", "instances": [ "got (https://github.com/sindresorhus/got)" ], "description": "Got HTTP client library for requests", "tags": [ "http-library" ] }, { "pattern": "CensysInspect\\/", "addition_date": "2025/04/22", "url": "https://about.censys.io", "instances": [ "Mozilla/5.0 (compatible; CensysInspect/1.1; +https://about.censys.io/)" ], "description": "Censys web crawler for security scanning", "tags": [ "scanner" ] }, { "pattern": "SBIntuitionsBot\\/", "addition_date": "2025/04/23", "url": "https://www.sbintuitions.co.jp/bot/", "instances": [ "Mozilla/5.0 (compatible; SBIntuitionsBot/0.1; +https://www.sbintuitions.co.jp/bot/)" ], "description": "SB Intuitions web crawler bot", "tags": [ "ai-crawler" ] }, { "pattern": "sitebulb", "addition_date": "2025/04/30", "url": "https://sitebulb.com/", "instances": [ "Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.6439.0 Mobile Safari/537.36 +https://sitebulb.com" ], "description": "Sitebulb SEO web crawler for analysis", "tags": [ "seo" ] }, { "pattern": "YextBot\\/", "addition_date": "2025/08/08", "url": "https://hitchhikers.yext.com/modules/kg140-yext-site-crawler/01-create-a-crawler/", "instances": [ "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) HeadlessChrome/87.0.4280.88 YextBot/Java Safari/537.36" ], "description": "Yext web crawler for site crawling", "tags": [ "seo" ] }, { "pattern": "DatadogSynthetics", "addition_date": "2025/08/19", "url": "https://docs.datadoghq.com/synthetics/", "instances": [ "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/138.0.7204.168 Safari/537.36 DatadogSynthetics" ], "description": "Datadog synthetics web crawler bot", "tags": [ "monitoring" ] }, { "pattern": "Google-Ads-Conversions", "addition_date": "2025/09/10", "url": "https://developers.google.com/google-ads/api/docs/conversions/upload-online", "instances": [ "Mozilla/5.0 AppleWebKit/537.36 Chrome/139.0.7258.127 Safari/537.36 Google-Ads-Conversions" ], "description": "Google Ads conversions web crawler", "tags": [ "advertising" ] }, { "pattern": "ObservePoint", "addition_date": "2025/12/23", "url": "https://help.observepoint.com/en/articles/9101465-allow-exclude-observepoint-traffic#h_2a8176c9b9", "instances": [], "description": "ObservePoint web crawler for monitoring", "tags": [ "monitoring" ] }, { "pattern": "Checkly", "addition_date": "2026/02/11", "url": "https://www.checklyhq.com/docs/", "instances": [ "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/134.0.6998.35 Safari/537.36 (Checkly, https://www.checklyhq.com)" ], "description": "Checkly web crawler for monitoring", "tags": [ "monitoring" ] }, { "pattern": "ALittle Client", "addition_date": "2026/04/07", "url": "https://udger.com/resources/ua-list/bot-detail?bot=ALittle+Client", "instances": [ "ALittle Client" ], "description": "ALittle web crawler for content discovery", "tags": [ "http-library" ] }, { "pattern": "AliyunSecBot", "addition_date": "2026/04/07", "url": "https://service.alibaba.com", "instances": [ "AliyunSecBot/Aliyun AliyunSecBot@service.alibaba.com" ], "description": "Alibaba Aliyun security web crawler bot", "tags": [ "scanner" ] }, { "pattern": "Claude-Web", "addition_date": "2026/04/07", "url": "https://anthropic.com", "instances": [ "Claude-Web/1.0 (web crawler; +https://www.anthropic.com/; bots@anthropic.com)" ], "description": "Anthropic Claude web crawler bot", "tags": [ "ai-crawler" ] }, { "pattern": "anthropic-ai", "addition_date": "2026/04/07", "url": "https://anthropic.com", "instances": [ "anthropic-ai" ], "description": "Anthropic AI web crawler bot", "tags": [ "ai-crawler" ] }, { "pattern": "Claude-User", "addition_date": "2026/04/07", "url": "https://useragents.io/uas/mozilla-5-0-applewebkit-537-36-khtml-like-gecko-compatible-claudebot-1-0-supportanthropic-com_954fa13a8e1e46d8267fb56e2d48100e", "instances": [ "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Claude-User/1.0; +Claude-User@anthropic.com)", "Claude-User (claude-code/2.1.86; +https://support.anthropic.com/)" ], "description": "Anthropic Claude user web crawler bot", "tags": [ "ai-crawler" ] }, { "pattern": "Claude-SearchBot", "addition_date": "2026/04/07", "url": "https://useragents.io/uas/mozilla-5-0-applewebkit-537-36-khtml-like-gecko-compatible-claudebot-1-0-supportanthropic-com_954fa13a8e1e46d8267fb56e2d48100e", "instances": [ "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Claude-SearchBot/1.0; +https://www.anthropic.com)" ], "description": "Anthropic Claude search web crawler bot", "tags": [ "ai-crawler", "search-engine" ] }, { "pattern": "Google-Extended", "addition_date": "2026/04/07", "url": "https://developers.google.com/search/docs/crawling-indexing/overview-google-crawlers", "instances": [ "Mozilla/5.0 (compatible; Google-Extended/1.0; +http://www.google.com/bot.html)" ], "description": "Google extended web crawler bot", "tags": [ "ai-crawler" ] }, { "pattern": "cohere-ai", "addition_date": "2026/04/07", "url": "https://cohere.com", "instances": [ "cohere-ai" ], "description": "Cohere AI web crawler bot", "tags": [ "ai-crawler" ] }, { "pattern": "Timpibot", "addition_date": "2026/04/07", "url": "https://timpi.io", "instances": [ "Timpibot/0.9 (+http://www.timpi.io)", "Mozilla/5.0 (compatible; Timpibot/0.8; +http://www.timpi.io)", "Mozilla/5.0 (compatible; Timpibot/0.9; +http://www.timpi.io)" ], "description": "Timpi web crawler for content discovery", "tags": [ "search-engine" ] }, { "pattern": "SERankingBacklinksBot", "addition_date": "2026/04/07", "url": "https://seranking.com/backlinks-crawler", "instances": [ "Mozilla/5.0 (compatible; SERankingBacklinksBot/1.0; +https://seranking.com/backlinks-crawler)" ], "description": "SEranking backlinks web crawler bot", "tags": [ "seo" ] }, { "pattern": "CMSChecker", "addition_date": "2026/04/07", "instances": [ "Mozilla/5.0 (compatible; CMSChecker/1.0; +https://cmschecker.net)" ], "description": "CMS Checker web crawler for detection", "tags": [ "seo" ] }, { "pattern": "Wayback", "addition_date": "2026/04/07", "url": "https://archive.org", "instances": [ "Mozilla/5.0 (compatible; archive.org_bot; Wayback Machine Live Record; +http://archive.org/details/archive.org_bot)" ], "description": "Internet Archive Wayback web crawler bot", "tags": [ "archiver" ] }, { "pattern": "Playwright", "addition_date": "2026/04/07", "url": "https://playwright.dev", "instances": [ "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36 Playwright/1.40.0" ], "description": "Playwright browser automation web crawler", "tags": [ "browser-automation" ] }, { "pattern": "Puppeteer", "addition_date": "2026/04/07", "url": "https://pptr.dev", "instances": [ "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) HeadlessChrome/120.0.0.0 Safari/537.36 Puppeteer" ], "description": "Puppeteer browser automation web crawler", "tags": [ "browser-automation" ] }, { "pattern": "Selenium", "addition_date": "2026/04/07", "url": "https://www.selenium.dev", "instances": [ "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36; Selenium" ], "description": "Selenium browser automation web crawler", "tags": [ "browser-automation" ] }, { "pattern": "Nikto", "addition_date": "2026/04/07", "url": "https://cirt.net/Nikto2", "instances": [ "Mozilla/5.00 (Nikto/2.1.5) (Evasions:None) (Test:Port Check)", "Mozilla/5.0 (X11; Linux x86_64) Nikto/2.5.0 (Evasions:None) (Test:Port Check)" ], "description": "Nikto web server security scanner bot", "tags": [ "scanner" ] }, { "pattern": "sqlmap", "addition_date": "2026/04/07", "url": "https://sqlmap.org", "instances": [ "sqlmap/1.7.8#stable (https://sqlmap.org)" ], "description": "SQLMap SQL injection testing web crawler", "tags": [ "scanner" ] }, { "pattern": "ZmEu", "addition_date": "2026/04/07", "url": "https://en.wikipedia.org/wiki/ZmEu_(vulnerability_scanner)", "instances": [ "ZmEu" ], "description": "ZmEu vulnerability scanner web crawler", "tags": [ "scanner" ] }, { "pattern": "masscan", "addition_date": "2026/04/07", "url": "https://github.com/robertdavidgraham/masscan", "instances": [ "masscan/1.0 (https://github.com/robertdavidgraham/masscan)" ], "description": "Masscan network scanner web crawler bot", "tags": [ "scanner" ] }, { "pattern": "WPScan", "addition_date": "2026/04/07", "url": "https://wpscan.com", "instances": [ "WPScan v3.8.22 (https://wpscan.com/wordpress-security-scanner)", "Mozilla/5.0 (compatible; WPScan; +https://wpscan.com/wordpress-security-scanner)" ], "description": "WPScan WordPress security scanner bot", "tags": [ "scanner" ] }, { "pattern": "[aA]cunetix", "addition_date": "2026/04/07", "url": "https://www.acunetix.com", "instances": [ "Mozilla/5.0 (Windows NT 6.1; WOW64) acunetix-product/wvs (Acunetix Web Vulnerability Scanner - Free Edition)", "acunetix-product/wvs" ], "description": "Acunetix web vulnerability scanner bot", "tags": [ "scanner" ] }, { "pattern": "Nessus", "addition_date": "2026/04/07", "url": "https://www.tenable.com/products/nessus", "instances": [ "Mozilla/5.0 (compatible; Nessus; http://www.nessus.org)" ], "description": "Nessus vulnerability scanner web crawler", "tags": [ "scanner" ] }, { "pattern": "[dD]ir[Bb]uster", "addition_date": "2026/04/07", "url": "https://github.com/KajanM/DirBuster", "instances": [ "DirBuster-1.0-RC1 (http://www.owasp.org/index.php/Category:OWASP_DirBuster_Project)" ], "description": "DirBuster directory scanner web crawler", "tags": [ "scanner" ] }, { "pattern": "StatusCake", "addition_date": "2026/04/07", "url": "https://www.statuscake.com", "instances": [ "StatusCake_Uptime_Checker/1.0" ], "description": "StatusCake uptime monitoring web crawler", "tags": [ "monitoring" ] }, { "pattern": "colly", "addition_date": "2026/04/07", "url": "https://go-colly.org", "instances": [ "colly - https://github.com/gocolly/colly", "colly/2.1.0" ], "description": "Colly Go web crawler framework", "tags": [ "http-library" ] }, { "pattern": "[mM]echanize", "addition_date": "2026/04/07", "url": "https://github.com/sparklemotion/mechanize", "instances": [ "Mechanize/2.9.1 Ruby/3.1.2 (http://github.com/sparklemotion/mechanize/)" ], "description": "Mechanize Ruby web crawler library", "tags": [ "http-library" ] }, { "pattern": "air\\.ai\\/scanning", "addition_date": "2026/04/07", "instances": [ "air.ai/scanning Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) Chrome/126.0.0.0 Safari/537.36" ], "description": "Air.ai web crawler for scanning", "tags": [ "scanner" ] }, { "pattern": "asnriskscorer", "addition_date": "2026/04/07", "instances": [ "asnriskscorer/1.0" ], "description": "ASN Risk Scorer web crawler bot", "tags": [ "scanner" ] }, { "pattern": "OICrawler", "addition_date": "2026/04/07", "url": "https://openindex.ai", "instances": [ "OICrawler/Nutch https://openindex.ai" ], "description": "OpenIndex web crawler for indexing", "tags": [ "search-engine" ] }, { "pattern": "l9scan", "addition_date": "2026/04/07", "url": "https://github.com/LeakIX/l9scan", "instances": [ "Mozilla/5.0 (l9scan/2.0; +https://github.com/LeakIX/l9scan)" ], "description": "L9scan web crawler for scanning", "tags": [ "scanner" ] }, { "pattern": "SlaccaleBot", "addition_date": "2026/04/07", "instances": [ "SlaccaleBot" ], "description": "Slaccale web crawler bot", "tags": [ "seo" ] }, { "pattern": "CustomAsyncHttpClient", "addition_date": "2026/04/07", "instances": [ "CustomAsyncHttpClient" ], "description": "Custom async HTTP client web crawler", "tags": [ "http-library" ] }, { "pattern": "^HTTPie\\/", "addition_date": "2026/04/07", "url": "https://httpie.io", "instances": [ "HTTPie/3.2.2" ], "description": "HTTPie command-line HTTP client tool", "tags": [ "http-library" ] }, { "pattern": "Gemini-Deep-Research", "addition_date": "2026/04/07", "url": "https://developers.google.com/search/docs/crawling-indexing/google-common-crawlers", "instances": [ "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Gemini-Deep-Research; +https://gemini.google/overview/deep-research/) Chrome/135.0.0.0 Safari/537.36" ], "description": "Google Gemini deep research web crawler", "tags": [ "ai-crawler", "search-engine" ] }, { "pattern": "Perplexity-User", "addition_date": "2026/04/07", "url": "https://docs.perplexity.ai/guides/bots", "instances": [ "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Perplexity-User/1.0; +https://perplexity.ai/perplexity-user)" ], "description": "Perplexity user web crawler bot", "tags": [ "ai-crawler", "search-engine" ] }, { "pattern": "PerplexityUser", "addition_date": "2026/04/07", "url": "https://perplexity.ai", "instances": [ "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; PerplexityUser/1.0; +https://perplexity.ai)" ], "description": "Perplexity AI web crawler bot", "tags": [ "ai-crawler", "search-engine" ] }, { "pattern": "meta-webindexer", "addition_date": "2026/04/07", "url": "https://developers.facebook.com/docs/sharing/webmasters/web-crawlers#meta-webindexer", "instances": [ "meta-webindexer/1.1 (+https://developers.facebook.com/docs/sharing/webmasters/crawler)" ], "description": "Meta web indexer bot for Facebook and Instagram content crawling", "tags": [ "social-preview" ] }, { "pattern": "DuckAssistBot", "addition_date": "2026/04/07", "url": "https://duckduckgo.com/duckduckgo-help-pages/results/duckassistbot", "instances": [ "DuckAssistBot/1.2; (+http://duckduckgo.com/duckassistbot.html)" ], "description": "DuckDuckGo assistant bot for web crawling and search", "tags": [ "search-engine", "ai-crawler" ] }, { "pattern": "MistralAI-User", "addition_date": "2026/04/07", "instances": [ "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; MistralAI-User/1.0; +https://docs.mistral.ai/robots)" ], "description": "Mistral AI web crawler bot for content indexing", "tags": [ "ai-crawler" ] }, { "pattern": "webzio", "addition_date": "2026/04/07", "url": "https://webz.io/blog/company/from-omgilibot-to-the-webzbot-duo-a-powerful-leap-for-ethical-and-comprehensive-data-collection/#", "instances": [ "webzio (+https://webz.io/bot.html)" ], "description": "Webz.io web crawler bot for ethical data collection", "tags": [ "seo" ] }, { "pattern": "newsai\\/", "addition_date": "2026/04/14", "url": "https://knownagents.com/agents/newsai", "instances": [ "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/140.0.0.0 newsai/1.0 Safari/537.36" ], "description": "NewsAI crawler for news content aggregation and indexing", "tags": [ "ai-crawler", "feed-reader" ] }, { "pattern": "^ArenaUnfurlBot", "url": "https://arena.ai/", "instances": [ "ArenaUnfurlBot/1.0 (Link preview bot for AI model comparison platform; +https://arena.ai) Mozilla/5.0 (compatible)" ], "addition_date": "2026/04/26", "description": "Arena.ai link preview generator", "tags": [ "social-preview" ] }, { "pattern": "A360-Search", "url": "https://area360.uk/", "instances": [ "Mozilla/5.0 (compatible; A360-Search; +https://area360.uk/)" ], "addition_date": "2026/04/26", "description": "Area360 real estate data crawler", "tags": [ "advertising" ] }, { "pattern": "AASA-Bot", "url": "https://developer.apple.com/documentation/xcode/allowing-apps-and-websites-to-link-to-your-content", "instances": [ "AASA-Bot/1.0.0" ], "addition_date": "2026/04/26", "description": "Apple's Universal Links file fetcher", "tags": [ "social-preview" ] }, { "pattern": "AccessStatus", "url": "https://accesslink.fr/page/a-propos-de-accessstatus/", "instances": [ "Mozilla/5.0 (compatible; AccessStatus/1.0; +https://www.accesslink.fr/page/a-propos-de-accessstatus/)" ], "addition_date": "2026/04/26", "description": "HTTP status code monitoring bot", "tags": [ "monitoring" ] }, { "pattern": "Acquia optimize", "url": "https://knownagents.com/agents/acquia-optimize-monsido", "instances": [ "Acquia optimize (Monsido)" ], "addition_date": "2026/04/26", "description": "Acquia website optimization monitoring", "tags": [ "monitoring" ] }, { "pattern": "ActiveComply", "url": "https://knownagents.com/agents/activecomply-bot", "instances": [ "Mozilla/5.0 (compatible; ActiveComply; +https://www.activecomply.com/)", "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/140.0.7339.16 Safari/537.36 (compatible; ActiveComply/2.0; +https://app.activecomply.com/bot)" ], "addition_date": "2026/04/26", "description": "ActiveComply compliance monitoring bot", "tags": [ "monitoring" ] }, { "pattern": "AdkernelTopicCrawler", "url": "http://adkernel.com/robot/", "instances": [ "Mozilla/5.0 (compatible; AdkernelTopicCrawler/1.0; +http://adkernel.com/robot/)" ], "addition_date": "2026/04/26", "description": "Adkernel ad tech solutions crawler", "tags": [ "advertising" ] }, { "pattern": "AlertSite", "url": "https://smartbear.com/product/alertsite/", "instances": [ "Mozilla/5.0 (compatible; AlertSite; +https://www.alertsite.com/)" ], "addition_date": "2026/04/26", "description": "SmartBear's synthetic monitoring service", "tags": [ "monitoring" ] }, { "pattern": "AllAfrica", "url": "https://allafrica.com/misc/info/about/", "instances": [ "Mozilla/5.0 (compatible; AllAfrica; +https://allafrica.com/)", "Mozilla/5.0 (compatible; AllAfrica NewsBot; +https://allafrica.com/)" ], "addition_date": "2026/04/26", "description": "AllAfrica news aggregation crawler", "tags": [ "feed-reader" ] }, { "pattern": "Amazing-SearchBot", "url": "https://amazing.com/bot.html", "instances": [ "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Amazing-SearchBot/0.1; +https://amazing.com/bot.html) Chrome/119.0.6045.214 Safari/537.36" ], "addition_date": "2026/04/26", "description": "Amazing e-commerce platform search crawler", "tags": [ "advertising" ] }, { "pattern": "Amazon-Bedrock-AgentCore-Browser", "url": "https://docs.aws.amazon.com/bedrock-agentcore/", "instances": [ "Mozilla/5.0 (compatible; Amazon-Bedrock-AgentCore-Browser; +https://aws.amazon.com/bedrock/)" ], "addition_date": "2026/04/26", "description": "AWS cloud browser for AI agents", "tags": [ "browser-automation" ] }, { "pattern": "AmazonBuyForMe", "url": "https://buyforme.amazon/", "instances": [ "Mozilla/5.0 (compatible; AmazonBuyForMe; +https://www.amazon.com/)" ], "addition_date": "2026/04/26", "description": "Amazon bot for Buy For Me service purchases", "tags": [ "browser-automation" ] }, { "pattern": "Amzn-SearchBot", "url": "https://developer.amazon.com/amazonbot", "instances": [ "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Amzn-SearchBot/0.1) Chrome/119.0.6045.214 Safari/537.36" ], "addition_date": "2026/04/26", "description": "Amazon's AI search indexer for Alexa", "tags": [ "ai-crawler" ] }, { "pattern": "Amzn-User", "url": "https://developer.amazon.com/amazonbot", "instances": [ "Mozilla/5.0 (compatible; Amzn-User; +https://developer.amazon.com/)" ], "addition_date": "2026/04/26", "description": "Amazon AI assistant for Alexa queries", "tags": [ "ai-crawler" ] }, { "pattern": "Anchor Browser", "url": "https://knownagents.com/agents/anchor-browser", "instances": [ "Mozilla/5.0 (compatible; Anchor Browser; +https://anchorbrowser.io/)" ], "addition_date": "2026/04/26", "description": "Anchor's cloud-hosted browser for AI agents", "tags": [ "browser-automation" ] }, { "pattern": "Anomura", "url": "https://docs.direqt-search.com/direqt-bots/direqt-crawlers-and-user-agents", "instances": [ "Anomura/1.2 (+https://www.direqt.ai)" ], "addition_date": "2026/04/26", "description": "Direqt's AI search indexer", "tags": [ "ai-crawler" ] }, { "pattern": "AP3A\\.240617\\.008", "url": "https://knownagents.com/agents/008", "instances": [ "Mozilla/5.0 (Linux; Android 15; CPH2557 Build/AP3A.240617.008; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/142.0.7444.142 Mobile Safari/537.36 Instagram 406.0.0.58.159 Android (35/15; 480dpi; 1080x2400; OPPO; CPH2557; OP573DL1; mt6833; en_MY; 822918295; IABMV/1) NV/1" ], "addition_date": "2026/04/26", "description": "80legs web scraping service", "tags": [ "scanner" ] }, { "pattern": "ApifyBot", "url": "https://knownagents.com/agents/apifybot", "instances": [ "Mozilla/5.0 (compatible; ApifyBot/1.0)" ], "addition_date": "2026/04/26", "description": "Apify's web scraping and data extraction", "tags": [ "ai-crawler" ] }, { "pattern": "ApifyWebsiteContentCrawler", "url": "https://apify.com/apify/website-content-crawler", "instances": [ "ApifyWebsiteContentCrawler/1.0 (+https://apify.com/apify/website-content-crawler)" ], "addition_date": "2026/04/26", "description": "Apify's full website content extractor", "tags": [ "ai-crawler" ] }, { "pattern": "Archive-It", "url": "http://archive-it.org/files/site-owners-special.html", "instances": [ "UM-Bentley-Archive-It", "Mozilla/5.0 (compatible; special_archiver; Archive-It; +http://archive-it.org/files/site-owners-special.html)" ], "addition_date": "2026/04/26", "description": "Internet Archive's web preservation crawler", "tags": [ "archiver" ] }, { "pattern": "artemis web reader", "url": "https://artemis.jamesg.blog/bot", "instances": [ "artemis web reader/1.0 - https://artemis.jamesg.blog/bot" ], "addition_date": "2026/04/26", "description": "Artemis personal web reader", "tags": [ "feed-reader" ] }, { "pattern": "atlassian-bot", "url": "https://support.atlassian.com/organization-administration/docs/connect-custom-website-to-rovo/", "instances": [ "Mozilla/5.0 (compatible; atlassian-bot; +https://www.atlassian.com/)" ], "addition_date": "2026/04/26", "description": "Atlassian Rovo's AI search crawler", "tags": [ "ai-crawler" ] }, { "pattern": "Attracta", "url": "https://attracta.com/", "instances": [ "Mozilla/5.0 (compatible; Attracta)" ], "addition_date": "2026/04/26", "description": "Attracta SEO optimization crawler", "tags": [ "seo" ] }, { "pattern": "AudigentAdBot", "url": "http://audigent.com/bot.html", "instances": [ "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; AudigentAdBot; +http://www.audigent.com/bot.html) Chrome/W.X.Y.Z Safari/537.36" ], "addition_date": "2026/04/26", "description": "Audigent targeted advertising crawler", "tags": [ "advertising" ] }, { "pattern": "Authory", "url": "https://authory.com/about", "instances": [ "Mozilla/5.0 (compatible; Authory/1.0; +https://authory.com/)" ], "addition_date": "2026/04/26", "description": "Automated content archiving for journalists", "tags": [ "archiver" ] }, { "pattern": "Automaton|Newsify Feed Fetcher", "url": "https://knownagents.com/agents/automaton", "instances": [ "Newsify Feed Fetcher - 3 subscribers - http://www.newsify.co/site/899429/automaton-twocanoes-software (Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_1) AppleWebKit/534.48.3 (KHTML, like Gecko) Version/5.1 Safari/534.48.3)" ], "addition_date": "2026/04/26", "description": "Automaton feed fetcher monitoring", "tags": [ "monitoring" ] }, { "pattern": "AwarioRendererBot", "url": "https://awario.com/help/", "instances": [ "AwarioRendererBot/1.0 (+https://awario.com/bots.html; bots@awario.com)" ], "addition_date": "2026/04/26", "description": "Awario social media monitoring crawler", "tags": [ "advertising" ] }, { "pattern": "AzureAI-SearchBot", "url": "https://azure.microsoft.com/en-us/products/ai-services", "instances": [ "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko); compatible; AzureAI-SearchBot/1.0;" ], "addition_date": "2026/04/26", "description": "Microsoft's Azure AI search indexer", "tags": [ "ai-crawler" ] }, { "pattern": "BestChange", "url": "https://bestchange.com/", "instances": [ "Mozilla/5.0 (compatible; BestChange; +https://bestchange.com/)", "Mozilla/5.0 (compatible; BestChange Bot; +https://bestchange.com/)" ], "addition_date": "2026/04/26", "description": "BestChange exchange rate monitoring crawler", "tags": [ "feed-reader" ] }, { "pattern": "bigsur\\.ai", "url": "https://bigsur.ai/", "instances": [ "bigsur.ai" ], "addition_date": "2026/04/26", "description": "Big Sur AI's web agent crawler", "tags": [ "ai-crawler" ] }, { "pattern": "bl\\.uk_lddc_bot", "url": "https://bl.uk/legal-deposit-web-archiving", "instances": [ "bl.uk_lddc_bot/3.4.0-20220727 (+https://www.bl.uk/legal-deposit-web-archiving)" ], "addition_date": "2026/04/26", "description": "British Library's legal deposit archiver", "tags": [ "archiver" ] }, { "pattern": "BlingERP", "url": "https://bling.com.br/", "instances": [ "Mozilla/5.0 (compatible; BlingERP; +https://www.bling.com.br/)" ], "addition_date": "2026/04/26", "description": "Bling ERP data synchronization crawler", "tags": [ "feed-reader" ] }, { "pattern": "Blockaid", "url": "https://knownagents.com/agents/blockaid", "instances": [ "Mozilla/5.0 (compatible; Blockaid; +https://www.blockaid.io/)" ], "addition_date": "2026/04/26", "description": "Blockaid security monitoring", "tags": [ "monitoring" ] }, { "pattern": "Bloglines", "url": "http://bloglines.com/", "instances": [ "Bloglines/3.1 (http://www.bloglines.com)" ], "addition_date": "2026/04/26", "description": "Bloglines RSS feed reader", "tags": [ "feed-reader" ] }, { "pattern": "BlogVault", "url": "https://blogvault.net/", "instances": [ "BlogVault/1.0 (+https://blogvault.net)" ], "addition_date": "2026/04/26", "description": "WordPress backup and monitoring service", "tags": [ "monitoring" ] }, { "pattern": "bluesky-domain-status-classifier", "url": "https://blueskyweb.xyz/", "instances": [ "bluesky-domain-status-classifier/1.0" ], "addition_date": "2026/04/26", "description": "Bluesky social network link preview bot", "tags": [ "social-preview" ] }, { "pattern": "Bluesky\\/", "url": "https://knownagents.com/agents/bluesky-link-preview-service", "instances": [ "Bluesky/" ], "addition_date": "2026/04/26", "description": "Bluesky link preview service", "tags": [ "social-preview" ] }, { "pattern": "bne\\.es_bot", "url": "https://bne.es/es/colecciones/archivo-web-espanola/aviso-webmasters", "instances": [ "Mozilla/5.0 (compatible; bne.es_bot; https://www.bne.es/es/colecciones/archivo-web-espanola/aviso-webmasters) Firefox/129.0.1" ], "addition_date": "2026/04/26", "description": "Spanish National Library web archiver", "tags": [ "archiver" ] }, { "pattern": "Brightbot", "url": "https://brightdata.com/brightbot", "instances": [ "Brightbot 1.0" ], "addition_date": "2026/04/26", "description": "Bright Data's AI-ready data collector", "tags": [ "ai-crawler" ] }, { "pattern": "BrowserBot-Observer", "url": "https://obsrvr.net/about", "instances": [ "BrowserBot-Observer (+https://siteobserver.co)" ], "addition_date": "2026/04/26", "description": "Observer third-party risk security scanner", "tags": [ "scanner" ] }, { "pattern": "BufferLinkPreviewBot", "url": "https://scraper.buffer.com/about/bots/link-preview-bot", "instances": [ "BufferLinkPreviewBot/1.0 (+https://scraper.buffer.com/about/bots/link-preview-bot)" ], "addition_date": "2026/04/26", "description": "Buffer social media link preview bot", "tags": [ "social-preview" ] }, { "pattern": "Bugsnag", "url": "https://knownagents.com/agents/bugsnag-script-fetcher", "instances": [ "Mozilla/5.0 (compatible; Bugsnag; +https://www.bugsnag.com/)", "Bugsnag script fetcher (support@bugsnag.com)" ], "addition_date": "2026/04/26", "description": "Bugsnag JavaScript source fetcher", "tags": [ "monitoring" ] }, { "pattern": "Buttondown", "url": "https://buttondown.email/features", "instances": [ "Buttondown RSS-Feed-Parser/1.0 (https://buttondown.com)" ], "addition_date": "2026/04/26", "description": "Buttondown RSS to email converter", "tags": [ "feed-reader" ] }, { "pattern": "CapitalOneBot", "url": "https://developer.capitalone.com/", "instances": [ "Mozilla/5.0 (compatible; CapitalOneBot; +https://www.capitalone.com/)" ], "addition_date": "2026/04/26", "description": "Capital One dealer website crawler", "tags": [ "feed-reader" ] }, { "pattern": "CertChief", "url": "https://cert.chief.app/", "instances": [ "Mozilla/5.0 (compatible; CertChief; +https://www.certchief.com/)" ], "addition_date": "2026/04/26", "description": "CertChief SSL certificate monitoring bot", "tags": [ "scanner" ] }, { "pattern": "channable", "url": "https://channable.com/", "instances": [ "Mozilla/5.0 (compatible; channable; +https://www.channable.com/)" ], "addition_date": "2026/04/26", "description": "Channable e-commerce marketing automation crawler", "tags": [ "advertising" ] }, { "pattern": "Channel3Bot", "url": "https://trychannel3.com/channel3bot", "instances": [ "Mozilla/5.0 (compatible; Channel3Bot/1.0; +https://trychannel3.com/channel3bot)" ], "addition_date": "2026/04/26", "description": "Channel3's universal product catalog indexer", "tags": [ "ai-crawler" ] }, { "pattern": "Chirp|gotosocial", "url": "http://binarycanary.com/", "instances": [ "gotosocial/0.19.2+git-90851fc (+https://chirp.zadzmo.org)" ], "addition_date": "2026/04/26", "description": "Binary Canary uptime monitoring bot", "tags": [ "monitoring" ] }, { "pattern": "ClickUpLinkUnfurler", "url": "https://clickup.com/", "instances": [ "ClickUpLinkUnfurler/1.0 (+https://clickup.com)" ], "addition_date": "2026/04/26", "description": "ClickUp link preview generator", "tags": [ "social-preview" ] }, { "pattern": "Cloudflare-AutoRAG", "url": "https://developers.cloudflare.com/autorag", "instances": [ "Cloudflare-AutoRAG (https://developers.cloudflare.com/autorag; autorag@cloudflare.com)" ], "addition_date": "2026/04/26", "description": "Cloudflare's RAG service indexer", "tags": [ "ai-crawler" ] }, { "pattern": "Cloudflare-Custom-Hostname-Verification", "url": "https://knownagents.com/agents/cloudflare-custom-hostname-verification", "instances": [ "Mozilla/5.0 (compatible; Cloudflare-Custom-Hostname-Verification; +https://www.cloudflare.com/)" ], "addition_date": "2026/04/26", "description": "Cloudflare hostname verification crawler", "tags": [ "seo" ] }, { "pattern": "Cloudflare-Stream-Webhook", "url": "https://knownagents.com/agents/cloudflare-stream-webhook", "instances": [ "Mozilla/5.0 (compatible; Cloudflare-Stream-Webhook; +https://www.cloudflare.com/)" ], "addition_date": "2026/04/26", "description": "Cloudflare Stream webhook crawler", "tags": [ "seo" ] }, { "pattern": "CloudflareRadarURLScanner", "url": "https://knownagents.com/agents/cloudflare-radar-url-scanner", "instances": [ "Mozilla/5.0 (compatible; CloudflareRadarURLScanner; +https://radar.cloudflare.com/)" ], "addition_date": "2026/04/26", "description": "Cloudflare URL security scanner", "tags": [ "monitoring" ] }, { "pattern": "Cloudtrellis", "url": "https://cloudtrellis.com/", "instances": [ "Mozilla/5.0 (compatible; Cloudtrellis; +https://www.cloudtrellis.com/)" ], "addition_date": "2026/04/26", "description": "Cloudtrellis website audit crawler", "tags": [ "seo" ] }, { "pattern": "[cC]ludo", "url": "https://knownagents.com/agents/cludo", "instances": [ "Mozilla/5.0 (compatible; Cludo; +https://www.cludo.com/)", "Mozilla/5.0 (Windows NT 10.0; cludo.com bot) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36" ], "addition_date": "2026/04/26", "description": "Cludo site search monitoring", "tags": [ "monitoring" ] }, { "pattern": "Code\\/1\\.", "url": "https://github.com/features/copilot", "instances": [ "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Code/1.115.0 Chrome/142.0.7444.265 Electron/39.8.5 Safari/537.36" ], "addition_date": "2026/04/26", "description": "GitHub Copilot AI coding agent", "tags": [ "browser-automation" ] }, { "pattern": "Collapsify", "url": "https://developers.cloudflare.com/", "instances": [ "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10; rv:60.0) Gecko/20100101 Firefox/60.0 Collapsify" ], "addition_date": "2026/04/26", "description": "Cloudflare error page caching service", "tags": [ "feed-reader" ] }, { "pattern": "ContextualBot[\\s\\S]*outcomes\\.net", "url": "http://outcomes.net/", "instances": [ "Mozilla/5.0 (compatible; ContextualBot/1.0; +http://outcomes.net)" ], "addition_date": "2026/04/26", "description": "Mobian contextual intelligence crawler", "tags": [ "advertising" ] }, { "pattern": "Convermax", "url": "https://docs.convermax.com/", "instances": [ "Convermax/1.0 (+https://docs.convermax.com/indexer)" ], "addition_date": "2026/04/26", "description": "Convermax auto parts search indexer", "tags": [ "seo" ] }, { "pattern": "cookie-maestro", "url": "https://cookiemaestro.com/documentatie/limit-cookie-maestro-using-robots-txt", "instances": [ "Mozilla/5.0 (compatible; cookie-maestro; +https://www.cookiemaestro.com/)" ], "addition_date": "2026/04/26", "description": "Cookie Maestro GDPR compliance scanner", "tags": [ "scanner" ] }, { "pattern": "CookieHubVerify", "url": "https://cookiehub.com/", "instances": [ "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36 CookieHubVerify/3.0" ], "addition_date": "2026/04/26", "description": "CookieHub consent banner verification scanner", "tags": [ "scanner" ] }, { "pattern": "CookieYesbot", "url": "http://cookieyes.com/documentation/cookieyesbot", "instances": [ "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko; CookieYesbot/1.0; +http://www.cookieyes.com/documentation/cookieyesbot) Chrome/131.0.6778.0 Safari/537.36" ], "addition_date": "2026/04/26", "description": "CookieYes consent compliance scanner", "tags": [ "scanner" ] }, { "pattern": "Crazy Egg", "url": "https://knownagents.com/agents/crazy-egg", "instances": [ "Mozilla/5.0 (compatible; Crazy Egg; +https://www.crazyegg.com/)" ], "addition_date": "2026/04/26", "description": "Crazy Egg analytics monitoring bot", "tags": [ "monitoring" ] }, { "pattern": "Current[\\s\\S]*RSS Reader", "url": "https://currentreader.app/", "instances": [ "Mozilla/5.0 (compatible; Current/1.0; +https://currentreader.app; RSS Reader)" ], "addition_date": "2026/04/26", "description": "Current RSS reader application", "tags": [ "feed-reader" ] }, { "pattern": "cypex\\.ai\\/scanning", "url": "https://cypex.ai/", "instances": [ "cypex.ai/scanning Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) Chrome/126.0.0.0 Safari/537.36" ], "addition_date": "2026/04/26", "description": "Cypex security vulnerability scanner", "tags": [ "scanner" ] }, { "pattern": "DeepCrawl", "url": "https://lumar.io/spdr/", "instances": [ "Mozilla/5.0 (compatible; MentalHealthLeadBot/DeepCrawl/1.0; +https://aiinstall.co)" ], "addition_date": "2026/04/26", "description": "Lumar enterprise SEO platform crawler", "tags": [ "seo" ] }, { "pattern": "DigiCert DCV", "url": "https://digicert.com/", "instances": [ "DigiCert DCV/1.1" ], "addition_date": "2026/04/26", "description": "DigiCert domain validation scanner", "tags": [ "scanner" ] }, { "pattern": "dlvr\\.it", "url": "http://dlvr.it/", "instances": [ "dlvr.it/1.0 (+http://dlvr.it/fetcher)" ], "addition_date": "2026/04/26", "description": "dlvr.it social media automation service", "tags": [ "feed-reader" ] }, { "pattern": "Dotcom-Monitor", "url": "https://knownagents.com/agents/doctom-monitor", "instances": [ "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.97 Safari/537.36 (Dotcom-Monitor)" ], "addition_date": "2026/04/26", "description": "Doctom website monitoring", "tags": [ "monitoring" ] }, { "pattern": "DrataAutopilot", "url": "https://knownagents.com/agents/drata-autopilot", "instances": [ "Mozilla/5.0 (compatible; DrataAutopilot; +https://drata.com/)" ], "addition_date": "2026/04/26", "description": "Drata compliance monitoring", "tags": [ "monitoring" ] }, { "pattern": "DreamHost Data Team", "url": "http://dreamhost.com/support/", "instances": [ "DreamHost Data Team (+http://www.dreamhost.com/support/)" ], "addition_date": "2026/04/26", "description": "DreamHost hosting monitoring bot", "tags": [ "monitoring" ] }, { "pattern": "ds9", "url": "https://data.dss.sps.copyright.com/docs/user_agent.html", "instances": [ "ds9 2.004.ec2" ], "addition_date": "2026/04/26", "description": "Copyright Clearance Center licensing crawler", "tags": [ "advertising" ] }, { "pattern": " DVbot", "url": "http://doubleverify.com/", "instances": [ "Mozilla/5.0 (Windows NT 6.3;compatible; DVbot/1.0; +http://www.doubleverify.com)" ], "addition_date": "2026/04/26", "description": "DoubleVerify ad verification crawler", "tags": [ "advertising" ] }, { "pattern": "EcoVadisSustainabilityBot", "url": "https://ecovadis.com/", "instances": [ "Mozilla/5.0 (compatible; EcoVadisSustainabilityBot/1.0; +https://www.ecovadis.com; Crawling on behalf of one of your business partners.)" ], "addition_date": "2026/04/26", "description": "EcoVadis sustainability ratings crawler", "tags": [ "advertising" ] }, { "pattern": "elmah\\.io Uptime Monitoring", "url": "https://knownagents.com/agents/elmah-io-uptime-monitoring", "instances": [ "elmah.io Uptime Monitoring" ], "addition_date": "2026/04/26", "description": "elmah.io uptime monitoring service", "tags": [ "monitoring" ] }, { "pattern": "EvernoteRichLinkBot", "url": "https://evernote.com/", "instances": [ "EvernoteRichLinkBot/1.0 (+https://evernote.com)" ], "addition_date": "2026/04/26", "description": "Evernote rich link preview bot", "tags": [ "social-preview" ] }, { "pattern": "EzLynx", "url": "http://ezoic.com/bot.html", "instances": [ "https://www.ezoic.com/bot/ Mozilla/5.0 (Linux; Android 8.0; Pixel 2 Build/OPD3.170816.012) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/72.0.3626.121 Mobile Safari/537.36 (compatible; EzLynx/0.1; +http://www.ezoic.com/bot.html)" ], "addition_date": "2026/04/26", "description": "Ezoic website optimization crawler", "tags": [ "advertising" ] }, { "pattern": "EzoicBot", "url": "https://ezoic.com/bot/", "instances": [ "Mozilla/5.0 (Linux; Android 8.0; Pixel 2 Build/OPD3.170816.012) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/72.0.3626.121 Mobile Safari/537.36 (compatible; EzoicBot-UptimeOwl; +http://www.ezoic.com/bot)", "Mozilla/5.0 (Linux; Android 8.0; Pixel 2 Build/OPD3.170816.012) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/72.0.3626.121 Mobile Safari/537.36 (compatible; EzoicBot-Nicheiq; +http://www.ezoic.com/bot)" ], "addition_date": "2026/04/26", "description": "Ezoic ad monetization platform crawler", "tags": [ "advertising" ] }, { "pattern": "FacebookBot", "url": "https://developers.facebook.com/docs/sharing/bot/", "instances": [ "Mozilla/5.0 (compatible; FacebookBot/1.0; +https://developers.facebook.com/docs/sharing/webmasters/facebookbot/)" ], "addition_date": "2026/04/26", "description": "Meta's AI speech recognition training crawler", "tags": [ "ai-crawler" ] }, { "pattern": "FastDAST", "url": "https://knownagents.com/agents/black-duck-fast-dynamic", "instances": [ "FastDAST", "Mozilla/5.0 (compatible; FastDAST; +https://www.blackduck.com/)" ], "addition_date": "2026/04/26", "description": "Black Duck security testing bot", "tags": [ "monitoring" ] }, { "pattern": "Feeder \\/", "url": "https://knownagents.com/agents/feeder", "instances": [ "Feeder / 1.10.8(88)" ], "addition_date": "2026/04/26", "description": "Feeder RSS reader app", "tags": [ "feed-reader" ] }, { "pattern": "FeedFlow", "url": "https://feedflow.dev/", "instances": [ "FeedFlow/1.0" ], "addition_date": "2026/04/26", "description": "FeedFlow RSS reader application", "tags": [ "feed-reader" ] }, { "pattern": "FindFiles\\.net", "url": "https://findfiles.net/bot", "instances": [ "FindFiles.net-FaviconFetcher/1.0 (+https://findfiles.net/bot)" ], "addition_date": "2026/04/26", "description": "FindFiles.net search engine crawler", "tags": [ "search-engine" ] }, { "pattern": "FirecrawlAgent", "url": "https://firecrawl.dev/", "instances": [ "Mozilla/5.0 (compatible; FirecrawlAgent; +https://firecrawl.dev/)" ], "addition_date": "2026/04/26", "description": "Firecrawl's LLM data extraction crawler", "tags": [ "ai-crawler" ] }, { "pattern": "FyndSearchEngine-Crawler", "url": "https://fynd.bot/", "instances": [ "FyndSearchEngine-Crawler (by fynd.bot; https://fynd.bot)" ], "addition_date": "2026/04/26", "description": "Fynd search engine indexing crawler", "tags": [ "search-engine" ] }, { "pattern": "FyndSearchEngine-ReCrawler", "url": "https://fynd.bot/", "instances": [ "FyndSearchEngine-ReCrawler (by fynd.bot; https://fynd.bot)" ], "addition_date": "2026/04/26", "description": "Fynd search engine re-indexing crawler", "tags": [ "search-engine" ] }, { "pattern": "Goodreads", "url": "https://goodreads.com/", "instances": [ "Mozilla/5.0 (compatible; Goodreads; +https://www.goodreads.com)" ], "addition_date": "2026/04/26", "description": "Goodreads book preview fetcher", "tags": [ "social-preview" ] }, { "pattern": "Google Trust Services", "url": "https://knownagents.com/agents/google-trust-services-dcv-check", "instances": [ "Google Trust Services (DCV Check)" ], "addition_date": "2026/04/26", "description": "Google SSL certificate validation", "tags": [ "monitoring" ] }, { "pattern": "Google-Agent", "url": "https://developers.google.com/crawling/docs/crawlers-fetchers/google-user-triggered-fetchers", "instances": [ "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko; compatible; Google-Agent)" ], "addition_date": "2026/04/26", "description": "Google's user-triggered agent for web navigation", "tags": [ "browser-automation" ] }, { "pattern": "Google-Gemini-CLI", "url": "https://geminicli.com/", "instances": [ "Mozilla/5.0 (compatible; Google-Gemini-CLI/1.0; +https://github.com/google-gemini/gemini-cli)" ], "addition_date": "2026/04/26", "description": "Google's AI coding agent for terminal", "tags": [ "browser-automation" ] }, { "pattern": "Google-NotebookLM", "url": "https://developers.google.com/search/docs/crawling-indexing/google-user-triggered-fetchers", "instances": [ "Mozilla/5.0 (compatible; Google-NotebookLM; +https://notebooklm.google.com/)" ], "addition_date": "2026/04/26", "description": "Google's AI research and note-taking assistant", "tags": [ "ai-crawler" ] }, { "pattern": "GoogleAgent-Mariner", "url": "https://deepmind.google/technologies/project-mariner/", "instances": [ "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko; compatible; GoogleAgent-Mariner) Chrome/142.0.7444.162 Safari/537.36" ], "addition_date": "2026/04/26", "description": "Google's AI agent for browser-based tasks", "tags": [ "browser-automation" ] }, { "pattern": "Greppr Web Crawler", "url": "https://greppr.org/", "instances": [ "Mozilla/5.0 (compatible; Greppr Web Crawler/0.12.0 (https://greppr.org/))" ], "addition_date": "2026/04/26", "description": "Greppr unfiltered search engine crawler", "tags": [ "search-engine" ] }, { "pattern": "Hardenize", "url": "https://hardenize.com/", "instances": [ "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/90.0.4430.212 Safari/537.36 Hardenize" ], "addition_date": "2026/04/26", "description": "Hardenize security configuration scanner", "tags": [ "scanner" ] }, { "pattern": "HoneybadgerBot", "url": "https://knownagents.com/agents/honeybadgerbot", "instances": [ "Mozilla/5.0 (compatible; HoneybadgerBot; +https://www.honeybadger.io/)" ], "addition_date": "2026/04/26", "description": "Honeybadger error monitoring bot", "tags": [ "monitoring" ] }, { "pattern": "IbouBot", "url": "https://ibou.io/iboubot.html", "instances": [ "Mozilla/5.0 (compatible; IbouBot/1.0; +bot@ibou.io; +https://ibou.io/iboubot.html)" ], "addition_date": "2026/04/26", "description": "Ibou web graph search crawler", "tags": [ "search-engine" ] }, { "pattern": "imageSpider", "url": "https://knownagents.com/agents/imagespider", "instances": [ "Mozilla/5.0 (compatible; imageSpider; +https://www.bytedance.com/)" ], "addition_date": "2026/04/26", "description": "ByteDance's image collection crawler", "tags": [ "ai-crawler" ] }, { "pattern": "Innologica", "url": "https://knownagents.com/agents/innologica", "instances": [ "Mozilla/5.0 (compatible; Innologica; +https://www.innologica.com/)" ], "addition_date": "2026/04/26", "description": "Innologica content fetcher", "tags": [ "feed-reader" ] }, { "pattern": "kagi-fetcher", "url": "https://help.kagi.com/kagi/ai/kagi-ai.html", "instances": [ "kagi-fetcher/1.0" ], "addition_date": "2026/04/26", "description": "Kagi AI assistant for web content", "tags": [ "ai-crawler" ] }, { "pattern": "Kangaroo Bot", "url": "https://kangaroollm.com.au/kangaroo-bot/", "instances": [ "Mozilla/5.0 (compatible; Kangaroo Bot/1.0; +http://www.kangaroo.com)" ], "addition_date": "2026/04/26", "description": "Australian AI model training crawler", "tags": [ "ai-crawler" ] }, { "pattern": "Known Agent", "url": "https://knownagents.com/", "instances": [ "Mozilla/5.0 (compatible; Known Agent/1.0; +https://knownagents.com)", "Mozilla/5.0 (compatible; Known Agents Browser/1.0; +https://knownagents.com)" ], "addition_date": "2026/04/26", "description": "Known Agents test and development bot", "tags": [ "monitoring" ] }, { "pattern": "KrawlerBot", "url": "https://krawler.app/robot", "instances": [ "Mozilla/5.0 (compatible; KrawlerBot; +https://www.krawler.com/)" ], "addition_date": "2026/04/26", "description": "Krawler web scraping service bot", "tags": [ "scanner" ] }, { "pattern": "laion-huggingface-processor", "url": "https://knownagents.com/agents/laion-huggingface-processor", "instances": [ "Mozilla/5.0 (compatible; laion-huggingface-processor; +https://laion.ai/)" ], "addition_date": "2026/04/26", "description": "LAION's image dataset builder for AI", "tags": [ "ai-crawler" ] }, { "pattern": "LinkCheckerBot", "url": "https://knownagents.com/agents/linkchecker-bot", "instances": [ "Mozilla/5.0 (compatible; LinkCheckerBot/2.0; +https://linkchecker.pro/robot/)" ], "addition_date": "2026/04/26", "description": "LinkChecker broken link bot", "tags": [ "seo" ] }, { "pattern": "LinkupBot", "url": "https://linkup.so/bot", "instances": [ "LinkupBot/1.0 (LinkupBot for web indexing; https://linkup.so/bot; bot@linkup.so)" ], "addition_date": "2026/04/26", "description": "Linkup's enterprise AI search crawler", "tags": [ "ai-crawler" ] }, { "pattern": "LMArenaUnfurlBot", "url": "https://lmarena.ai/", "instances": [ "LMArenaUnfurlBot/1.0 (Link preview bot for AI model comparison platform; +https://lmarena.ai) Mozilla/5.0 (compatible)" ], "addition_date": "2026/04/26", "description": "LM Arena link preview bot", "tags": [ "social-preview" ] }, { "pattern": "lyonl-asset-proxy", "url": "https://lyonl.com/crawler", "instances": [ "lyonl-asset-proxy/1.0 (+https://lyonl.com/crawler)" ], "addition_date": "2026/04/26", "description": "Lyonl search engine asset proxy", "tags": [ "search-engine" ] }, { "pattern": "lyonl-crawler", "url": "https://lyonl.com/crawler", "instances": [ "lyonl-crawler/0.1 (+https://lyonl.com/crawler; crawler@lyonl.com)" ], "addition_date": "2026/04/26", "description": "Lyonl open web search crawler", "tags": [ "search-engine" ] }, { "pattern": "MagiBot", "url": "https://magi.com/bots", "instances": [ "Mozilla/5.0 (compatible; MagiBot; +https://www.magi.com/)" ], "addition_date": "2026/04/26", "description": "Peak Labs information extraction crawler", "tags": [ "search-engine" ] }, { "pattern": "MagpieRSS", "url": "http://magpierss.sf.net/", "instances": [ "MagpieRSS/0.7 ( http://magpierss.sf.net)" ], "addition_date": "2026/04/26", "description": "MagpieRSS PHP feed parser", "tags": [ "feed-reader" ] }, { "pattern": "mail\\.ru", "url": "https://knownagents.com/agents/mailrubot", "instances": [ "Mozilla/5.0 (Windows NT 6.1; WOW64; https://top.mail.ru/passremind) AppleWebKit/537.22 (KHTML, like Gecko)" ], "addition_date": "2026/04/26", "description": "Mail.ru content fetcher", "tags": [ "feed-reader" ] }, { "pattern": "MailChimp", "url": "http://mailchimp.com/", "instances": [ "Mozilla/5.0 (compatible; MailChimp; +https://mailchimp.com/)" ], "addition_date": "2026/04/26", "description": "Mailchimp email campaign content fetcher", "tags": [ "feed-reader" ] }, { "pattern": "Manus-User", "url": "https://knownagents.com/agents/manus-user", "instances": [ "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/132.0.0.0 Safari/537.36; Manus-User/1.0" ], "addition_date": "2026/04/26", "description": "Butterfly Effect's AI agent for web tasks", "tags": [ "browser-automation" ] }, { "pattern": "McontextualBot", "url": "http://mcontextual.net/mcontextual-bot", "instances": [ "Mozilla/5.0 (compatible; McontextualBot/0.1; +http://mcontextual.net/mcontextual-bot)" ], "addition_date": "2026/04/26", "description": "MContextual advertising targeting crawler", "tags": [ "advertising" ] }, { "pattern": "Mediumbot-MetaTagFetcher", "url": "https://medium.com/", "instances": [ "Mediumbot-MetaTagFetcher/0.3 (+https://medium.com/)" ], "addition_date": "2026/04/26", "description": "Medium link preview meta tag fetcher", "tags": [ "social-preview" ] }, { "pattern": "MetaIAB Facebook", "url": "https://knownagents.com/agents/facebook", "instances": [ "Mozilla/5.0 (Linux; Android 16; Pixel 10 Pro XL Build/CP1A.260305.018; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/146.0.7680.174 Mobile Safari/537.36 MetaIAB Facebook" ], "addition_date": "2026/04/26", "description": "Facebook link preview fetcher", "tags": [ "social-preview" ] }, { "pattern": "MixrankBot", "url": "https://knownagents.com/agents/mixrankbot", "instances": [ "Mozilla/5.0 (compatible; MixrankBot; crawler@mixrank.com)" ], "addition_date": "2026/04/26", "description": "MixRank business intelligence crawler", "tags": [ "advertising" ] }, { "pattern": "ModernizeBot", "url": "https://modernizeyourwebsite.com/bot", "instances": [ "ModernizeBot/0.1 (+https://modernizeyourwebsite.com/bot)" ], "addition_date": "2026/04/26", "description": "ModernizeYourWebsite analysis crawler", "tags": [ "seo" ] }, { "pattern": "MontasticMonitor", "url": "https://knownagents.com/agents/montasticmonitor", "instances": [ "Mozilla/5.0 (compatible; MontasticMonitor; +https://montastic.com/)" ], "addition_date": "2026/04/26", "description": "Montastic uptime monitoring", "tags": [ "monitoring" ] }, { "pattern": "NanoInteractive", "url": "https://nanointeractive.com/crawler/", "instances": [ "Mozilla/5.0 (iPhone; CPU iPhone OS 16_3 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.1 Mobile/15E148 Safari/604.1 (compatible; NanoInteractive/1.0; +https://nanointeractive.com/crawler/)" ], "addition_date": "2026/04/26", "description": "Nano Interactive ad targeting crawler", "tags": [ "advertising" ] }, { "pattern": "NestDaddybot", "url": "https://nestdaddy.com/bot", "instances": [ "NestDaddybot/1.0.0 (+https://nestdaddy.com/bot)" ], "addition_date": "2026/04/26", "description": "NestDaddy search engine crawler", "tags": [ "search-engine" ] }, { "pattern": "Netcraft SSL Server Survey", "url": "https://knownagents.com/agents/netcraft-ssl-server-survey", "instances": [ "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; Netcraft SSL Server Survey - contact info@netcraft.com)" ], "addition_date": "2026/04/26", "description": "Netcraft SSL certificate intelligence scanner", "tags": [ "scanner" ] }, { "pattern": "Netcraft Web Server Survey", "url": "https://netcraft.com/blog/june-2025-web-server-survey", "instances": [ "Mozilla/4.0 (compatible; Netcraft Web Server Survey)" ], "addition_date": "2026/04/26", "description": "Netcraft web server survey crawler", "tags": [ "advertising" ] }, { "pattern": "NetSeer crawler", "url": "http://netseer.com/crawler.html", "instances": [ "Mozilla/5.0 (compatible; NetSeer crawler/2.0; +http://www.netseer.com/crawler.html; crawler@netseer.com)" ], "addition_date": "2026/04/26", "description": "NetSeer intelligence gathering crawler", "tags": [ "advertising" ] }, { "pattern": "Netumo|netumo", "url": "https://docs.netumo.com/", "instances": [ "Netumo/0.2 (+https://www.netumo.app/)" ], "addition_date": "2026/04/26", "description": "Netumo uptime monitoring bot", "tags": [ "monitoring" ] }, { "pattern": "NewRelicSynthetics", "url": "https://knownagents.com/agents/new-relic", "instances": [ "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4896.60 Safari/537.36 NewRelicSynthetics/1.0" ], "addition_date": "2026/04/26", "description": "New Relic synthetic monitoring", "tags": [ "monitoring" ] }, { "pattern": "NewsRoom\\.BI", "url": "http://newsroom.bi/bot.html", "instances": [ "Mozilla/5.0 (compatible; NewsRoom.BI/0.1; +http://www.newsroom.bi/bot.html)" ], "addition_date": "2026/04/26", "description": "Marfeel publisher analytics crawler", "tags": [ "advertising" ] }, { "pattern": "Nitro-", "url": "https://knownagents.com/agents/nitro", "instances": [ "Mozilla/5.0 (compatible; Nitro-; +https://www.nitropack.io/)", "Mozilla/5.0 (Linux; Android 15; GEC77) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Mobile Safari/537.36 Nitro-Optimizer-Agent" ], "addition_date": "2026/04/26", "description": "NitroPack speed optimization crawler", "tags": [ "seo" ] }, { "pattern": "NitroBot", "url": "https://knownagents.com/agents/nitrobot", "instances": [ "Mozilla/5.0 (compatible; NitroBot; +https://www.nitropack.io/)" ], "addition_date": "2026/04/26", "description": "NitroPack SEO optimization bot", "tags": [ "seo" ] }, { "pattern": "Noibu", "url": "https://noibu.com/", "instances": [ "Mozilla/5.0 (compatible; Noibu; +https://www.noibu.com/)" ], "addition_date": "2026/04/26", "description": "Noibu ecommerce error monitoring", "tags": [ "monitoring" ] }, { "pattern": "NostoCrawlerBot", "url": "http://my.nosto.com/tagging", "instances": [ "Mozilla/5.0 (compatible; NostoCrawlerBot/1.0; +http://my.nosto.com/tagging)" ], "addition_date": "2026/04/26", "description": "Nosto ecommerce personalization crawler", "tags": [ "advertising" ] }, { "pattern": "OneTrust", "url": "https://knownagents.com/agents/onetrust-cmp-scanner", "instances": [ "OneTrust" ], "addition_date": "2026/04/26", "description": "OneTrust consent management scanner", "tags": [ "monitoring" ] }, { "pattern": "opencode-smartfetch", "url": "https://opencode.ai/", "instances": [ "opencode-smartfetch/1.0" ], "addition_date": "2026/04/26", "description": "Open-source AI coding agent", "tags": [ "browser-automation" ] }, { "pattern": ";Owler", "url": "https://knownagents.com/agents/owler", "instances": [ "robot;Owler;sthiyagarajan@owler-inc.com" ], "addition_date": "2026/04/26", "description": "Owler competitive intelligence crawler", "tags": [ "advertising" ] }, { "pattern": "ParselySharesBot", "url": "https://docs.parse.ly/", "instances": [ "ParselySharesBot (+http://parsely.com)" ], "addition_date": "2026/04/26", "description": "Parse.ly content analytics crawler", "tags": [ "monitoring" ] }, { "pattern": "PhindBot", "url": "https://knownagents.com/agents/phindbot", "instances": [ "Mozilla/5.0 (compatible; PhindBot; +https://www.phind.com/)" ], "addition_date": "2026/04/26", "description": "AI-powered developer answer engine crawler", "tags": [ "ai-crawler" ] }, { "pattern": "PodchaserParser", "url": "https://podchaser.com/", "instances": [ "PodchaserParser/2.0 (https://podchaser.com)" ], "addition_date": "2026/04/26", "description": "Podchaser podcast database crawler", "tags": [ "feed-reader" ] }, { "pattern": "Podimo", "url": "https://podimo.com/", "instances": [ "Podimo/1.0 (+https://podimo.com/)" ], "addition_date": "2026/04/26", "description": "Podimo podcast streaming platform bot", "tags": [ "feed-reader" ] }, { "pattern": "Poggio-Citations", "url": "https://docs.poggio.io/api/robots", "instances": [ "Poggio-Citations 1.0 (+https://docs.poggio.io/api/robots)" ], "addition_date": "2026/04/26", "description": "Poggio's AI sales enablement data collector", "tags": [ "ai-crawler" ] }, { "pattern": "productsup\\.io\\/crawler", "url": "https://help.productsup.com/en/29437-29446-import-data-by-crawling-your-website.html", "instances": [ "productsup.io/crawler" ], "addition_date": "2026/04/26", "description": "Productsup product feed management crawler", "tags": [ "advertising" ] }, { "pattern": "qcbot", "url": "http://quic.cloud/bot.html", "instances": [ "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; qcbot/1.0; +http://quic.cloud/bot.html) Chrome/112.0.0.0 Safari/537.36" ], "addition_date": "2026/04/26", "description": "QUIC.cloud optimization service crawler", "tags": [ "feed-reader" ] }, { "pattern": "Qualys", "url": "https://knownagents.com/agents/qualys", "instances": [ "Mozilla/5.0 (compatible; Qualys; +https://www.qualys.com/)" ], "addition_date": "2026/04/26", "description": "Qualys security scanning", "tags": [ "monitoring" ] }, { "pattern": "Quora-Bot", "url": "http://quora.com/", "instances": [ "Quora-Bot/1.0 (http://www.quora.com)" ], "addition_date": "2026/04/26", "description": "Quora link preview crawler", "tags": [ "social-preview" ] }, { "pattern": "Qwantbot", "url": "https://help.qwant.com/", "instances": [ "Mozilla/5.0 (compatible; Qwantbot/1.0_800166; +https://help.qwant.com/bot/)", "Mozilla/5.0 (compatible; Qwantbot/1.0_12345; +https://help.qwant.com/bot/)", "Mozilla/5.0 (compatible; Qwantbot-news/2.0; +https://help.qwant.com/bot/)" ], "addition_date": "2026/04/26", "description": "Qwant privacy-focused search crawler", "tags": [ "search-engine" ] }, { "pattern": "Qwarrybot", "url": "http://qwarry.com/bot.html", "instances": [ "Mozilla/5.0 (compatible; Qwarrybot/2.0; +http://www.qwarry.com/bot.html)" ], "addition_date": "2026/04/26", "description": "Qwarry contextual advertising crawler", "tags": [ "advertising" ] }, { "pattern": "RSiteAuditor", "url": "https://dataforseo.com/apis/on-page-api", "instances": [ "Mozilla/5.0 (compatible; RSiteAuditor)" ], "addition_date": "2026/04/26", "description": "DataForSEO on-page audit crawler", "tags": [ "seo" ] }, { "pattern": "RSS\\.Social", "url": "https://rss.social/bot", "instances": [ "RSS.Social/1.0 +https://rss.social/bot" ], "addition_date": "2026/04/26", "description": "RSS.Social feed automation bot", "tags": [ "feed-reader" ] }, { "pattern": "Salesforce\\.com", "url": "https://knownagents.com/agents/sfdc-callout", "instances": [ "Salesforce.com" ], "addition_date": "2026/04/26", "description": "Salesforce HTTP callout user agent", "tags": [ "http-library" ] }, { "pattern": "Scope3", "url": "https://docs.scope3.com/docs/scope3-crawler", "instances": [ "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/133.0.0.0 Safari/537.36 Scope3/2.0 (scope3.com)" ], "addition_date": "2026/04/26", "description": "Scope3 content classification and brand safety", "tags": [ "advertising" ] }, { "pattern": "scraping@nytimes\\.com", "url": "https://github.com/nytimes", "instances": [ "scraping@nytimes.com" ], "addition_date": "2026/04/26", "description": "NYTimes newsroom data collection bot", "tags": [ "advertising" ] }, { "pattern": "Scrubby", "url": "http://scrubtheweb.com/", "instances": [ "Scrubby/2.2 (http://www.scrubtheweb.com/)" ], "addition_date": "2026/04/26", "description": "Scrub the Web business directory crawler", "tags": [ "search-engine" ] }, { "pattern": "Scrunchbot", "url": "https://scrunchai.com/bots", "instances": [ "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Scrunchbot/1.0; +https://scrunchai.com/bots)" ], "addition_date": "2026/04/26", "description": "Scrunch AI brand visibility tracker", "tags": [ "monitoring" ] }, { "pattern": "seo4ajax\\.com", "url": "https://seo4ajax.com/", "instances": [ "seo4ajax.com" ], "addition_date": "2026/04/26", "description": "seo4ajax SPA pre-rendering crawler", "tags": [ "seo" ] }, { "pattern": "SequelWP", "url": "https://sequelwp.com/", "instances": [ "Mozilla/5.0 (compatible; SequelWP; +https://sequelwp.com/)" ], "addition_date": "2026/04/26", "description": "SequelWP hosting uptime monitoring", "tags": [ "monitoring" ] }, { "pattern": "ServerDensity", "url": "https://knownagents.com/agents/server-density", "instances": [ "Mozilla/5.0 (compatible; ServerDensity; +https://www.serverdensity.com/)" ], "addition_date": "2026/04/26", "description": "Server Density infrastructure monitoring", "tags": [ "monitoring" ] }, { "pattern": "ShapBot", "url": "https://docs.parallel.ai/resources/crawler", "instances": [ "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko); compatible; ShapBot/0.1.0" ], "addition_date": "2026/04/26", "description": "Parallel's AI agent web context provider", "tags": [ "ai-crawler" ] }, { "pattern": "ShortPixel", "url": "https://shortpixel.com/", "instances": [ "ShortPixel/1.0 (+https://shortpixel.com)" ], "addition_date": "2026/04/26", "description": "ShortPixel image optimization service", "tags": [ "feed-reader" ] }, { "pattern": "Silktide", "url": "https://silktide.com/", "instances": [ "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/95.0.4638.69 Safari/537.36 Silktide" ], "addition_date": "2026/04/26", "description": "Silktide website quality monitoring", "tags": [ "monitoring" ] }, { "pattern": "SiteLock", "url": "https://knownagents.com/agents/sitelock", "instances": [ "SiteLock (AccMan Connector)/1.0" ], "addition_date": "2026/04/26", "description": "SiteLock security monitoring", "tags": [ "monitoring" ] }, { "pattern": "SmarshBot", "url": "https://smarsh.com/platform/compliance-management/web-archive", "instances": [ "SmarshBot/1.0" ], "addition_date": "2026/04/26", "description": "Smarsh's compliance archiving bot", "tags": [ "archiver" ] }, { "pattern": "SMTnetPMBot", "url": "https://smtnet.com/", "instances": [ "SMTnetPMBot/" ], "addition_date": "2026/04/26", "description": "SMTnet electronics manufacturing search crawler", "tags": [ "search-engine" ] }, { "pattern": "Snapchat[\\s\\S]*panda", "url": "https://developers.snap.com/robots", "instances": [ "Mozilla/5.0 (iPhone; CPU iPhone OS 16_2_2 like Mac OS X) AppleWebKit/526.3.23 (KHTML, like Gecko) Version/16.2 Mobile/15E148 Snapchat/11.13.0.31 (like Safari/526.3, panda)" ], "addition_date": "2026/04/26", "description": "Snapchat link preview fetcher", "tags": [ "social-preview" ] }, { "pattern": "Software-Security-Research", "url": "https://reverse-proxies-measurements.softsec.ruhr-uni-bochum.de/", "instances": [ "Mozilla/5.0 (compatible; Software-Security-Research; +https://www.ruhr-uni-bochum.de/)" ], "addition_date": "2026/04/26", "description": "Ruhr University CDN security research crawler", "tags": [ "scanner", "academic" ] }, { "pattern": "SottopopNone", "url": "https://upcontent.com/robots", "instances": [ "SottopopNone/1.0(+https://upcontent.com/robots)" ], "addition_date": "2026/04/26", "description": "UpContent content curation crawler", "tags": [ "feed-reader" ] }, { "pattern": "Spider[\\s\\S]*spider\\.com", "url": "https://www.spider.com/solutions/web-crawler", "instances": [ "Mozilla/5.0 (compatible; Spider; +https://www.spider.com/)" ], "addition_date": "2026/04/26", "description": "AI project web data crawler", "tags": [ "ai-crawler" ] }, { "pattern": "Splunk", "url": "https://knownagents.com/agents/splunk", "instances": [ "Mozilla/5.0 (compatible; Splunk/1.0.0; https://splunk.com)" ], "addition_date": "2026/04/26", "description": "Splunk monitoring and analytics", "tags": [ "monitoring" ] }, { "pattern": "StatusNestBacklinkSpider", "url": "https://statusnest.com/bot", "instances": [ "Mozilla/5.0 (compatible; StatusNestBacklinkSpider; +https://statusnest.com/)" ], "addition_date": "2026/04/26", "description": "StatusNest backlink tracking crawler", "tags": [ "seo" ] }, { "pattern": "stepstoneCrawlBot", "url": "https://thestepstonegroup.com/crawler/", "instances": [ "Mozilla/5.0 (compatible; stepstoneCrawlBot; +https://www.thestepstonegroup.com/crawler/)" ], "addition_date": "2026/04/26", "description": "Stepstone Group job search crawler", "tags": [ "search-engine" ] }, { "pattern": "TavilyBot", "url": "https://knownagents.com/agents/tavilybot", "instances": [ "Mozilla/5.0 (compatible; TavilyBot; +https://tavily.com/)" ], "addition_date": "2026/04/26", "description": "Tavily's real-time AI agent data crawler", "tags": [ "ai-crawler" ] }, { "pattern": "ThousandEyes", "url": "https://knownagents.com/agents/thousand-eyes-cloud-agent", "instances": [ "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/97.0.4692.71 Safari/537.36 (ThousandEyes Agent)" ], "addition_date": "2026/04/26", "description": "ThousandEyes network monitoring", "tags": [ "monitoring" ] }, { "pattern": "Trae\\/", "url": "https://trae.ai/", "instances": [ "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Trae/1.107.1 Chrome/142.0.7444.235 Electron/39.2.7 Safari/537.36" ], "addition_date": "2026/04/26", "description": "ByteDance's AI coding agent", "tags": [ "browser-automation" ] }, { "pattern": "TwinAgent", "url": "https://twin.so/", "instances": [ "Mozilla/5.0 (compatible; TwinAgent; +https://www.twinagent.com/)" ], "addition_date": "2026/04/26", "description": "Twin's automated worker for workflow execution", "tags": [ "browser-automation" ] }, { "pattern": "uipbot", "url": "https://knownagents.com/agents/uipbot", "instances": [ "uipbot/1.0 (uipbot@semasio.com)" ], "addition_date": "2026/04/26", "description": "Semasio semantic targeting crawler", "tags": [ "advertising" ] }, { "pattern": "um-FC", "url": "https://ubermetrics-technologies.com/", "instances": [ "Mozilla/5.0 (compatible; um-FC/1.0; https://www.ubermetrics-technologies.com/; Windows NT 6.1; WOW64; rv:125.0) Gecko/20100101 Firefox/125.1" ], "addition_date": "2026/04/26", "description": "Ubermetrics media monitoring crawler", "tags": [ "advertising" ] }, { "pattern": "um-IC", "url": "https://ubermetrics-technologies.com/", "instances": [ "Mozilla/5.0 (compatible; um-IC/1.0; https://www.ubermetrics-technologies.com/; Windows NT 6.1; WOW64; rv:125.0) Gecko/20100101 Firefox/125.1" ], "addition_date": "2026/04/26", "description": "Ubermetrics communications intelligence crawler", "tags": [ "advertising" ] }, { "pattern": "UptimeStatistics", "url": "https://knownagents.com/agents/uptimestatistics", "instances": [ "Mozilla/5.0 (compatible; UptimeStatistics; +https://www.uptimestatistics.com/)" ], "addition_date": "2026/04/26", "description": "UptimeStatistics monitoring service", "tags": [ "monitoring" ] }, { "pattern": "Verispider", "url": "http://projecthoneypot.org/", "instances": [ "Mozilla/5.0 (compatible; Verispider; +https://www.projecthoneypot.org/)" ], "addition_date": "2026/04/26", "description": "Verispider spam and security scanner", "tags": [ "scanner" ] }, { "pattern": "visionheight\\.com\\/scan", "url": "https://knownagents.com/agents/visionheight-comscan", "instances": [ "visionheight.com/scan Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) Chrome/126.0.0.0 Safari/537.36" ], "addition_date": "2026/04/26", "description": "Vision Height security scanner", "tags": [ "scanner" ] }, { "pattern": "Watchbot monitoring robot", "url": "https://watchbot.fflow.net/", "instances": [ "Watchbot monitoring robot (https://watchbot.fflow.net)" ], "addition_date": "2026/04/26", "description": "Watchbot SSL and uptime monitoring", "tags": [ "monitoring" ] }, { "pattern": "Watchful", "url": "https://knownagents.com/agents/watchful", "instances": [ "Mozilla/5.0 (compatible; Watchful; +https://www.watchful.net/)" ], "addition_date": "2026/04/26", "description": "Watchful website monitoring", "tags": [ "monitoring" ] }, { "pattern": "weborama-fetcher", "url": "http://weborama.com/", "instances": [ "weborama-fetcher (+http://www.weborama.com)" ], "addition_date": "2026/04/26", "description": "Weborama advertising optimization crawler", "tags": [ "advertising" ] }, { "pattern": "webspidermount", "url": "https://webspidermount.com/features/", "instances": [ "Mozilla/5.0 (compatible; webspidermount; +https://www.webspidermount.com/)" ], "addition_date": "2026/04/26", "description": "Webspidermount job listing extraction service", "tags": [ "advertising" ] }, { "pattern": "WepchSearchEngine", "url": "https://wepch.com/search-engine", "instances": [ "Mozilla/5.0 (compatible; WepchSearchEngine; +https://www.wepch.com/)" ], "addition_date": "2026/04/26", "description": "Wepch search engine development crawler", "tags": [ "search-engine" ] }, { "pattern": "wknd-bot", "url": "https://developer.wunderkind.co/docs/server-side-tracking-implementation", "instances": [ "wknd-bot" ], "addition_date": "2026/04/26", "description": "Wunderkind marketing automation tracker", "tags": [ "advertising" ] }, { "pattern": "WPMU DEV Hub", "url": "https://wpmudev.com/", "instances": [ "Mozilla/5.0 (compatible; WPMU DEV Hub/2.0; +https://wpmudev.com)" ], "addition_date": "2026/04/26", "description": "WPMU DEV WordPress management bot", "tags": [ "monitoring" ] }, { "pattern": "WTotem", "url": "https://knownagents.com/agents/wtotem", "instances": [ "Mozilla/5.0 (compatible; WTotem; +https://www.webtotem.com/)" ], "addition_date": "2026/04/26", "description": "WTotem malware detection scanner", "tags": [ "scanner" ] }, { "pattern": "XoviOnpageCrawler", "url": "http://xovi.de/", "instances": [ "Mozilla/5.0 (compatible; XoviOnpageCrawler; +http://www.xovi.de/)" ], "addition_date": "2026/04/26", "description": "XOVI SEO suite on-page crawler", "tags": [ "seo" ] }, { "pattern": "yelpspider", "url": "https://yelp.com/", "instances": [ "Mozilla/5.0 compatible; yelpspider/yelpspider-1.0 (Crawlerbot run by Yelp Inc; yelpbot at yelp dot com)" ], "addition_date": "2026/04/26", "description": "Yelp business information crawler", "tags": [ "feed-reader" ] }, { "pattern": "ZanistaBot", "url": "https://zanista.ai/crawler-info", "instances": [ "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; ZanistaBot/1.0; +https://zanista.ai/crawler-info) Chrome/W.X.Y.Z Safari/537.36" ], "addition_date": "2026/04/26", "description": "Zanista's AI search crawler", "tags": [ "ai-crawler" ] }, { "pattern": "ZoomInfo-", "url": "https://knownagents.com/agents/zoominfo", "instances": [ "Mozilla/5.0 (compatible; ZoomInfo-DataEnrichment/1.0; +https://www.zoominfo.com)" ], "addition_date": "2026/04/26", "description": "ZoomInfo business intelligence search crawler", "tags": [ "search-engine" ] }, { "pattern": "7Siters", "url": "https://7ooo.ru/siters/", "instances": [ "7Siters/1.2 ( https://7ooo.ru/siters/ )" ], "addition_date": "2026/04/26", "description": "Russian web crawler for site indexing and analysis", "tags": [ "search-engine" ] }, { "pattern": "Accessible Web Bot", "url": "https://accessibleweb.com/bot/", "instances": [ "Accessible Web Bot" ], "addition_date": "2026/04/26", "description": "Monitors websites for WCAG accessibility compliance violations", "tags": [ "monitoring" ] }, { "pattern": "AtVowBot", "url": "https://brandeem.com/", "instances": [ "AtVowBot" ], "addition_date": "2026/04/26", "description": "Brand monitoring and reputation management web crawler", "tags": [ "monitoring" ] }, { "pattern": "Bibliotheque Nacional de France Crawler", "url": "https://www.bnf.fr/en/web-legal-deposit", "instances": [ "Bibliotheque Nacional de France Crawler" ], "addition_date": "2026/04/26", "description": "French National Library web archiving and preservation bot", "tags": [ "archiver" ] }, { "pattern": "Bling ERP", "url": "https://www.bling.com.br/", "instances": [ "Bling ERP" ], "addition_date": "2026/04/26", "description": "Brazilian ERP system data synchronization and integration crawler", "tags": [ "monitoring" ] }, { "pattern": "CDSCbot", "url": "https://wiki.communitydata.science/CommunityData:Fediverse_research", "instances": [ "CDSCbot/2024.08.08 https://wiki.communitydata.science/CommunityData:Fediverse_research" ], "addition_date": "2026/04/26", "description": "Academic research crawler for Fediverse decentralized social networks", "tags": [ "academic" ] }, { "pattern": "Critical CSS Bot", "url": "https://criticalcss.com/", "instances": [ "Critical CSS Bot" ], "addition_date": "2026/04/26", "description": "Extracts critical CSS for web performance optimization tools", "tags": [ "monitoring" ] }, { "pattern": "CybaaBot", "url": "https://cybaa.io/bot-policy", "instances": [ "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/140.0.0.0 Safari/537.36 CybaaBot" ], "addition_date": "2026/04/26", "description": "Cybaa advertising and marketing analytics web crawler bot", "tags": [ "advertising" ] }, { "pattern": "CyberFindCrawler", "url": "https://cyberfind.net/bot.html", "instances": [ "CyberFindCrawler" ], "addition_date": "2026/04/26", "description": "Web search engine crawler for content discovery indexing", "tags": [ "search-engine" ] }, { "pattern": "Dark Visitor", "url": "https://darkvisitors.com/", "instances": [ "Mozilla/5.0 (compatible; Dark Visitor/1.0; +https://darkvisitors.com)" ], "addition_date": "2026/04/26", "description": "Bot analytics and AI agent tracking service crawler", "tags": [ "monitoring" ] }, { "pattern": "Determ", "url": "https://www.determ.com/", "instances": [ "Determ" ], "addition_date": "2026/04/26", "description": "Media monitoring and brand reputation tracking web crawler", "tags": [ "monitoring" ] }, { "pattern": "DNSScanner", "url": "https://rapef.info/_contacts/", "instances": [ "DNSScanner" ], "addition_date": "2026/04/26", "description": "DNS and network infrastructure security scanning crawler bot", "tags": [ "scanner" ] }, { "pattern": "Drupalbot", "url": "https://www.drupal.org/", "instances": [ "Mozilla/5.0 (compatible; Drupalbot; +https://www.drupal.org)" ], "addition_date": "2026/04/26", "description": "Drupal CMS community website crawler and content indexer", "tags": [ "monitoring" ] }, { "pattern": "eMoney Advisor", "url": "https://emoneyadvisor.com/", "instances": [ "eMoney Advisor" ], "addition_date": "2026/04/26", "description": "Financial planning software data integration and sync crawler", "tags": [ "monitoring" ] }, { "pattern": "everyfeed-spider", "url": "http://everyfeed.com/", "instances": [ "everyfeed-spider/2.0 (http://www.everyfeed.com)" ], "addition_date": "2026/04/26", "description": "RSS and Atom feed aggregation and discovery crawler", "tags": [ "feed-reader" ] }, { "pattern": "ExteContextCrawl", "url": "http://crawl001.exte.ai/", "instances": [ "Mozilla/5.0 (compatible; ExteContextCrawl/1.0; +http://crawl001.exte.ai)" ], "addition_date": "2026/04/26", "description": "AI context extraction and content understanding web crawler", "tags": [ "ai-crawler" ] }, { "pattern": "FediDB", "url": "https://fedidb.org/crawler.html", "instances": [ "FediDB/0.5.0; +https://fedidb.org/crawler.html" ], "addition_date": "2026/04/26", "description": "Fediverse instance database and statistics collection crawler bot", "tags": [ "monitoring" ] }, { "pattern": "FediIndex", "url": "https://fedi.wrm.sr/about", "instances": [ "FediIndex/1.0 (+https://fedi.wrm.sr/about)" ], "addition_date": "2026/04/26", "description": "Fediverse server indexing and discovery service web crawler", "tags": [ "search-engine" ] }, { "pattern": "FediList Agent", "url": "https://fedilist.com/", "instances": [ "FediList Agent/3 (https://fedilist.com/)" ], "addition_date": "2026/04/26", "description": "Fediverse instance directory and listing service crawler bot", "tags": [ "monitoring" ] }, { "pattern": "Fedineko", "url": "https://fedineko.org/about", "instances": [ "Fedineko (crabo/0.3.1; +https://fedineko.org/about)" ], "addition_date": "2026/04/26", "description": "Fediverse network mapping and analysis research crawler tool", "tags": [ "academic" ] }, { "pattern": "FedReporter Bot for FFIEC", "url": "https://www.fedreporter.com/", "instances": [ "FedReporter Bot for FFIEC" ], "addition_date": "2026/04/26", "description": "Federal financial institution regulatory compliance data crawler bot", "tags": [ "monitoring" ] }, { "pattern": "Feedsearch Bot", "url": "https://feedearch.dev/", "instances": [ "Mozilla/5.0 (Compatible; Feedsearch Bot; +https://feedearch.dev)" ], "addition_date": "2026/04/26", "description": "RSS and feed discovery search engine crawler bot", "tags": [ "feed-reader" ] }, { "pattern": "Feedsearch-Crawler", "url": "https://pypi.org/project/feedsearch-crawler", "instances": [ "Mozilla/5.0 (compatible; Feedsearch-Crawler; +https://pypi.org/project/feedsearch-crawler)" ], "addition_date": "2026/04/26", "description": "Python library for RSS feed detection and extraction", "tags": [ "feed-reader" ] }, { "pattern": "fiperbot", "url": "https://fiper.net/", "instances": [ "Mozilla/5.0 (compatible; fiperbot/0.1 +https://www.fiper.net/bot.html)" ], "addition_date": "2026/04/26", "description": "Web content indexing and search engine crawler bot", "tags": [ "search-engine" ] }, { "pattern": "FleebsBot", "url": "https://fleebs.com/bot", "instances": [ "Mozilla/5.0 (compatible; FleebsBot/1~w; +https://fleebs.com/bot)" ], "addition_date": "2026/04/26", "description": "Web content aggregation and discovery service crawler bot", "tags": [ "search-engine" ] }, { "pattern": "Fluid", "url": "http://leak.info/bot.html", "instances": [ "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_5_6; en-us) AppleWebKit/528.16 (KHTML, like Gecko) Fluid/0.9.6 Safari/528.16" ], "addition_date": "2026/04/26", "description": "Site-specific browser application for web content access", "tags": [ "browser-automation" ] }, { "pattern": "Flyriverbot", "url": "https://flyriver.com/crawler", "instances": [ "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36 Flyriverbot/1.1 (+https://www.flyriver.com/; AI Content Source Check)" ], "addition_date": "2026/04/26", "description": "AI content source verification and attribution checking crawler", "tags": [ "ai-crawler" ] }, { "pattern": "Freshbot", "url": "http://webagent.wise-guys.nl/", "instances": [ "Mozilla/5.0 (X11; Linux i686; rv:6.0) Gecko/20100101 Firefox/6.0 (Freshbot/1.0/EU; http://webagent.wise-guys.nl/)" ], "addition_date": "2026/04/26", "description": "European web content monitoring and freshness checking crawler", "tags": [ "monitoring" ] }, { "pattern": "Gaisbot", "url": "http://gais.cs.ccu.edu.tw/robot.php", "instances": [ "Gaisbot/3.0 (robot@gais.cs.ccu.edu.tw; http://gais.cs.ccu.edu.tw/robot.php)" ], "addition_date": "2026/04/26", "description": "Academic research web crawler from Taiwan university project", "tags": [ "academic" ] }, { "pattern": "GenomeCrawlerd", "url": "https://nokia.com/genomecrawler", "instances": [ "Mozilla/5.0 (compatible; GenomeCrawlerd/1.0; +https://www.nokia.com/genomecrawler)" ], "addition_date": "2026/04/26", "description": "Nokia web content analysis and data collection crawler", "tags": [ "monitoring" ] }, { "pattern": "HaloBot", "url": "https://haloscan.com/", "instances": [ "Mozilla/5.0 (compatible; HaloBot/1.0)" ], "addition_date": "2026/04/26", "description": "Haloscan comment system integration and content fetching bot", "tags": [ "monitoring" ] }, { "pattern": "IRLbot", "url": "http://irl.cs.tamu.edu/crawler", "instances": [ "IRLbot/3.0 (compatible; MSIE 6.0; http://irl.cs.tamu.edu/crawler)" ], "addition_date": "2026/04/26", "description": "Texas A&M University academic research web crawler bot", "tags": [ "academic" ] }, { "pattern": "kaikki\\.org-digital-archive", "url": "https://kaikki.org/", "instances": [ "kaikki.org-digital-archive" ], "addition_date": "2026/04/26", "description": "Digital archiving and preservation service web crawler bot", "tags": [ "archiver" ] }, { "pattern": "kb\\.dk_bot", "url": "https://www.kb.dk/en/", "instances": [ "kb.dk_bot" ], "addition_date": "2026/04/26", "description": "Royal Danish Library web archiving and preservation crawler", "tags": [ "archiver" ] }, { "pattern": "Library Of Congress Web Archiving", "url": "https://www.loc.gov/programs/web-archiving/", "instances": [ "Library Of Congress Web Archiving" ], "addition_date": "2026/04/26", "description": "US Library of Congress web preservation archiving crawler", "tags": [ "archiver" ] }, { "pattern": "MagnetmeBot", "url": "https://magnet.me/", "instances": [ "MagnetmeBot" ], "addition_date": "2026/04/26", "description": "Job board and recruitment platform web scraping crawler", "tags": [ "monitoring" ] }, { "pattern": "MatchorySearch", "url": "https://matchory.com/", "instances": [ "Mozilla/5.0 (compatible; MatchorySearch/1.3; +https://www.matchory.com)" ], "addition_date": "2026/04/26", "description": "Search engine and content discovery indexing crawler bot", "tags": [ "search-engine" ] }, { "pattern": "Minoru's Fediverse Crawler", "url": "https://nodes.fediverse.party/", "instances": [ "Minoru's Fediverse Crawler (+https://nodes.fediverse.party)" ], "addition_date": "2026/04/26", "description": "Fediverse network node discovery and mapping crawler tool", "tags": [ "monitoring" ] }, { "pattern": "MirrorWebCrawler", "url": "https://www.mirrorweb.com/", "instances": [ "MirrorWebCrawler" ], "addition_date": "2026/04/26", "description": "Enterprise web archiving and compliance preservation crawler bot", "tags": [ "archiver" ] }, { "pattern": "mithril-crawler", "url": "https://498-search-engine.github.io/website/", "instances": [ "mithril-crawler" ], "addition_date": "2026/04/26", "description": "Academic search engine project web indexing crawler bot", "tags": [ "academic" ] }, { "pattern": "ModatScanner", "url": "https://modat.io/", "instances": [ "Mozilla/5.0 (compatible; ModatScanner/1.2; +https://modat.io/)" ], "addition_date": "2026/04/26", "description": "Website monitoring and change detection service crawler bot", "tags": [ "monitoring" ] }, { "pattern": "NapBot", "url": "http://napbot.com/", "instances": [ "NapBot" ], "addition_date": "2026/04/26", "description": "Web content indexing and search engine crawler bot", "tags": [ "search-engine" ] }, { "pattern": "New York Times Newsgathering", "url": "https://www.nytimes.com/", "instances": [ "New York Times Newsgathering" ], "addition_date": "2026/04/26", "description": "News media content aggregation and research crawler bot", "tags": [ "monitoring" ] }, { "pattern": "NLUX_IAHarvester", "url": "http://crawl.bnl.lu/", "instances": [ "Mozilla/5.0 (compatible; NLUX_IAHarvester/3.4.0; +http://crawl.bnl.lu/)" ], "addition_date": "2026/04/26", "description": "Luxembourg National Library web archiving and harvesting crawler", "tags": [ "archiver" ] }, { "pattern": "NoahBot", "url": "https://noahwire.com/bot-info", "instances": [ "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; NoahBot/1.0; +https://noahwire.com/bot-info)" ], "addition_date": "2026/04/26", "description": "News aggregation and content discovery service crawler bot", "tags": [ "monitoring" ] }, { "pattern": "PlagAwareBot", "url": "https://plagaware.com/bot", "instances": [ "Mozilla/5.0 (compatible; PlagAwareBot/3.2; +https://www.plagaware.com/bot)" ], "addition_date": "2026/04/26", "description": "Plagiarism detection and content originality checking crawler bot", "tags": [ "monitoring" ] }, { "pattern": "Rakuten Image extraction bot", "url": "https://www.rakuten.com/", "instances": [ "Rakuten Image extraction bot" ], "addition_date": "2026/04/26", "description": "E-commerce product image extraction and indexing crawler bot", "tags": [ "monitoring" ] }, { "pattern": "ResearchBot", "url": "https://kaust.edu.sa/bot", "instances": [ "StateOfPlay-ResearchBot/1.0" ], "addition_date": "2026/04/26", "description": "Academic research data collection and analysis crawler bot", "tags": [ "academic" ] }, { "pattern": "rss-is-dead\\.lol web bot", "url": "https://rss-is-dead.lol/", "instances": [ "Mozilla/5.0 (compatible; rss-is-dead.lol web bot; +https://rss-is-dead.lol)" ], "addition_date": "2026/04/26", "description": "RSS feed monitoring and availability checking service crawler", "tags": [ "feed-reader" ] }, { "pattern": "seoLyt", "url": "https://seolyt.com/", "instances": [ "Mozilla/5.0 (compatible; seoLyt/1.0; +https://seolyt.com)" ], "addition_date": "2026/04/26", "description": "SEO analysis and website optimization tool crawler bot", "tags": [ "seo" ] }, { "pattern": "SirdataBot", "url": "https://semantic-api.docs.sirdata.net/contextual-api/contextual-api/introduction", "instances": [ "SirdataBot (+https://semantic-api.docs.sirdata.net/contextual-api/contextual-api/introduction)" ], "addition_date": "2026/04/26", "description": "Contextual advertising and semantic content analysis crawler bot", "tags": [ "advertising" ] }, { "pattern": "SitesOverPagesBot", "url": "https://sitesoverpages.com/bot", "instances": [ "SitesOverPagesBot" ], "addition_date": "2026/04/26", "description": "Website structure analysis and sitemap generation crawler bot", "tags": [ "seo" ] }, { "pattern": "SleepBot", "url": "http://sleepbot.com/", "instances": [ "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; SleepBot/1.0; +http://sleepbot.com/) Chrome/131.0.0.0 Safari/537.36" ], "addition_date": "2026/04/26", "description": "Web content monitoring and tracking service crawler bot", "tags": [ "monitoring" ] }, { "pattern": "Sosospider", "url": "http://help.soso.com/webspider.htm", "instances": [ "Sosospider" ], "addition_date": "2026/04/26", "description": "Chinese search engine web indexing and crawling bot", "tags": [ "search-engine" ] }, { "pattern": "Termly", "url": "https://termly.io/", "instances": [ "Termly" ], "addition_date": "2026/04/26", "description": "Privacy policy and compliance scanning service crawler bot", "tags": [ "monitoring" ] }, { "pattern": "TLS tester", "url": "https://testssl.sh/dev/", "instances": [ "TLS tester from https://testssl.sh/" ], "addition_date": "2026/04/26", "description": "SSL/TLS security testing and vulnerability scanning crawler bot", "tags": [ "scanner" ] }, { "pattern": "trafilatura", "url": "https://github.com/adbar/trafilatura", "instances": [ "trafilatura/2.0.0 (+https://github.com/adbar/trafilatura)" ], "addition_date": "2026/04/26", "description": "Python library for web content extraction and scraping", "tags": [ "http-library" ] }, { "pattern": "UrlBeeBot", "url": "https://urlbee.com/", "instances": [ "UrlBeeBot" ], "addition_date": "2026/04/26", "description": "URL analysis and threat detection security crawler bot", "tags": [ "scanner" ] }, { "pattern": "videootv Bot", "url": "https://www.digitalgreen.org/", "instances": [ "videootv Bot" ], "addition_date": "2026/04/26", "description": "Video content aggregation and educational media crawler bot", "tags": [ "monitoring" ] }, { "pattern": "vmcrawl", "url": "https://docs.vmst.io/vmcrawl", "instances": [ "vmcrawl/0.17.7 (https://vmcrawl.com)", "vmcrawl/0.x (https://docs.vmst.io/vmcrawl)" ], "addition_date": "2026/04/26", "description": "Fediverse instance monitoring and discovery service crawler bot", "tags": [ "monitoring" ] }, { "pattern": "WadooBot", "url": "https://wadoo.net/wadoobot/", "instances": [ "WadooBot" ], "addition_date": "2026/04/26", "description": "Web content indexing and search engine crawler bot", "tags": [ "search-engine" ] }, { "pattern": "Website-info\\.net-Robot", "url": "https://website-info.net/robot", "instances": [ "Mozilla/5.0 (compatible; Website-info.net-Robot; https://website-info.net/robot)" ], "addition_date": "2026/04/26", "description": "Website information and analytics data collection crawler bot", "tags": [ "seo" ] }, { "pattern": "WebZIP", "url": "http://spidersoft.com/", "instances": [ "WebZIP/3.5 (http://www.spidersoft.com)" ], "addition_date": "2026/04/26", "description": "Website downloading and offline browsing tool crawler bot", "tags": [ "archiver" ] }, { "pattern": "WikiDo", "url": "http://wikido.com/", "instances": [ "WikiDo/1.1 (http://wikido.com; crawler@wikido.com)" ], "addition_date": "2026/04/26", "description": "Wiki content aggregation and search engine crawler bot", "tags": [ "search-engine" ] }, { "pattern": "WOVN Crawler", "url": "https://wovn.io/", "instances": [ "WOVN Crawler" ], "addition_date": "2026/04/26", "description": "Website translation and localization service crawler bot", "tags": [ "monitoring" ] }, { "pattern": "YoudaoBot", "url": "http://youdao.com/help/webmaster/spider/", "instances": [ "YoudaoBot" ], "addition_date": "2026/04/26", "description": "Chinese search engine and translation service crawler bot", "tags": [ "search-engine" ] }, { "pattern": "ZyBorg", "url": "http://wisenutbot.com/", "instances": [ "Mozilla/4.0 compatible ZyBorg/1.0 (wn-14.zyborg@looksmart.net; http://www.WISEnutbot.com)" ], "addition_date": "2026/04/26", "description": "WISEnut search engine web indexing and crawling bot", "tags": [ "search-engine" ] }, { "pattern": "Aranet-SearchBot", "url": "https://aranet.ai/bot", "instances": [ "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Aranet-SearchBot/1.0; +https://aranet.ai/bot) Chrome/131.0.0.0 Safari/537.36" ], "addition_date": "2026/04/26", "description": "AI-powered search engine and content indexing crawler bot", "tags": [ "ai-crawler" ] }, { "pattern": "crawl4ai", "url": "https://github.com/unclecode/crawl4ai", "instances": [ "crawl4ai-adapter/1.0" ], "addition_date": "2026/04/26", "description": "Open-source LLM-friendly web scraping and extraction library", "tags": [ "ai-crawler" ] }, { "pattern": "DeepSeekBot", "url": "http://deepseek.com/bot", "instances": [ "Mozilla/5.0 (compatible; DeepSeekBot/1.0; +https://www.deepseek.com/bot)" ], "addition_date": "2026/04/26", "description": "DeepSeek AI model training and data collection crawler", "tags": [ "ai-crawler" ] }, { "pattern": "iaskspider", "url": "https://www.iask.com/", "instances": [ "iaskspider/2.0" ], "addition_date": "2026/04/26", "description": "Chinese AI search engine and question answering crawler", "tags": [ "ai-crawler" ] }, { "pattern": "KunatoCrawler", "url": "http://kunato.ai/bot.html", "instances": [ "Mozilla/5.0 (compatible; KunatoCrawler/1.0; +http://kunato.ai/bot.html)" ], "addition_date": "2026/04/26", "description": "AI training data collection and web scraping crawler", "tags": [ "ai-crawler" ] }, { "pattern": "TerraCotta", "url": "https://github.com/CeramicTeam/CeramicTerracotta", "instances": [ "TerraCotta https://github.com/CeramicTeam/CeramicTerracotta" ], "addition_date": "2026/04/26", "description": "Ceramic Network decentralized data indexing and crawling bot", "tags": [ "ai-crawler" ] }, { "pattern": "ABEvalBot", "url": "https://knownagents.com/agents/abevalbot", "instances": [ "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; ABEvalBot/0.1) Version/11.1.2 Safari/605.1.15" ], "addition_date": "2026/04/26", "description": "A/B testing evaluation and website analysis crawler bot", "tags": [ "monitoring" ] }, { "pattern": "blekkobot", "url": "https://knownagents.com/agents/blekkobot", "instances": [ "blekkobot" ], "addition_date": "2026/04/26", "description": "Blekko search engine web indexing and crawling bot", "tags": [ "search-engine" ] }, { "pattern": "br-crawler", "url": "https://knownagents.com/agents/br-crawler", "instances": [ "br-crawler/0.5" ], "addition_date": "2026/04/26", "description": "Brazilian web content indexing and discovery crawler bot", "tags": [ "search-engine" ] }, { "pattern": "BuddyBot", "url": "https://knownagents.com/agents/buddybot", "instances": [ "BuddyBot" ], "addition_date": "2026/04/26", "description": "Social networking and content aggregation service crawler bot", "tags": [ "monitoring" ] }, { "pattern": "CapterraBot", "url": "https://www.capterra.com", "instances": [ "Mozilla/5.0 (compatible; CapterraBot/1.0; +https://www.capterra.com)" ], "addition_date": "2026/04/26", "description": "Capterra software review platform data collection crawler bot", "tags": [ "monitoring" ] }, { "pattern": "carbon-umbrella-bot", "url": "https://knownagents.com/agents/carbon-umbrella-bot", "instances": [ "carbon-umbrella-bot/1.0" ], "addition_date": "2026/04/26", "description": "Carbon footprint tracking and environmental monitoring crawler bot", "tags": [ "monitoring" ] }, { "pattern": "caveman-hunter", "url": "https://fedi.buzz/", "instances": [ "caveman-hunter/0.0.0 (+https://fedi.buzz/)" ], "addition_date": "2026/04/26", "description": "Fediverse content discovery and social network crawler bot", "tags": [ "monitoring" ] }, { "pattern": "Centro Ads\\.txt Crawler", "url": "https://knownagents.com/agents/centro-ads-txt-crawler", "instances": [ "Centro Ads.txt Crawler/1.0" ], "addition_date": "2026/04/26", "description": "Advertising transparency ads.txt file verification crawler bot", "tags": [ "advertising" ] }, { "pattern": "WISEbot", "url": "http://www.cision.com", "instances": [ "Mozilla/5.0 (compatible; WISEbot/1.0; +http://www.cision.com)" ], "addition_date": "2026/04/26", "description": "Cision media monitoring and PR analytics crawler bot", "tags": [ "monitoring" ] }, { "pattern": "CodaBot", "url": "https://coda.io/", "instances": [ "CodaBot/1.0" ], "addition_date": "2026/04/26", "description": "Coda document collaboration platform link preview crawler bot", "tags": [ "social-preview" ] }, { "pattern": "Corporama matcher", "url": "https://corporama.fr/", "instances": [ "Corporama matcher 1.2" ], "addition_date": "2026/04/26", "description": "Corporate data matching and business intelligence crawler bot", "tags": [ "monitoring" ] }, { "pattern": "CyotekWebCopy", "url": "https://www.cyotek.com/cyotek-webcopy", "instances": [ "CyotekWebCopy/1.9 CyotekHTTP/6.4" ], "addition_date": "2026/04/26", "description": "Website copying and offline browsing tool crawler bot", "tags": [ "archiver" ] }, { "pattern": "Datadog Agent", "url": "https://www.datadoghq.com/", "instances": [ "Datadog Agent/7.69.2" ], "addition_date": "2026/04/26", "description": "Datadog infrastructure monitoring and synthetic testing agent bot", "tags": [ "monitoring" ] }, { "pattern": "Dazzle BlueSky Bot", "url": "https://knownagents.com/agents/dazzle-bluesky-bot", "instances": [ "Dazzle BlueSky Bot/1.1" ], "addition_date": "2026/04/26", "description": "BlueSky social network link preview and content crawler", "tags": [ "social-preview" ] }, { "pattern": "DominicBot", "url": "https://vanylla.org/bot", "instances": [ "DominicBot/1.0 (+https://vanylla.org/bot)" ], "addition_date": "2026/04/26", "description": "Vanylla social network content indexing and discovery crawler", "tags": [ "monitoring" ] }, { "pattern": "Dow Jones Searchbot", "url": "https://www.dowjones.com/", "instances": [ "Mozilla/5.0 (compatible; Dow Jones Searchbot)" ], "addition_date": "2026/04/26", "description": "Dow Jones news and financial content aggregation crawler", "tags": [ "monitoring" ] }, { "pattern": "Download Ninja", "url": "https://knownagents.com/agents/download-ninja", "instances": [ "Download Ninja/4.0" ], "addition_date": "2026/04/26", "description": "File download manager and web content fetching tool", "tags": [ "http-library" ] }, { "pattern": "EmailWolf", "url": "https://knownagents.com/agents/emailwolf", "instances": [ "EmailWolf 1.00" ], "addition_date": "2026/04/26", "description": "Email address extraction and contact scraping crawler bot", "tags": [ "scanner" ] }, { "pattern": "fedistatsCrawler", "url": "https://knownagents.com/agents/fedistatscrawler", "instances": [ "fedistatsCrawler/1.0" ], "addition_date": "2026/04/26", "description": "Fediverse statistics collection and network analysis crawler bot", "tags": [ "monitoring" ] }, { "pattern": "GoParserBot", "url": "https://knownagents.com/agents/goparserbot", "instances": [ "Mozilla/5.0 (compatible; GoParserBot/1.0)" ], "addition_date": "2026/04/26", "description": "Go-based web content parsing and extraction crawler bot", "tags": [ "http-library" ] }, { "pattern": "gsa-crawler", "url": "https://knownagents.com/agents/gsa-crawler", "instances": [ "gsa-crawler (Enterprise; T3-LNMUGC6JBKQNE; thomas@brainfuck.space)" ], "addition_date": "2026/04/26", "description": "Google Search Appliance enterprise search indexing crawler bot", "tags": [ "search-engine" ] }, { "pattern": "HanaleiBot", "url": "https://knownagents.com/agents/hanaleibot", "instances": [ "HanaleiBot runid=beta-stage-integration-test" ], "addition_date": "2026/04/26", "description": "Testing and integration verification web crawler bot", "tags": [ "monitoring" ] }, { "pattern": "NicheIndex", "url": "https://nicheindex.co", "instances": [ "NicheIndex/1.0 RSS Harvest (+https://nicheindex.co)" ], "addition_date": "2026/04/26", "description": "Niche market RSS feed harvesting and indexing crawler", "tags": [ "feed-reader" ] }, { "pattern": "HeadOnlyScraper", "url": "https://knownagents.com/agents/headonlyscraper", "instances": [ "Mozilla/5.0 (HeadOnlyScraper)", "Mozilla/5.0 (HeadOnlyScraperV2)" ], "addition_date": "2026/04/26", "description": "HTTP HEAD request metadata collection and validation crawler", "tags": [ "monitoring" ] }, { "pattern": "HenkBot", "url": "https://valyu.ai/crawler", "instances": [ "Mozilla/5.0 (compatible; HenkBot/1.0; +https://valyu.ai/crawler)" ], "addition_date": "2026/04/26", "description": "Valyu AI content analysis and data collection crawler", "tags": [ "ai-crawler" ] }, { "pattern": "Impact\\.com Agent", "url": "https://impact.com/", "instances": [ "Mozilla/5.0 (compatible;Impact.com Agent) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/129.0.0.0 Safari/537.36" ], "addition_date": "2026/04/26", "description": "Impact.com affiliate marketing and partnership tracking crawler bot", "tags": [ "advertising" ] }, { "pattern": "Keydrop\\.io", "url": "https://onlyscans.com/about", "instances": [ "Mozilla/5.0; Keydrop.io/1.0(onlyscans.com/about);" ], "addition_date": "2026/04/26", "description": "Keydrop gaming marketplace security scanning and monitoring crawler", "tags": [ "scanner" ] }, { "pattern": "larbin", "url": "http://larbin.sourceforge.net/", "instances": [ "larbin_2.6.2 (larbin@correa.org)" ], "addition_date": "2026/04/26", "description": "Open-source web crawler for large-scale indexing projects", "tags": [ "http-library" ] }, { "pattern": "SENTINEL-LinkCheck", "url": "https://sentinel.oblivionzone.com/bot", "instances": [ "SENTINEL-LinkCheck/1.0 (+https://sentinel.oblivionzone.com/bot)" ], "addition_date": "2026/04/26", "description": "Sentinel link validation and broken link detection crawler", "tags": [ "monitoring" ] }, { "pattern": "linko", "url": "https://linko.app/crawler", "instances": [ "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.120 Safari/537.36 https://linko.app/crawler" ], "addition_date": "2026/04/26", "description": "Linko link management and URL tracking service crawler", "tags": [ "monitoring" ] }, { "pattern": "LinkpadBot", "url": "https://linkpad.org/robot/", "instances": [ "Mozilla/5.0 (compatible; LinkpadBot/2.4; +https://linkpad.org/robot/)" ], "addition_date": "2026/04/26", "description": "Linkpad bookmark management and link discovery crawler bot", "tags": [ "monitoring" ] }, { "pattern": "lwp-trivial", "url": "https://metacpan.org/pod/LWP", "instances": [ "lwp-trivial/1.35" ], "addition_date": "2026/04/26", "description": "Perl LWP library simple HTTP client and fetcher", "tags": [ "http-library" ] }, { "pattern": "Magus Bot", "url": "https://knownagents.com/agents/magus-bot", "instances": [ "Magus Bot 1.0" ], "addition_date": "2026/04/26", "description": "Web content analysis and data extraction crawler bot", "tags": [ "monitoring" ] }, { "pattern": "NaverBot", "url": "https://www.naver.com/", "instances": [ "Mozilla/5.0 (compatible; NaverBot/1.0; nhnbot@naver.com)" ], "addition_date": "2026/04/26", "description": "Naver Korean search engine web indexing crawler bot", "tags": [ "search-engine" ] }, { "pattern": "loopimprovements\\.com", "url": "http://loopimprovements.com/robot.html", "instances": [ "NetResearchServer/4.0(loopimprovements.com/robot.html)" ], "addition_date": "2026/04/26", "description": "Network research and web data collection crawler bot", "tags": [ "monitoring" ] }, { "pattern": "OpenTheBoxBot", "url": "https://knownagents.com/agents/opentheboxbot", "instances": [ "OpenTheBoxBot/1.0" ], "addition_date": "2026/04/26", "description": "Content discovery and web indexing service crawler bot", "tags": [ "search-engine" ] }, { "pattern": "OWLer-W", "url": "https://openwebsearch.eu/", "instances": [ "OWLer-W (owler@ows.eu)" ], "addition_date": "2026/04/26", "description": "Business intelligence and company data collection crawler bot", "tags": [ "monitoring" ] }, { "pattern": "peer39_crawler", "url": "https://www.peer39.com/", "instances": [ "peer39_crawler/1.0" ], "addition_date": "2026/04/26", "description": "Peer39 contextual advertising and brand safety crawler bot", "tags": [ "advertising" ] }, { "pattern": "Pixalate\\.com", "url": "https://www.pixalate.com/", "instances": [ "Pixalate.com/1.0" ], "addition_date": "2026/04/26", "description": "Pixalate ad fraud detection and brand safety crawler", "tags": [ "advertising" ] }, { "pattern": "Poduptime", "url": "https://fediverse.observer", "instances": [ "Mozilla/5.0 (compatible; Poduptime; +fediverse.observer)" ], "addition_date": "2026/04/26", "description": "Fediverse pod uptime monitoring and availability checker crawler", "tags": [ "monitoring" ] }, { "pattern": "Pomothy-Bot", "url": "https://knownagents.com/agents/pomothy-bot", "instances": [ "Mozilla/5.0 (compatible; Pomothy-Bot/1.0)" ], "addition_date": "2026/04/26", "description": "Web content monitoring and change detection crawler bot", "tags": [ "monitoring" ] }, { "pattern": "PulsePoint-Crawler", "url": "https://www.pulsepoint.com/", "instances": [ "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko; compatible; PulsePoint-Crawler) Chrome/120.0.0.0 Safari/537.36" ], "addition_date": "2026/04/26", "description": "PulsePoint programmatic advertising and ad verification crawler bot", "tags": [ "advertising" ] }, { "pattern": "rawweb-bot", "url": "https://knownagents.com/agents/rawweb-bot", "instances": [ "rawweb-bot/1.0" ], "addition_date": "2026/04/26", "description": "Raw web content extraction and data scraping crawler", "tags": [ "http-library" ] }, { "pattern": "semantic-visions", "url": "https://semantic-visions.com/", "instances": [ "Mozilla/5.0 (Linux; CentOS; compatible; semantic-visions-discovery; HTTPClient 4.5)", "Mozilla/5.0 (X11; compatible; semantic-visions.com crawler; HTTPClient 4.5)" ], "addition_date": "2026/04/26", "description": "Semantic content analysis and AI training data crawler", "tags": [ "ai-crawler" ] }, { "pattern": "Sindup", "url": "https://www.sindup.com/", "instances": [ "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/138.0.0.0 Safari/537.36 Sindup/1.0.0" ], "addition_date": "2026/04/26", "description": "Sindup competitive intelligence and market monitoring crawler bot", "tags": [ "monitoring" ] }, { "pattern": "SiteSucker", "url": "https://ricks-apps.com/osx/sitesucker/", "instances": [ "SiteSucker for macOS/6.1.5" ], "addition_date": "2026/04/26", "description": "macOS website downloading and offline browsing tool crawler", "tags": [ "archiver" ] }, { "pattern": "SpringserveBot", "url": "https://www.springserve.com/", "instances": [ "SpringserveBot/1.0" ], "addition_date": "2026/04/26", "description": "Springserve ad server and video advertising platform crawler", "tags": [ "advertising" ] }, { "pattern": "SQWatcher", "url": "http://sqcompliance.com/sqwatcher.html", "instances": [ "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/118.0.0.0 Safari/537.36 SQWatcher/202312 (sqcompliance.com/sqwatcher.html)" ], "addition_date": "2026/04/26", "description": "Software quality compliance monitoring and testing crawler bot", "tags": [ "monitoring" ] }, { "pattern": "Supabase Paired Crawler", "url": "https://supabase.com/", "instances": [ "Mozilla/5.0 (Supabase Paired Crawler HTTP/2)" ], "addition_date": "2026/04/26", "description": "Supabase database platform link preview and metadata crawler", "tags": [ "social-preview" ] }, { "pattern": "sv-watchagent", "url": "https://semantic-visions.com/", "instances": [ "Mozilla/5.0 (watchOS 6.2; Watch; compatible; sv-watchagent; HTTPClient 4.5)" ], "addition_date": "2026/04/26", "description": "Semantic Visions content monitoring and analysis crawler agent", "tags": [ "monitoring" ] }, { "pattern": "Swiftbot", "url": "http://swiftype.com/swiftbot", "instances": [ "Mozilla/5.0 (compatible; Swiftbot/1.0; UID/649dce5512ab734096a50277; +http://swiftype.com/swiftbot)" ], "addition_date": "2026/04/26", "description": "Swiftype site search engine indexing and crawling bot", "tags": [ "search-engine" ] }, { "pattern": "SynthesiBot", "url": "https://knownagents.com/agents/synthesibot", "instances": [ "Mozilla/5.0 (compatible; SynthesiBot/1.0)" ], "addition_date": "2026/04/26", "description": "Content synthesis and data aggregation crawler bot", "tags": [ "monitoring" ] }, { "pattern": "TaraGroup Intelligent Bot", "url": "https://knownagents.com/agents/taragroup-intelligent-bot", "instances": [ "TaraGroup Intelligent Bot V1" ], "addition_date": "2026/04/26", "description": "TaraGroup AI-powered web intelligence and analysis crawler", "tags": [ "ai-crawler" ] }, { "pattern": "Thinkbot", "url": "https://boston.conman.org/2025/08/21.1", "instances": [ "Mozilla/5.0 (compatible; Thinkbot/0.5.8; +In_the_test_phase,_if_the_Thinkbot_brings_you_trouble,_please_block_its_IP_address._Thank_you.)" ], "addition_date": "2026/04/26", "description": "Experimental AI thinking and reasoning web crawler bot", "tags": [ "ai-crawler" ] }, { "pattern": "TSMbot", "url": "https://knownagents.com/agents/tsmbot", "instances": [ "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7; TSMbot) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/136.0.0.0 Safari/537.36" ], "addition_date": "2026/04/26", "description": "TSM brand monitoring and content tracking crawler bot", "tags": [ "monitoring" ] }, { "pattern": "TSM-turingos", "url": "https://knownagents.com/agents/turingos", "instances": [ "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7; TSM-turingos-1253296984) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36" ], "addition_date": "2026/04/26", "description": "TuringOS AI-powered content analysis and monitoring crawler", "tags": [ "ai-crawler" ] }, { "pattern": "UGAResearchAgent", "url": "https://nislabuga-scan.uga.edu/", "instances": [ "Mozilla/5.0 (compatible; UGAResearchAgent/1.0; Please visit: NISLabUGA.github.io)" ], "addition_date": "2026/04/26", "description": "University of Georgia academic network research crawler bot", "tags": [ "academic" ] }, { "pattern": "UrlSuMa\\.de crawler", "url": "https://urlsuma.de/", "instances": [ "Mozilla/5.0 (compatible; UrlSuMa.de crawler)" ], "addition_date": "2026/04/26", "description": "URL summarization and content preview generation crawler bot", "tags": [ "monitoring" ] }, { "pattern": "WanscannerBot", "url": "https://abuse.pend.re", "instances": [ "Mozilla/5.0 (X11; Linux x86_64; WanscannerBot/1.2; +https://abuse.pend.re) Gecko/20100101 Firefox/10.0" ], "addition_date": "2026/04/26", "description": "Network security scanning and vulnerability detection crawler bot", "tags": [ "scanner" ] }, { "pattern": "WebCapture", "url": "https://knownagents.com/agents/webcapture-2-0", "instances": [ "Mozilla/3.0 (compatible; WebCapture 2.0; Auto; Windows)" ], "addition_date": "2026/04/26", "description": "Website screenshot and page capture archiving tool crawler", "tags": [ "archiver" ] }, { "pattern": "WebCopier", "url": "http://www.maximumsoft.com/", "instances": [ "WebCopier v4.6" ], "addition_date": "2026/04/26", "description": "Website copying and offline browsing tool crawler bot", "tags": [ "archiver" ] }, { "pattern": "cognitiveseo\\.com", "url": "http://cognitiveseo.com/bot.html", "instances": [ "Mozilla/5.0 (Windows; U; Windows NT 5.0; en-US; rv:1.7.10) Gecko/20050716 Thunderbird/1.0.6 - WebCrawler http://cognitiveseo.com/bot.html" ], "addition_date": "2026/04/26", "description": "CognitiveSEO backlink analysis and SEO research crawler bot", "tags": [ "seo" ] }, { "pattern": "Xing Bot", "url": "https://www.xing.com/", "instances": [ "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:15.0) Xing Bot" ], "addition_date": "2026/04/26", "description": "Xing professional network link preview and content crawler", "tags": [ "social-preview" ] }, { "pattern": "XML Sitemaps Generator", "url": "http://www.xml-sitemaps.com", "instances": [ "Mozilla/5.0 (compatible; XML Sitemaps Generator; http://www.xml-sitemaps.com) Gecko XML-Sitemaps/1.0" ], "addition_date": "2026/04/26", "description": "XML sitemap generation and website structure mapping crawler", "tags": [ "seo" ] }, { "pattern": "YandoriRSSBot", "url": "https://knownagents.com/agents/yandorirssbot", "instances": [ "YandoriRSSBot/3.0 (Go)" ], "addition_date": "2026/04/26", "description": "RSS feed aggregation and content syndication crawler bot", "tags": [ "feed-reader" ] }, { "pattern": "Zealbot", "url": "https://knownagents.com/agents/zealbot", "instances": [ "Mozilla/4.0 (compatible; Zealbot 1.0)" ], "addition_date": "2026/04/26", "description": "Web content indexing and search engine crawler bot", "tags": [ "search-engine" ] }, { "pattern": "008\\/", "url": "https://datadome.co/bots/008-2/", "instances": [ "008/0.83 (http://www.advancedwebranking.com)" ], "description": "SEO monitoring bot tracks keyword rankings across regions", "addition_date": "2026/04/17", "tags": [ "seo" ] }, { "pattern": "monitoring360bot\\/", "url": "https://app.360monitoring.com/bot.html", "instances": [ "Mozilla/5.0 (compatible; monitoring360bot/1.1; +https://app.360monitoring.com/bot.html)" ], "description": "Monitors website uptime and performance at intervals", "addition_date": "2026/04/17", "tags": [ "monitoring" ] }, { "pattern": "AdagioBot", "url": "https://datadome.co/bots/adagio-digital/", "instances": [ "AdagioBot" ], "description": "SEO analysis bot for performance and optimization", "addition_date": "2026/04/17", "tags": [ "seo" ] }, { "pattern": "adbeat\\.com", "url": "https://www.adbeat.com/operation_policy", "instances": [ "Mozilla/5.0 (iPad; CPU OS 15_3_1 like Mac OS X) adbeat.com/policy AppleWebKit/605.1.15 (KHTML, like Gecko) Version/15.3.1 Mobile/15E148 Safari/604.1" ], "description": "Competitive intelligence bot for digital ad tracking", "addition_date": "2026/04/17", "tags": [ "advertising" ] }, { "pattern": "AdminLabs", "url": "https://datadome.co/bots/adminlabs/", "instances": [ "AdminLabs" ], "description": "Web scraping tool for data collection and analysis", "addition_date": "2026/04/17", "tags": [ "seo" ] }, { "pattern": "advanced_crawler", "url": "https://datadome.co/bots/advanced-crawler/", "instances": [ "advanced_crawler (+http://example.com)" ], "description": "Sophisticated crawler with dynamic content rendering", "addition_date": "2026/04/17", "tags": [ "seo" ] }, { "pattern": "Adventurer", "url": "https://datadome.co/bots/adventurer/", "instances": [ "Adventurer" ], "description": "Web crawler for indexing content and analysis", "addition_date": "2026/04/17", "tags": [ "seo" ] }, { "pattern": "AGAKIDSBOT", "url": "https://agakids.ru/project/", "instances": [ "Mozilla/5.0 (compatible; Linux; AGAKIDSBOT/3.1 +agakids.ru/project) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36" ], "description": "Web crawler for indexing and data gathering", "addition_date": "2026/04/17", "tags": [ "seo" ] }, { "pattern": "AgencyAnalyticsBot", "url": "https://agencyanalytics.com/features/seo-site-audit", "instances": [ "AgencyAnalyticsBot" ], "description": "SEO audit bot for performance and backlink analysis", "addition_date": "2026/04/17", "tags": [ "seo" ] }, { "pattern": "AI2Bot", "url": "https://datadome.co/bots/ai2bot/", "instances": [ "AI2Bot" ], "description": "Academic research crawler for NLP dataset creation", "addition_date": "2026/04/17", "tags": [ "academic", "ai-crawler" ] }, { "pattern": "AkismetBot", "url": "https://akismet.com/development/api/", "instances": [ "AkismetBot" ], "description": "Spam filtering service for WordPress and CMS", "addition_date": "2026/04/17", "tags": [ "scanner" ] }, { "pattern": "alexa site audit", "url": "https://www.alexa.com/help/webmasters", "instances": [ "Mozilla/5.0 (compatible; alexa site audit/1.0; +http://www.alexa.com/help/webmasters; )" ], "description": "Web crawler for SEO data aggregation and analysis", "addition_date": "2026/04/17", "tags": [ "seo" ] }, { "pattern": "Algolia Crawler", "url": "https://www.algolia.com/doc/", "instances": [ "Algolia Crawler" ], "description": "Hosted search engine and site search crawler", "addition_date": "2026/04/17", "tags": [ "search-engine" ] }, { "pattern": "alienfarm", "url": "https://datadome.co/bots/alienfarm/", "instances": [ "alienfarm" ], "description": "Web crawler for data aggregation and intelligence", "addition_date": "2026/04/17", "tags": [ "seo" ] }, { "pattern": "allOrigins", "url": "https://allorigins.win/", "instances": [ "Mozilla/5.0 (compatible; allOrigins/3; +http://allorigins.win/)" ], "description": "Web scraping proxy tool bypassing CORS restrictions", "addition_date": "2026/04/17", "tags": [ "seo" ] }, { "pattern": "AmazonAdBot", "url": "https://advertising.amazon.com/resources/", "instances": [ "AmazonAdBot" ], "description": "Ad verification bot for delivery and impressions", "addition_date": "2026/04/17", "tags": [ "advertising" ] }, { "pattern": "KendraBot", "url": "https://docs.aws.amazon.com/kendra/latest/dg/what-is-kendra.html", "instances": [ "KendraBot" ], "description": "AI-powered enterprise search crawler for documents", "addition_date": "2026/04/17", "tags": [ "search-engine", "ai-crawler" ] }, { "pattern": "AppSiteAssociation", "url": "https://developer.apple.com/documentation/applications/allowing-app-linking-to-your-website", "instances": [ "AppSiteAssociation" ], "description": "Verifies app connection to website domains", "addition_date": "2026/04/17", "tags": [ "seo" ] }, { "pattern": "Aragog\\/", "url": "https://wordads.co/", "instances": [ "Aragog/1.0 (+https://wordads.co)" ], "description": "Web crawler for indexing and data collection", "addition_date": "2026/04/17", "tags": [ "seo" ] }, { "pattern": "Aranea", "url": "http://unesco.uniba.sk/guest/", "instances": [ "Aranea Web-Crawled Corpora Project (+http://unesco.uniba.sk/guest (Hebrew 2024 Spring Crawl))" ], "description": "Automated web crawler for content aggregation", "addition_date": "2026/04/17", "tags": [ "academic" ] }, { "pattern": "ArchiveBox", "url": "https://archivebox.io/", "instances": [ "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/118.0.0.0 Safari/537.36 ArchiveBox/0.7.2 (+https://github.com/ArchiveBox/ArchiveBox/)" ], "description": "Web archiving tool for long-term content preservation", "addition_date": "2026/04/17", "tags": [ "archiver" ] }, { "pattern": "ArquivoBot", "url": "https://arquivo.pt/about", "instances": [ "ArquivoBot" ], "description": "Preserves Portuguese websites historical content", "addition_date": "2026/04/17", "tags": [ "archiver" ] }, { "pattern": "Arquivo-web-crawler", "url": "https://arquivo.pt/robot", "instances": [ "Arquivo-web-crawler (compatible; +https://arquivo.pt/robot)" ], "description": "Portuguese web archive for digital heritage", "addition_date": "2026/04/17", "tags": [ "archiver" ] }, { "pattern": "ArtemisBot", "url": "https://datadome.co/bots/artemis-web-reader/", "instances": [ "ArtemisBot" ], "description": "Vulnerability scanning and cybersecurity research bot", "addition_date": "2026/04/17", "tags": [ "scanner" ] }, { "pattern": "Asana\\/", "url": "https://datadome.co/bots/asana-crawler/", "instances": [ "Asana/1.4.0 WebsiteMetadataRetriever" ], "description": "Work management platform metadata retriever bot", "addition_date": "2026/04/17", "tags": [ "seo" ] }, { "pattern": "AudistoBot", "url": "https://audisto.com/webcrawler/", "instances": [ "AudistoBot" ], "description": "Technical SEO and website audit bot", "addition_date": "2026/04/17", "tags": [ "seo" ] }, { "pattern": "Autoconfig Test from USTC", "url": "https://datadome.co/bots/autoconfig-test-from-ustc/", "instances": [ "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/96.0.4664.93 Safari/537.36 (Autoconfig Test from USTC, Quit please contact: mailservertest2023@gmail.com)" ], "description": "USTC university research crawler for web analysis", "addition_date": "2026/04/17", "tags": [ "academic" ] }, { "pattern": "tracking-quality-spider", "url": "https://datadome.co/bots/awin-com-crawler/", "instances": [ "Mozilla/5.0 (compatible; tracking-quality-spider/0.1; https://www.awin.com)" ], "description": "Affiliate marketing network tracking and verification", "addition_date": "2026/04/17", "tags": [ "advertising" ] }, { "pattern": "Bad Neighborhood Header Detector", "url": "https://datadome.co/bots/bad-neighborhood/", "instances": [ "Bad Neighborhood Header Detector (http://www.bad-neighborhood.com/header_detector.php)" ], "description": "Website security risk analysis for link quality", "addition_date": "2026/04/17", "tags": [ "scanner" ] }, { "pattern": "BaiduAdsBot", "url": "https://datadome.co/bots/baidu-ads-server-proxy/", "instances": [ "BaiduAdsBot" ], "description": "Baidu ad verification and tracking bot", "addition_date": "2026/04/17", "tags": [ "advertising" ] }, { "pattern": "BDBot\\/", "url": "https://datadome.co/bots/bdbot/", "instances": [ "BDBot/1.0" ], "description": "Web crawler for data aggregation and analysis", "addition_date": "2026/04/17", "tags": [ "seo" ] }, { "pattern": "BeeperBot", "url": "https://www.beeper.com/", "instances": [ "BeeperBot/0 Matrix-Media-Repo/1" ], "description": "Web crawler for aggregation and business intelligence", "addition_date": "2026/04/17", "tags": [ "seo" ] }, { "pattern": "BetterUptimeBot", "url": "https://betteruptime.com/docs", "instances": [ "BetterUptimeBot" ], "description": "Website uptime and performance monitoring service", "addition_date": "2026/04/17", "tags": [ "monitoring" ] }, { "pattern": "BnFBot", "url": "https://www.bnf.fr/en/web-services", "instances": [ "BnFBot" ], "description": "French national library archiving cultural content", "addition_date": "2026/04/17", "tags": [ "archiver" ] }, { "pattern": "BigUpDataBot", "url": "https://datadome.co/bots/bigupdata/", "instances": [ "BigUpDataBot" ], "description": "Analytics crawler for SEO and metric tracking", "addition_date": "2026/04/17", "tags": [ "seo" ] }, { "pattern": "BinaryCanary", "url": "https://www.binarycanary.com/monitoring/", "instances": [ "BinaryCanary/1.0 (+https://www.binarycanary.com)" ], "description": "Uptime monitoring for servers and services", "addition_date": "2026/04/17", "tags": [ "monitoring" ] }, { "pattern": "Bitbucket-Webhooks", "url": "https://support.atlassian.com/bitbucket-cloud", "instances": [ "Bitbucket-Webhooks/2.0" ], "description": "Webhook notifications for CI/CD pipeline automation", "addition_date": "2026/04/17", "tags": [ "seo" ] }, { "pattern": "bl\\.uk_ldfc_bot", "url": "https://www.bl.uk/legal-deposit/web-archiving", "instances": [ "bl.uk_ldfc_bot/3.4.0-20220727 (+https://www.bl.uk/legal-deposit/web-archiving)" ], "description": "British Library web archiver for content preservation", "addition_date": "2026/04/17", "tags": [ "archiver" ] }, { "pattern": "BlackDuck-FD", "url": "https://www.synopsys.com/software-integrity/security-testing/dynamic-analysis.html", "instances": [ "BlackDuck-FD" ], "description": "Vulnerability scanner for application security testing", "addition_date": "2026/04/17", "tags": [ "scanner" ] }, { "pattern": "Blogtrottr", "url": "https://datadome.co/bots/blogtrottr/", "instances": [ "Blogtrottr/2.0" ], "description": "Email subscription service for blog and feed updates", "addition_date": "2026/04/17", "tags": [ "feed-reader" ] }, { "pattern": "BlueskyPreviewBot", "url": "https://docs.bsky.app", "instances": [ "BlueskyPreviewBot/1.0" ], "description": "Bluesky social platform link preview fetcher", "addition_date": "2026/04/17", "tags": [ "social-preview" ] }, { "pattern": "BoardGamePricesBot", "url": "https://datadome.co/bots/boardgameprices/", "instances": [ "Likely BoardGamePricesBot or similar (not officially published)" ], "description": "Board game price comparison and availability tracker", "addition_date": "2026/04/17", "tags": [ "seo" ] }, { "pattern": "BotPoke", "url": "https://datadome.co/bots/botpoke/", "instances": [ "BotPoke" ], "description": "Web crawler for data aggregation and analysis", "addition_date": "2026/04/17", "tags": [ "seo" ] }, { "pattern": "BDFetch", "url": "http://www.brandprotect.com/", "instances": [ "BDFetch" ], "description": "Brand protection bot for unauthorized use detection", "addition_date": "2026/04/17", "tags": [ "seo" ] }, { "pattern": "Brandwatch", "url": "https://www.brandwatch.com/legal/crawlers/", "instances": [ "Brandwatch/1.0 (varies)" ], "description": "Social listening and sentiment analysis platform", "addition_date": "2026/04/17", "tags": [ "seo" ] }, { "pattern": "BraveBot", "url": "https://search.brave.com/help/web-discovery-project", "instances": [ "BraveBot / Mozilla/5.0 (compatible; BraveBot/1.0)" ], "description": "Privacy-focused search engine indexing crawler", "addition_date": "2026/04/17", "tags": [ "search-engine" ] }, { "pattern": "brokenlinkcheck\\.com", "url": "https://datadome.co/bots/brokenlinkcheck-com/", "instances": [ "Mozilla/5.0 (Windows NT 5.1) brokenlinkcheck.com/1.2" ], "description": "Broken link detection and website integrity checker", "addition_date": "2026/04/17", "tags": [ "seo" ] }, { "pattern": "BW\\/", "url": "https://builtwith.com/biup", "instances": [ "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko; compatible; BW/1.2; rb.gy/oupwis) Chrome/124.0.0.0 Safari/537.36" ], "description": "Technology profiling crawler for web analytics", "addition_date": "2026/04/17", "tags": [ "seo" ] }, { "pattern": "Bushbaby", "url": "https://datadome.co/bots/bushbaby/", "instances": [ "Bushbaby (reported in logs)" ], "description": "Content aggregation and market intelligence crawler", "addition_date": "2026/04/17", "tags": [ "seo" ] }, { "pattern": "Butterfly", "url": "http://labs.topsy.com/butterfly/", "instances": [ "Mozilla/5.0 (compatible; Butterfly/1.0; +http://labs.topsy.com/butterfly/) Gecko/2009032608 Firefox/3.0.8" ], "description": "Experimental data collection crawler", "addition_date": "2026/04/17", "tags": [ "seo" ] }, { "pattern": "rss-parser", "url": "https://buttondown.email/about", "instances": [ "rss-parser / Buttondown" ], "description": "Newsletter RSS feed parser and aggregator", "addition_date": "2026/04/17", "tags": [ "feed-reader" ] }, { "pattern": "CaliberBot", "url": "https://www.calibermind.com/platform", "instances": [ "CaliberBot (varies)" ], "description": "Market research and analytics crawler", "addition_date": "2026/04/17", "tags": [ "seo" ] }, { "pattern": "CapitalOneShopping", "url": "https://datadome.co/bots/capital-one-shopping-bot/", "instances": [ "CapitalOneShopping/1.0 (example)" ], "description": "E-commerce price comparison and shopping bot", "addition_date": "2026/04/17", "tags": [ "seo" ] }, { "pattern": "Catchpoint", "url": "ttps://catchpoint.com/bots", "instances": [ "Catchpoint" ], "description": "Digital experience and performance monitoring bot", "addition_date": "2026/04/17", "tags": [ "monitoring" ] }, { "pattern": "centuryb\\.o\\.t9", "url": "https://datadome.co/bots/centurybot9/", "instances": [ "Mozilla/5.0 (compatible; +centuryb.o.t9[at]gmail.com)" ], "description": "General web crawler for data aggregation", "addition_date": "2026/04/17", "tags": [ "seo" ] }, { "pattern": "CERT PL", "url": "https://cert.pl/skanowanie", "instances": [ "Mozilla/5.0 (compatible; Artemis; CERT PL; +https://cert.pl/skanowanie)" ], "description": "Polish cybersecurity vulnerability and threat scanner", "addition_date": "2026/04/17", "tags": [ "scanner" ] }, { "pattern": "certytags", "url": "https://certybot.certytags.com/", "instances": [ "Mozilla/5.0 (compatible; certytags/1.0; +https://certybot.certytags.com)" ], "description": "Web content indexing and verification crawler", "addition_date": "2026/04/17", "tags": [ "seo" ] }, { "pattern": "ChargeBeeBot", "url": "https://chargebee.com/resources", "instances": [ "ChargeBeeBot" ], "description": "Subscription billing verification and integration bot", "addition_date": "2026/04/17", "tags": [ "seo" ] }, { "pattern": "Charlotte", "url": "https://datadome.co/bots/charlotte-bot/", "instances": [ "Commonly seen variations include: Mozilla/5.0 (X11; U; Linux i686 (x86_64); en-US; rv:1.8.1.11) Gecko/20080109 (Charlotte/0.9t; http://www.searchme.com/support/)" ], "description": "Financial institution and robot-friendly crawler", "addition_date": "2026/04/17", "tags": [ "seo" ] }, { "pattern": "ChatGLM-Spider", "url": "https://chatglm.cn/", "instances": [ "Mozilla/5.0 (compatible; ChatGLM-Spider/1.0; +https://chatglm.cn/)" ], "description": "AI model training data collection crawler", "addition_date": "2026/04/17", "tags": [ "ai-crawler" ] }, { "pattern": "Chatwork LinkPreview", "url": "https://www.chatwork.com/", "instances": [ "Chatwork LinkPreview v1" ], "description": "Collaboration platform URL preview fetcher", "addition_date": "2026/04/17", "tags": [ "social-preview" ] }, { "pattern": "CheckHost", "url": "https://datadome.co/bots/check-host/", "instances": [ "CheckHost (https://check-host.net/)" ], "description": "Website uptime and performance monitoring bot", "addition_date": "2026/04/17", "tags": [ "monitoring" ] }, { "pattern": "Goodzer", "url": "https://discord.com/discovery/applications/1065250549408223252", "instances": [ "Mozilla/5.0 (compatible; Goodzer/1.0) and Mozilla/5.0 (compatible; Goodzer/2.0; crawler@goodzer.com)" ], "description": "Discord bot enhancement and server functionality crawler", "addition_date": "2026/04/17", "tags": [ "seo" ] }, { "pattern": "Chrome Privacy Preserving Prefetch Proxy", "url": "https://datadome.co/bots/chrome-privacypreserving-prefetch-proxy/", "instances": [ "Chrome Privacy Preserving Prefetch Proxy" ], "description": "Chrome browser resource prefetching with privacy protection", "addition_date": "2026/04/17", "tags": [ "seo" ] }, { "pattern": "CirrusExplorer", "url": "https://cseu.ro/explorer.php", "instances": [ "CirrusExplorer/2 (https://cseu.ro/explorer.php)" ], "description": "Web crawler for indexing and competitive analysis", "addition_date": "2026/04/17", "tags": [ "seo" ] }, { "pattern": "CLASSLA-web", "url": "https://www.clarin.si/info/classla-web-crawler/", "instances": [ "Mozilla/5.0 (compatible; CLASSLA-web; +https://www.clarin.si/info/classla-web-crawler/)" ], "description": "Linguistic research data crawler for NLP", "addition_date": "2026/04/17", "tags": [ "academic" ] }, { "pattern": "Clearscopebot", "url": "https://datadome.co/bots/clearscope-clearscopebot/", "instances": [ "UA Clearscopebot or similar" ], "description": "Content optimization and keyword performance analysis", "addition_date": "2026/04/17", "tags": [ "seo" ] }, { "pattern": "WorldBot", "url": "https://datadome.co/bots/clickagy-intelligence-bot/", "instances": [ "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; WorldBot/0.1; +https://worldlabs.ai)" ], "description": "Behavioral analytics and audience segmentation crawler", "addition_date": "2026/04/17", "tags": [ "seo" ] }, { "pattern": "Cloudflare-Validator", "url": "https://datadome.co/bots/cloudflare-crawler/", "instances": [ "Cloudflare-Validator/1.0" ], "description": "Cloudflare security and performance validation", "addition_date": "2026/04/17", "tags": [ "monitoring" ] }, { "pattern": "cloudflare-csup", "url": "https://datadome.co/bots/cloudflare-csup/", "instances": [ "cloudflare-csup (Cloudflare Radar)" ], "description": "Cache status updates and performance monitoring", "addition_date": "2026/04/17", "tags": [ "monitoring" ] }, { "pattern": "Cloudflare-Custom-Error-Page-Crawler", "url": "https://developers.cloudflare.com", "instances": [ "Cloudflare-Custom-Error-Page-Crawler" ], "description": "Custom error page validation and testing", "addition_date": "2026/04/17", "tags": [ "monitoring" ] }, { "pattern": "Cloudflare-Radar-Scanner", "url": "https://radar.cloudflare.com", "instances": [ "Cloudflare-Radar-Scanner" ], "description": "URL scanning for traffic and threat intelligence", "addition_date": "2026/04/17", "tags": [ "scanner" ] }, { "pattern": "Cloudflare-SpeedTest", "url": "https://www.cloudflare.com/speedtest", "instances": [ "Cloudflare-SpeedTest" ], "description": "Website performance and latency testing bot", "addition_date": "2026/04/17", "tags": [ "monitoring" ] }, { "pattern": "Cloudflare-Stream-Hook", "url": "https://developers.cloudflare.com/stream/webhooks/", "instances": [ "Cloudflare-Stream-Hook" ], "description": "Video lifecycle event notifications and management", "addition_date": "2026/04/17", "tags": [ "seo" ] }, { "pattern": "cognitiveSEO Bot", "url": "https://cognitiveseo.com/bot", "instances": [ "cognitiveSEO Bot" ], "description": "SEO analytics and digital marketing crawler", "addition_date": "2026/04/17", "tags": [ "seo" ] }, { "pattern": "cohere-training-data-crawler", "url": "https://datadome.co/bots/cohere-training-data-crawler/", "instances": [ "cohere-training-data-crawler (+crawler@cohere.ai)" ], "description": "LLM training data collection for AI models", "addition_date": "2026/04/17", "tags": [ "ai-crawler" ] }, { "pattern": "CommaFeed", "url": "https://datadome.co/bots/commafeed/", "instances": [ "CommaFeed/4.4.0 (https://github.com/Athou/commafeed)" ], "description": "RSS feed aggregator and reader bot", "addition_date": "2026/04/17", "tags": [ "feed-reader" ] }, { "pattern": "researchscan\\.comsys\\.rwth-aachen\\.de", "url": "http://researchscan.comsys.rwth-aachen.de/", "instances": [ "Mozilla/5.0 researchscan.comsys.rwth-aachen.de" ], "description": "Academic network measurement and analysis", "addition_date": "2026/04/17", "tags": [ "academic" ] }, { "pattern": "contentkingapp", "url": "https://whatis.contentkingapp.com/", "instances": [ "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.113 Safari/537.36 (+https://whatis.contentkingapp.com)" ], "description": "Real-time SEO auditing and monitoring tool", "addition_date": "2026/04/17", "tags": [ "seo" ] }, { "pattern": "CookieHub Bot", "url": "https://www.cookiehub.com/docs", "instances": [ "CookieHub Bot" ], "description": "Cookie consent and privacy compliance auditor", "addition_date": "2026/04/17", "tags": [ "scanner" ] }, { "pattern": "Cotoyogi", "url": "https://ds.rois.ac.jp/center8/crawler/", "instances": [ "Mozilla/5.0 (compatible; Cotoyogi/4.0; +https://ds.rois.ac.jp/center8/crawler/)" ], "description": "Web crawler for content and data aggregation", "addition_date": "2026/04/17", "tags": [ "seo" ] }, { "pattern": "Coveobot", "url": "https://platform.cloud.coveo.com/", "instances": [ "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko) (compatible; Coveobot/2.0;+http://www.coveo.com/bot.html)" ], "description": "Enterprise search and content indexing platform", "addition_date": "2026/04/17", "tags": [ "search-engine" ] }, { "pattern": "Crawlson", "url": "https://www.crawlson.com/", "instances": [ "Mozilla/5.0 (compatible; Crawlson/1.0; +https://www.crawlson.com/domain)" ], "description": "Web crawler for auditing and monitoring", "addition_date": "2026/04/17", "tags": [ "seo" ] }, { "pattern": "RepoLookoutBot", "url": "https://www.repo-lookout.org/", "instances": [ "RepoLookoutBot/v1.1.0-297-gcf436d3 (abuse reports to abuse@repo-lookout.org)" ], "description": "Repository and web content lookup crawler", "addition_date": "2026/04/17", "tags": [ "seo" ] }, { "pattern": "Criticalcss\\.com", "url": "https://criticalcss.com/", "instances": [ "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.0 Safari/537.36 Criticalcss.com/2.0.0" ], "description": "Critical CSS extraction for performance optimization", "addition_date": "2026/04/17", "tags": [ "seo" ] }, { "pattern": "cron-job\\.org", "url": "https://cron-job.org/en/", "instances": [ "cron-job.org monitor" ], "description": "Automated website monitoring and uptime checking", "addition_date": "2026/04/17", "tags": [ "monitoring" ] }, { "pattern": "DnBCrawler", "url": "https://datadome.co/bots/dnbcrawler/", "instances": [ "DnBCrawler-Analytics" ], "description": "Business data analytics crawler gathering company information insights", "addition_date": "2026/04/17", "tags": [ "seo" ] }, { "pattern": "DMBrowser", "url": "https://www.dotcom-monitor.com/wiki/knowledge-base-main/", "instances": [ "DMBrowser/" ], "description": "Synthetic monitoring browser checking website performance and availability", "addition_date": "2026/04/17", "tags": [ "monitoring" ] }, { "pattern": "DomCopBot", "url": "https://www.domcop.com/bot", "instances": [ "DomCopBot (https://www.domcop.com/bot)" ], "description": "Domain research crawler analyzing website metrics and SEO data", "addition_date": "2026/04/17", "tags": [ "seo" ] }, { "pattern": "downnotifier\\.com", "url": "https://datadome.co/bots/downnotifier-com-monitoring/", "instances": [ "downnotifier.com monitoring" ], "description": "Uptime monitoring service alerting on website downtime events", "addition_date": "2026/04/17", "tags": [ "monitoring" ] }, { "pattern": "DowntimeDetector\\/", "url": "https://datadome.co/bots/downtimedetector/", "instances": [ "DowntimeDetector/4.0 (+https://downforeveryoneorjustme.com)" ], "description": "Availability monitor detecting and reporting website downtime incidents", "addition_date": "2026/04/17", "tags": [ "monitoring" ] }, { "pattern": "Dlc\\/", "url": "https://www.drlinkcheck.com/", "instances": [ "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4896.127 Safari/537.36 Dlc/2.0.1" ], "description": "Link checking crawler auditing broken URLs across websites", "addition_date": "2026/04/17", "tags": [ "seo" ] }, { "pattern": "Dratabot", "url": "https://dratabot.com", "instances": [ "Dratabot (+https://dratabot.com)" ], "description": "Compliance automation crawler auditing security controls and documentation", "addition_date": "2026/04/17", "tags": [ "scanner" ] }, { "pattern": "EasyBib AutoCite", "url": "http://www.easybib.com/", "instances": [ "EasyBib AutoCite (http://autocite-info.citation-api.com/)" ], "description": "Citation generator fetching metadata for bibliography and reference creation", "addition_date": "2026/04/17", "tags": [ "academic" ] }, { "pattern": "easybill-ImportManager", "url": "https://www.easybill.de/api/", "instances": [ "easybill-ImportManager/ShopwareClient" ], "description": "Billing platform crawler importing product data from ecommerce stores", "addition_date": "2026/04/17", "tags": [ "seo" ] }, { "pattern": "EasyCron\\/", "url": "https://www.easycron.com", "instances": [ "EasyCron/1.0 (https://www.easycron.com/)" ], "description": "Cron job service executing scheduled HTTP requests and monitoring endpoints", "addition_date": "2026/04/17", "tags": [ "monitoring" ] }, { "pattern": "easyDNS Monitoring", "url": "http://easyurl.net/monitoring", "instances": [ "easyDNS Monitoring ( http://easyurl.net/monitoring )" ], "description": "DNS monitoring service verifying domain resolution and availability", "addition_date": "2026/04/17", "tags": [ "monitoring" ] }, { "pattern": "EchoboxBot\\/", "url": "https://www.echobox.com/", "instances": [ "Mozilla/5.0 (compatible; EchoboxBot/1.0; hash/w4mwnpbXf3MFAbxOkJRw; +http://www.echobox.com)" ], "description": "Social media automation crawler fetching content for publishing optimization", "addition_date": "2026/04/17", "tags": [ "seo" ] }, { "pattern": "Cronless", "url": "https://datadome.co/bots/cronless/", "instances": [ "Cronless/2.0 (+http://cronless.com)" ], "description": "Dynamic task execution crawler without fixed scheduling", "addition_date": "2026/04/17", "tags": [ "monitoring" ] }, { "pattern": "crusty\\/", "url": "https://github.com/let4be/crusty", "instances": [ "crusty/0.12.0" ], "description": "Undocumented web crawler of unknown origin", "addition_date": "2026/04/17", "tags": [ "seo" ] }, { "pattern": "csirt\\.cz", "url": "https://csirt.cz/cs/dns-crawler", "instances": [ "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.131 Safari/537.36 1.6.4 (+https://csirt.cz/cs/dns-crawler)" ], "description": "Czech cybersecurity team vulnerability scanning bot", "addition_date": "2026/04/17", "tags": [ "scanner" ] }, { "pattern": "CXK_Bot", "url": "https://datadome.co/bots/cxk_bot/", "instances": [ "CXK/5.0 (linux; 43A1AAD0; ARMv9) assistenC5/v6.5.6 (KHTML, lcxk) CXK_Bot/2.3.1" ], "description": "Web content indexing and SEO analysis crawler", "addition_date": "2026/04/17", "tags": [ "seo" ] }, { "pattern": "daumoa", "url": "http://cs.daum.net/faq/15/4118.html?faqId=28966", "instances": [ "daumoa,damoa,daum,daumos,duamoa,duam,duamo" ], "description": "South Korean Daum search engine indexing bot", "addition_date": "2026/04/17", "tags": [ "search-engine" ] }, { "pattern": "DaspeedBot", "url": "https://datadome.co/bots/dawap-bot/", "instances": [ "DaspeedBot/1.0 (+https://daspeed.io/bot-info)" ], "description": "Web content indexing and analytics crawler", "addition_date": "2026/04/17", "tags": [ "seo" ] }, { "pattern": "Dead Link Checker", "url": "http://www.dead-link-checker.com/", "instances": [ "Mozilla/5.0 (compatible; Dead Link Checker; http://www.dead-link-checker.com/)" ], "description": "Broken link detection and website maintenance bot", "addition_date": "2026/04/17", "tags": [ "seo" ] }, { "pattern": "Deskyobot", "url": "https://www.deskyo.com/bot", "instances": [ "Deskyobot/1.0 (+https://www.deskyo.com/bot)" ], "description": "Website content indexing and SEO analysis", "addition_date": "2026/04/17", "tags": [ "seo" ] }, { "pattern": "Detectify", "url": "https://detectify.com/what-is-detectify", "instances": [ "Mozilla/5.0 (compatible; Detectify) +https://detectify.com/bot/" ], "description": "Vulnerability and attack surface scanning bot", "addition_date": "2026/04/17", "tags": [ "scanner" ] }, { "pattern": "Devin", "url": "https://docs.devin.ai/get-started/devin-intro", "instances": [ "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/137.0.0.0 Safari/537.36; Devin/1.0; +https://devin.ai" ], "description": "AI-driven browser automation and content extraction", "addition_date": "2026/04/17", "tags": [ "browser-automation", "ai-crawler" ] }, { "pattern": "DF Bot", "url": "https://datadome.co/bots/df-bot/", "instances": [ "DF Bot 1.0" ], "description": "Web content indexing and competitive analysis", "addition_date": "2026/04/17", "tags": [ "seo" ] }, { "pattern": "DingTalkBot-LinkService", "url": "https://www.dingtalk.com/", "instances": [ "DingTalkBot-LinkService/1.0 (+https://open-doc.dingtalk.com/microapp/faquestions/ftpfeu)" ], "description": "Alibaba DingTalk link preview and metadata fetcher", "addition_date": "2026/04/17", "tags": [ "social-preview" ] }, { "pattern": "Discourse Forum Onebox", "url": "https://discourse.org/", "instances": [ "Discourse Forum Onebox v3.3.0.beta1-dev" ], "description": "Forum link preview and embed content fetcher", "addition_date": "2026/04/17", "tags": [ "social-preview" ] }, { "pattern": "Dmbot", "url": "https://datadome.co/bots/dmbot/", "instances": [ "Mozilla/5.0 (compatible; Dmbot/1.1)" ], "description": "Web crawler for data aggregation and SEO", "addition_date": "2026/04/17", "tags": [ "seo" ] }, { "pattern": "SustainabilityCrawler", "url": "https://datadome.co/bots/ecovadis-bot/", "instances": [ "ecotrek GmbH - SustainabilityCrawler (bot@ecotrek.tech)" ], "description": "ESG and sustainability rating data collection", "addition_date": "2026/04/17", "tags": [ "seo" ] }, { "pattern": "edansbot", "url": "https://datadome.co/bots/edansbot/", "instances": [ "Mozilla/5.0 edansbot (Windows NT 10.0; Win64; x64; rv:90.0) Gecko/20100101 Firefox/118.0" ], "description": "Web content indexing and analytics crawler", "addition_date": "2026/04/17", "tags": [ "seo" ] }, { "pattern": "EdgeWatch", "url": "https://about.edgewatch.com/", "instances": [ "Mozilla/5.0 (compatible; EdgeWatch/1.1; https://about.edgewatch.com/)" ], "description": "Performance monitoring and security assessment bot", "addition_date": "2026/04/17", "tags": [ "monitoring" ] }, { "pattern": "Do Not Track Verifier", "url": "https://datadome.co/bots/eff-crawler/", "instances": [ "Electronic Frontier Foundation's Do Not Track Verifier (for questions or concerns email dnt-policy@eff.org)" ], "description": "Privacy practices and security protocol analyzer", "addition_date": "2026/04/17", "tags": [ "scanner" ] }, { "pattern": "elmahio-uptimebot", "url": "https://elmah.io", "instances": [ "elmahio-uptimebot/2.0" ], "description": "Error logging uptime and availability monitoring", "addition_date": "2026/04/17", "tags": [ "monitoring" ] }, { "pattern": "eMoneyBot", "url": "https://emoneyadvisor.com", "instances": [ "eMoneyBot/1.0; +https://emoneyadvisor.com/DataAggregationNotice/" ], "description": "Financial account aggregation and data scraping", "addition_date": "2026/04/17", "tags": [ "seo" ] }, { "pattern": "EpivozCrawler", "url": "https://www.techmeme.com", "instances": [ "EpivozCrawler/1.7" ], "description": "Content scraping and indexing crawler", "addition_date": "2026/04/17", "tags": [ "seo" ] }, { "pattern": "eRepublik\\.tools", "url": "https://erepublik.tools", "instances": [ "eRepublik.tools - Multithreaded Crawler - " ], "description": "MMO game data and analytics crawler", "addition_date": "2026/04/17", "tags": [ "seo" ] }, { "pattern": "EvoUptimeBot", "url": "https://www.evo.agency", "instances": [ "EvoUptimeBot/1.0" ], "description": "Uptime and performance monitoring service", "addition_date": "2026/04/17", "tags": [ "monitoring" ] }, { "pattern": "ExodusMovement", "url": "https://www.exodus.io", "instances": [ "ExodusMovement/1.0 GlobalCoinHeight/1.0" ], "description": "Web scraping and data extraction bot", "addition_date": "2026/04/17", "tags": [ "seo" ] }, { "pattern": "Ezgif", "url": "https://ezgif.com/about", "instances": [ "Mozilla/5.0 (compatible; Ezgif/69.420; +https://ezgif.com/about)" ], "description": "GIF and image fetching and editing bot", "addition_date": "2026/04/17", "tags": [ "seo" ] }, { "pattern": "factset_spyderbot", "url": "https://www.factset.com/", "instances": [ "Mozilla/5.0(windows NT 10.0; Win64; x64) AppleWebkit/537.36(KHTML, like Gecko) Chrome/90.0.4430.72 Safari/537.36 factset_spyderbot" ], "description": "Financial data and regulatory filing crawler", "addition_date": "2026/04/17", "tags": [ "seo" ] }, { "pattern": "FastmailUA", "url": "https://www.fastmail.com/policies/bots/", "instances": [ "FastmailUA/1.0" ], "description": "Email link preview and image proxy fetcher", "addition_date": "2026/04/17", "tags": [ "social-preview" ] }, { "pattern": "FDL Stats Bot", "url": "https://ftwentertainment.com", "instances": [ "FDL Stats Bot" ], "description": "Firebase dynamic link analytics and validation", "addition_date": "2026/04/17", "tags": [ "seo" ] }, { "pattern": "Fedicabot", "url": "https://fedica.com/info/fedicabot", "instances": [ "Fedicabot / FedicaApp" ], "description": "Social media analytics and link preview bot", "addition_date": "2026/04/17", "tags": [ "seo" ] }, { "pattern": "FedReporterDataBot", "url": "https://fedreporter.net/FedReporterBotDocumentation/Readme.txt", "instances": [ "FedReporterDataBot/1.0 (+https://fedreporter.net/FedReporterBotDocumentation;)" ], "description": "Financial and regulatory filing data aggregator", "addition_date": "2026/04/17", "tags": [ "seo" ] }, { "pattern": "Feed Image Audit", "url": "https://image-validator.com/", "instances": [ "Feed Image Audit -- App" ], "description": "Image validation and compliance auditing", "addition_date": "2026/04/17", "tags": [ "scanner" ] }, { "pattern": "FeedBurner", "url": "http://www.feedburner.com/", "instances": [ "FeedBurner/1.0 (http://www.FeedBurner.com)" ], "description": "RSS feed distribution and subscriber management", "addition_date": "2026/04/17", "tags": [ "feed-reader" ] }, { "pattern": "feeder\\.co", "url": "https://feeder.co/crawler", "instances": [ "Mozilla/5.0 (feeder.co; Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36" ], "description": "RSS feed aggregation and monitoring service", "addition_date": "2026/04/17", "tags": [ "feed-reader" ] }, { "pattern": "Feedpresso Content Index Bot", "url": "https://datadome.co/bots/feedpresso-crawler/", "instances": [ "Feedpresso Content Index Bot" ], "description": "News aggregation platform content indexer", "addition_date": "2026/04/17", "tags": [ "feed-reader" ] }, { "pattern": "Feedwind", "url": "http://feed.mikle.com/support/description/", "instances": [ "Mozilla/5.0 (compatible; Feedwind/3.0; +http://feed.mikle.com/support/description/)" ], "description": "Dynamic content aggregation widget fetcher", "addition_date": "2026/04/17", "tags": [ "feed-reader" ] }, { "pattern": "fidget-spinner-bot", "url": "https://datadome.co/bots/fidget-spinner-bot/", "instances": [ "fidget-spinner-bot" ], "description": "Content indexing and SEO analysis crawler", "addition_date": "2026/04/17", "tags": [ "seo" ] }, { "pattern": "FirmoGraph", "url": "https://datadome.co/bots/firmograph/", "instances": [ "FirmoGraph (+https://firmograph.io)" ], "description": "Business data collection and analytics crawler", "addition_date": "2026/04/17", "tags": [ "seo" ] }, { "pattern": "FlipboardRSS", "url": "http://flipboard.com/browserproxy", "instances": [ "Mozilla/5.0 (compatible; FlipboardRSS/1.2; +http://flipboard.com/browserproxy)" ], "description": "Social magazine content aggregation and curation", "addition_date": "2026/04/17", "tags": [ "feed-reader" ] }, { "pattern": "Foregenix", "url": "http://www.foregenix.com/scan", "instances": [ "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko, Foregenix) Chrome/91.0.4472.77 Safari/537.36" ], "description": "Cybersecurity vulnerability and weakness scanner", "addition_date": "2026/04/17", "tags": [ "scanner" ] }, { "pattern": "Freespoke\\/", "url": "https://docs.freespoke.com/search/bot/", "instances": [ "Mozilla/5.0 (compatible; Freespoke/2.0; +https://crawler.freespoke.com)" ], "description": "Privacy-focused search engine indexing web content for results", "addition_date": "2026/04/17", "tags": [ "search-engine" ] }, { "pattern": "Friendly testing bot", "url": "https://datadome.co/bots/friendly-testing-bot/", "instances": [ "Friendly testing bot" ], "description": "Testing crawler verifying website functionality and response behavior", "addition_date": "2026/04/17", "tags": [ "seo" ] }, { "pattern": "friendly-spider", "url": "https://datadome.co/bots/friendly-spider/", "instances": [ "friendly-spider" ], "description": "Web crawler indexing content for search and data aggregation", "addition_date": "2026/04/17", "tags": [ "seo" ] }, { "pattern": "FriendlyCrawler\\/", "url": "https://datadome.co/bots/friendlycrawler/", "instances": [ "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/605.1.15 (KHTML, like Gecko; compatible; FriendlyCrawler/1.0) Chrome/120.0.6099.216 Safari/605.1.15" ], "description": "General web crawler indexing content for data collection purposes", "addition_date": "2026/04/17", "tags": [ "seo" ] }, { "pattern": "FullStoryBot\\/", "url": "https://help.fullstory.com/spp-ref/343521-what-is-the-fullstorybot", "instances": [ "FullStoryBot/1.0" ], "description": "Analytics platform crawler collecting session and behavioral data", "addition_date": "2026/04/17", "tags": [ "seo" ] }, { "pattern": "Funnelback", "url": "https://docs.squiz.net/funnelback/docs/latest/", "instances": [ "Funnelback" ], "description": "Enterprise search crawler indexing content for internal search platforms", "addition_date": "2026/04/17", "tags": [ "search-engine" ] }, { "pattern": "FuseonBot\\/", "url": "https://datadome.co/bots/fuseonbot/", "instances": [ "FuseonBot/1.1 (+http://linkaffinity.io)" ], "description": "Link affinity crawler indexing web content for SEO analysis", "addition_date": "2026/04/17", "tags": [ "seo" ] }, { "pattern": "Gabanzabot\\/", "url": "https://datadome.co/bots/gabanzabot/", "instances": [ "Gabanzabot/1.1 (Gabanza Search Engine; https://www.gabanza.com)" ], "description": "Search engine crawler indexing websites for Gabanza search results", "addition_date": "2026/04/17", "tags": [ "search-engine" ] }, { "pattern": "gdnplus\\.com", "url": "https://datadome.co/bots/gdnp-crawler/", "instances": [ "https://gdnplus.com:Gather Analyze Provide." ], "description": "Data gathering crawler analyzing and providing web content insights", "addition_date": "2026/04/17", "tags": [ "seo" ] }, { "pattern": "getthit\\.com", "url": "https://www.getthit.com/bot", "instances": [ "www.GettHIT.com | Free Traffic Exchange Bot | If you are seeing this, then your website has been listed in our traffic exchange service. | Visit Us : https://www.getthit.com/bot | Macintosh; Intel Mac OS X 10_7_5 (compatible; getthit.com/3.1;)" ], "description": "Traffic exchange service bot generating visits to listed websites", "addition_date": "2026/04/17", "tags": [ "seo" ] }, { "pattern": "GG PeekBot", "url": "https://www.gg.pl/", "instances": [ "GG PeekBot 2.0 ( https://www.gg.pl/ https://www.gg.pl/info/praca/ )" ], "description": "Polish messaging platform crawler generating link previews for users", "addition_date": "2026/04/17", "tags": [ "social-preview" ] }, { "pattern": "Ghost Inspector", "url": "https://ghostinspector.com/", "instances": [ "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/107.0 Safari/537.36 Ghost Inspector (63c85ddbde52d6697f57c623)" ], "description": "Browser testing service automating UI tests and monitoring website changes", "addition_date": "2026/04/17", "tags": [ "browser-automation" ] }, { "pattern": "github-camo", "url": "https://github.com/atmos/camo", "instances": [ "github-camo (c006e452)" ], "description": "GitHub image proxy fetching external images for secure rendering", "addition_date": "2026/04/17", "tags": [ "social-preview" ] }, { "pattern": "GlobalWebSearch", "url": "https://datadome.co/bots/globalwebsearch/", "instances": [ "GlobalWebSearchx" ], "description": "Web crawler indexing content for global search engine results", "addition_date": "2026/04/17", "tags": [ "search-engine" ] }, { "pattern": "Golfe\\/", "url": "https://datadome.co/bots/6hphjxgx/", "instances": [ "Mozilla/7.0 (compatible; Golfe/1.1; +http://www.goo-olfe.ae/bot.html)" ], "description": "UAE-based search engine crawler indexing web content for search", "addition_date": "2026/04/17", "tags": [ "search-engine" ] }, { "pattern": "Google-Apps-Script", "url": "https://script.google.com/", "instances": [ "Mozilla/5.0 (compatible; Google-Apps-Script; beanserver; +https://script.google.com; id: UAEmdDd_XLoqpGxtGfu5uvUaQlW77VLYz-w)" ], "description": "Google scripting platform fetching URLs for automation workflows", "addition_date": "2026/04/17", "tags": [ "seo" ] }, { "pattern": "GoogleStackdriverMonitoring", "url": "https://cloud.google.com/monitoring", "instances": [ "GoogleStackdriverMonitoring-UptimeChecks(https://cloud.google.com/monitoring)" ], "description": "Google Cloud monitoring service performing uptime checks on endpoints", "addition_date": "2026/04/17", "tags": [ "monitoring" ] }, { "pattern": "GoogleAssociationService\\/", "url": "https://developers.google.com/identity/credential-sharing/digital-asset-links#:~:text=then%20act%20upon.-,Overview,as%20location%2C%20with%20website%20B.", "instances": [ "GoogleAssociationService/" ], "description": "Google service verifying digital asset links between apps and websites", "addition_date": "2026/04/17", "tags": [ "seo" ] }, { "pattern": "GoogleImageProxy", "url": "https://support.google.com/webmasters/answer/1061943?hl=en", "instances": [ "Mozilla/5.0 (Windows NT 5.1; rv:11.0) Gecko Firefox/11.0 (via ggpht.com GoogleImageProxy)" ], "description": "Google image proxy fetching and caching images for Gmail display", "addition_date": "2026/04/17", "tags": [ "social-preview" ] }, { "pattern": "GoogleProducer", "url": "https://developers.google.com/search/docs/crawling-indexing/google-user-triggered-fetchers#googleproducer", "instances": [ "GoogleProducer; (+https://developers.google.com/search/docs/crawling-indexing/google-producer)" ], "description": "Google user-triggered fetcher retrieving content for Google products", "addition_date": "2026/04/17", "tags": [ "search-engine" ] }, { "pattern": "Googlebot-IA\\/", "url": "https://scholar.google.com/intl/en/scholar/libraries.html", "instances": [ "Googlebot-IA/2.1" ], "description": "Google Scholar crawler indexing academic papers and library content", "addition_date": "2026/04/17", "tags": [ "search-engine", "academic" ] }, { "pattern": "Google-Trust-Services\\/", "url": "https://pki.goog/", "instances": [ "Mozilla/5.0 (compatible; Google-Trust-Services/2.0; http://pki.goog/)" ], "description": "Google PKI service validating SSL certificates and checking revocation status", "addition_date": "2026/04/17", "tags": [ "scanner" ] }, { "pattern": "Google-Area120", "url": "https://area120.google.com/", "instances": [ "Google-Area120-PrivacyPolicyFetcher" ], "description": "Google Area 120 experimental tool fetching privacy policies from websites", "addition_date": "2026/04/17", "tags": [ "seo" ] }, { "pattern": "Google-CloudVertexBot", "url": "https://developers.google.com/search/docs/crawling-indexing/google-common-crawlers#google-cloudvertexbot", "instances": [ "Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/W.X.Y.Z Mobile Safari/537.36 (compatible; Google-CloudVertexBot; +https://cloud.google.com/enterprise-search)", "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Google-CloudVertexBot; +https://cloud.google.com/enterprise-search) Chrome/W.X.Y.Z Safari/537.36" ], "description": "Google Vertex AI crawler indexing content for enterprise search solutions", "addition_date": "2026/04/17", "tags": [ "search-engine", "ai-crawler" ] }, { "pattern": "GoogleAssociationService$", "url": "https://developers.google.com/digital-asset-links", "instances": [ "GoogleAssociationService" ], "description": "Google service verifying digital asset link associations between platforms", "addition_date": "2026/04/17", "tags": [ "seo" ] }, { "pattern": "GoogleDocs", "url": "https://docs.google.com/", "instances": [ "Mozilla/5.0 (compatible; GoogleDocs; apps-spreadsheets; +http://docs.google.com)" ], "description": "Google Docs crawler fetching web content for spreadsheet import functions", "addition_date": "2026/04/17", "tags": [ "seo" ] }, { "pattern": "GoPay", "url": "https://doc.gopay.com/", "instances": [ "GoPay" ], "description": "Payment gateway service verifying merchant websites and payment endpoints", "addition_date": "2026/04/17", "tags": [ "seo" ] }, { "pattern": "GotSiteMonitor\\.com", "url": "https://datadome.co/bots/gotsitemonitor/", "instances": [ "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.11 (KHTML, like Gecko) Chrome/23.0.1271.64 Safari/537.11 GotSiteMonitor.com" ], "description": "Website monitoring service tracking uptime and alerting on downtime", "addition_date": "2026/04/17", "tags": [ "monitoring" ] }, { "pattern": "synthetic-monitoring-agent\\/", "url": "https://grafana.com/", "instances": [ "synthetic-monitoring-agent/v0.20.1-0-g66af84ad (linux amd64; 66af84ad6a8755895d4b69606281ca7354c1589a; 2024-02-12 16:50:07+00:00; +https://github.com/grafana/synthetic-monitoring-agent)" ], "description": "Grafana synthetic monitoring agent probing endpoints for availability", "addition_date": "2026/04/17", "tags": [ "monitoring" ] }, { "pattern": "Grammarly\\/", "url": "https://datadome.co/bots/grammarly/", "instances": [ "Grammarly/1.0 (http://www.grammarly.com)" ], "description": "Writing assistant crawler fetching web content for grammar analysis", "addition_date": "2026/04/17", "tags": [ "seo" ] }, { "pattern": "gregcrawler", "url": "https://datadome.co/bots/gregcrawler/", "instances": [ "gregcrawler" ], "description": "Web crawler indexing content for data aggregation and analysis", "addition_date": "2026/04/17", "tags": [ "seo" ] }, { "pattern": "GroovinaAdsbot\\/", "url": "https://www.groovinads.com/", "instances": [ "Mozilla/5.0 (compatible; GroovinaAdsbot/1.0; +https://www.groovinads.com/en/#bot)" ], "description": "Advertising platform crawler indexing products and pricing for campaigns", "addition_date": "2026/04/17", "tags": [ "advertising" ] }, { "pattern": "Grover\\/", "url": "https://datadome.co/bots/grover-bot/", "instances": [ "Grover/Grover-1.18 (Web Crawler)" ], "description": "Anonymous web crawler likely used for research or content scraping", "addition_date": "2026/04/17", "tags": [ "seo" ] }, { "pattern": "GTmetrix", "url": "https://gtmetrix.com/", "instances": [ "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/117.0.0.0 Safari/537.36 GTmetrix" ], "description": "Performance testing tool analyzing page speed and optimization recommendations", "addition_date": "2026/04/17", "tags": [ "monitoring" ] }, { "pattern": "GuestpostsBot\\/", "url": "https://guestposts.com.br/", "instances": [ "Mozilla/5.0 (compatible; GuestpostsBot/2.0; +https://guestposts.com.br/blog/robot/)" ], "description": "Guest posting platform crawler discovering and indexing blog opportunities", "addition_date": "2026/04/17", "tags": [ "seo" ] }, { "pattern": "Gulper Web Bot", "url": "https://datadome.co/bots/gulperbot/", "instances": [ "Gulper Web Bot 0.2.4 (www.ecsl.cs.sunysb.edu/~maxim/cgi-bin/Link/GulperBot)" ], "description": "Academic web crawler collecting data for research and link analysis", "addition_date": "2026/04/17", "tags": [ "academic" ] }, { "pattern": "Verity\\/", "url": "https://gumgum.com/verity", "instances": [ "Verity/1.1 (https://gumgum.com/verity; verity-support@gumgum.com)" ], "description": "Contextual advertising crawler analyzing page content for brand safety", "addition_date": "2026/04/17", "tags": [ "advertising" ] }, { "pattern": "HappyWing", "url": "https://datadome.co/bots/happywing/", "instances": [ "HappyWing" ], "description": "Web crawler indexing content for data aggregation and analysis", "addition_date": "2026/04/17", "tags": [ "seo" ] }, { "pattern": "harsilbot\\/", "url": "http://www.harsil.com/bot", "instances": [ "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36 (compatible; harsilbot/1.1; +http://www.harsil.com/bot)" ], "description": "Web crawler indexing and collecting data for SEO analysis purposes", "addition_date": "2026/04/17", "tags": [ "seo" ] }, { "pattern": "HawaiiBot", "url": "https://datadome.co/bots/hawaiibot/", "instances": [ "HawaiiBot" ], "description": "Web crawler gathering content for data aggregation and indexing", "addition_date": "2026/04/17", "tags": [ "seo" ] }, { "pattern": "hCardValidator", "url": "http://hcard.geekhood.net/", "instances": [ "hCardValidator/1 PHP/5 (http://hcard.geekhood.net)" ], "description": "Microformat validator checking hCard contact data on web pages", "addition_date": "2026/04/17", "tags": [ "seo" ] }, { "pattern": "Hello World", "url": "https://datadome.co/bots/hello-world/", "instances": [ "Hello World" ], "description": "Test crawler sending generic requests to verify web server responses", "addition_date": "2026/04/17", "tags": [ "seo" ] }, { "pattern": "HelloworkJobPostingBot\\/", "url": "https://www.hellowork-group.com/en/", "instances": [ "HelloworkJobPostingBot/1.0" ], "description": "Job board crawler indexing employment listings and posting data", "addition_date": "2026/04/17", "tags": [ "seo" ] }, { "pattern": "HetrixTools", "url": "https://hetrixtools.com/uptime-monitoring-bot.html", "instances": [ "HetrixTools Uptime Monitoring Bot. https://hetrix.tools/uptime-monitoring-bot.html" ], "description": "Uptime monitoring bot checking server availability and alerting downtime", "addition_date": "2026/04/17", "tags": [ "monitoring" ] }, { "pattern": "HIFIBot\\/", "url": "https://hi.fi/", "instances": [ "Mozilla/5.0 (compatible; HIFIBot/1; +https://hi.fi)" ], "description": "Web crawler indexing content for search and information aggregation", "addition_date": "2026/04/17", "tags": [ "seo" ] }, { "pattern": "Hlidam\\.to robot", "url": "https://hlidam.to/", "instances": [ "Hlidam.to robot" ], "description": "Czech job listing crawler indexing employment offers and postings", "addition_date": "2026/04/17", "tags": [ "seo" ] }, { "pattern": "Honeybadger Uptime Check", "url": "https://www.honeybadger.io/", "instances": [ "Honeybadger Uptime Check" ], "description": "Error monitoring service checking website availability and health", "addition_date": "2026/04/17", "tags": [ "monitoring" ] }, { "pattern": "HostTracker\\/", "url": "https://datadome.co/bots/hosttracker/", "instances": [ "Mozilla/5.0 (compatible;HostTracker/2.0;+http://www.host-tracker.com/)" ], "description": "Website monitoring service checking availability from multiple global locations", "addition_date": "2026/04/17", "tags": [ "monitoring" ] }, { "pattern": "Hotjar", "url": "https://www.hotjar.com/", "instances": [ "Mozilla/5.0 (iPhone; CPU iPhone OS 11_3 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Hotjar Version/11.0 Mobile/15E148 Safari/604.1" ], "description": "Analytics platform crawler collecting heatmap and behavior tracking data", "addition_date": "2026/04/17", "tags": [ "seo" ] }, { "pattern": "hstspreload-bot", "url": "https://hstspreload.org/", "instances": [ "hstspreload-bot" ], "description": "HSTS preload list checker verifying HTTPS enforcement across websites", "addition_date": "2026/04/17", "tags": [ "scanner" ] }, { "pattern": "Huckabot\\/", "url": "https://huckabuy.com/", "instances": [ "Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2272.96 Mobile Safari/537.36 (compatible; Huckabot/0.0; +https://huckabuy.com/)" ], "description": "E-commerce SEO crawler analyzing product pages for optimization insights", "addition_date": "2026/04/17", "tags": [ "seo" ] }, { "pattern": "Hype Machine\\/", "url": "https://hypem.com/latest", "instances": [ "Hype Machine/4.0 hypem.com (anthony@hypem.com)" ], "description": "Music blog aggregator crawler indexing tracks and artist content", "addition_date": "2026/04/17", "tags": [ "seo" ] }, { "pattern": "Web Screen Service By hyperhost", "url": "https://datadome.co/bots/hyperhost-ua-crawler/", "instances": [ "Web Screen Service By hyperhost.ua" ], "description": "Web screenshot service capturing page visuals for hosting platform", "addition_date": "2026/04/17", "tags": [ "seo" ] }, { "pattern": "AdsBot-IAB", "url": "https://iabtechlab.com/ads-txt/", "instances": [ "AdsBot-IAB" ], "description": "IAB technology crawler verifying ads.txt files for programmatic advertising", "addition_date": "2026/04/17", "tags": [ "advertising" ] }, { "pattern": "iAskBot\\/", "url": "https://datadome.co/bots/iaskspider/", "instances": [ "Mozilla/5.0 AppleWebKit/605.1.15 (KHTML, like Gecko; compatible; iAskBot/1.0; +https://iask.ai/) Chrome/120.0.6099.119 Safari/605.1.15" ], "description": "AI search engine crawler indexing web content for question answering", "addition_date": "2026/04/17", "tags": [ "search-engine", "ai-crawler" ] }, { "pattern": "IBM Crawler", "url": "https://datadome.co/bots/ibm-crawler/", "instances": [ "IBM Crawler" ], "description": "IBM enterprise crawler indexing content for Watson and cloud services", "addition_date": "2026/04/17", "tags": [ "seo" ] }, { "pattern": "IFTTT\\/", "url": "https://ifttt.com/feed/details", "instances": [ "IFTTT/1.0 (https://ifttt.com/support)" ], "description": "Automation platform crawler fetching feeds for conditional workflow triggers", "addition_date": "2026/04/17", "tags": [ "seo" ] }, { "pattern": "ImageFetcher\\/", "url": "http://wsrv.nl/", "instances": [ "Mozilla/5.0 (compatible; ImageFetcher/9.0; +http://wsrv.nl/)" ], "description": "Image proxy service fetching and caching images for web applications", "addition_date": "2026/04/17", "tags": [ "seo" ] }, { "pattern": "ImageMind", "url": "https://datadome.co/bots/imagemind/", "instances": [ "ImageMind" ], "description": "Image analysis crawler collecting visual data for AI training purposes", "addition_date": "2026/04/17", "tags": [ "ai-crawler" ] }, { "pattern": "img2dataset", "url": "https://github.com/rom1504/img2dataset", "instances": [ "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:72.0) Gecko/20100101 Firefox/72.0 (compatible; img2dataset; +https://github.com/rom1504/img2dataset)" ], "description": "Open source tool downloading images for machine learning dataset creation", "addition_date": "2026/04/17", "tags": [ "ai-crawler" ] }, { "pattern": "imgproxy\\/", "url": "https://imgproxy.net/", "instances": [ "imgproxy/3.21.0" ], "description": "Image processing proxy resizing and optimizing images for web delivery", "addition_date": "2026/04/17", "tags": [ "seo" ] }, { "pattern": "impendoom-bot\\/", "url": "https://impendoom.com/", "instances": [ "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; impendoom-bot/0.1.0; +https://impendoom.com/) Safari/537.36" ], "description": "Web crawler indexing content for competitive intelligence and data analysis", "addition_date": "2026/04/17", "tags": [ "seo" ] }, { "pattern": "IndeedJobBot", "url": "https://www.indeed.com/about/indeed-crawlers", "instances": [ "IndeedJobBot" ], "description": "Job search engine crawler indexing employment listings and company pages", "addition_date": "2026/04/17", "tags": [ "search-engine" ] }, { "pattern": "Innguma\\/", "url": "https://factory.innguma.com/fetcher/", "instances": [ "Innguma/1.0 (+https://factory.innguma.com/fetcher/)" ], "description": "Content fetcher crawler gathering web data for information extraction", "addition_date": "2026/04/17", "tags": [ "seo" ] }, { "pattern": "Instapaper\\/", "url": "https://www.instapaper.com/publishers", "instances": [ "Instapaper/4.0Instaparser/1.0" ], "description": "Read-later service crawler fetching articles for offline reading storage", "addition_date": "2026/04/17", "tags": [ "seo" ] }, { "pattern": "Integromat\\/", "url": "https://developers.make.com/api-documentation", "instances": [ "Integromat/production" ], "description": "Automation platform crawler fetching web data for workflow integration", "addition_date": "2026/04/17", "tags": [ "seo" ] }, { "pattern": "intelx\\.io_bot", "url": "https://intelx.io/", "instances": [ "Mozilla/5.0 (compatible; intelx.io_bot +https://intelx.io)" ], "description": "Intelligence search engine crawler indexing leaks and dark web content", "addition_date": "2026/04/17", "tags": [ "search-engine" ] }, { "pattern": "internetVista monitor", "url": "https://datadome.co/bots/internetvista-monitor/", "instances": [ "internetVista monitor (Mozilla compatible)" ], "description": "Website monitoring service checking availability performance and SSL status", "addition_date": "2026/04/17", "tags": [ "monitoring" ] }, { "pattern": "Irokez\\.cz monitoring", "url": "https://datadome.co/bots/irokez-cz-monitoring/", "instances": [ "Irokez.cz monitoring v1.2 - (http://www.irokez.cz, Irokez.cz, crawl)" ], "description": "Czech web monitoring service tracking availability and performance metrics", "addition_date": "2026/04/17", "tags": [ "monitoring" ] }, { "pattern": "IsDownBot\\/", "url": "https://help.isdown.app/custom-monitors/isdownbot", "instances": [ "Mozilla/5.0 (compatible; IsDownBot/1.0;)" ], "description": "Downtime monitoring bot checking website and service availability status", "addition_date": "2026/04/17", "tags": [ "monitoring" ] }, { "pattern": "ISSCyberRiskCrawler\\/", "url": "https://datadome.co/bots/isscyberriskcrawler/", "instances": [ "Mozilla/5.0 (Macintosh; Intel Mac OS X 13_0_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/107.0.0.0 Safari/537.36 ISSCyberRiskCrawler/1.1.4" ], "description": "Cybersecurity risk crawler scanning websites for vulnerabilities and exposures", "addition_date": "2026/04/17", "tags": [ "scanner" ] }, { "pattern": "iubenda-radar\\/", "url": "https://www.iubenda.com/", "instances": [ "iubenda-radar/2.25.2" ], "description": "Compliance platform crawler auditing privacy policies and cookie consent", "addition_date": "2026/04/17", "tags": [ "scanner" ] }, { "pattern": "UptimeBot\\/", "url": "https://jaggedpixel.co/", "instances": [ "Mozilla/5.0 (compatible; UptimeBot/1.0; +https://www.getuptime.co)" ], "description": "Uptime monitoring service verifying website availability and response latency", "addition_date": "2026/04/17", "tags": [ "monitoring" ] }, { "pattern": "jetmon\\/", "url": "https://automattic.com/", "instances": [ "jetmon/1.0 (Jetpack Site Uptime Monitor by WordPress.com)" ], "description": "WordPress Jetpack site uptime monitor and health checking service", "addition_date": "2026/04/17", "tags": [ "monitoring" ] }, { "pattern": "jobswithgptcom-bot", "url": "https://jobswithgpt.com/bot.html", "instances": [ "jobswithgptcom-bot" ], "description": "Job portal crawler indexing career pages and employment listings", "addition_date": "2026/04/17", "tags": [ "search-engine" ] }, { "pattern": "Jumio", "url": "https://github.com/Jumio/implementation-guides/blob/master/netverify/callback.md", "instances": [ "Jumio" ], "description": "Identity and KYC verification service web content analyzer", "addition_date": "2026/04/17", "tags": [ "seo" ] }, { "pattern": "Kagibot\\/", "url": "https://kagi.com/", "instances": [ "Mozilla/5.0 (compatible; Kagibot/1.0; +https://kagi.com/bot)" ], "description": "Privacy-focused search engine crawler indexing content for rankings", "addition_date": "2026/04/17", "tags": [ "search-engine" ] }, { "pattern": "KangarooBot\\/", "url": "https://datadome.co/bots/kangaroo-bot/", "instances": [ "Mozilla/5.0 (compatible; KangarooBot/1.0; +https://kangaroobot.com)" ], "description": "Data scraping and content aggregation crawler mimicking human behavior", "addition_date": "2026/04/17", "tags": [ "seo" ] }, { "pattern": "KargoBot-Artemis", "url": "https://www.kargo.com/", "instances": [ "Mozilla/5.0 (iPhone; CPU iPhone OS 13_2_3 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/13.0.3 Mobile/15E148 Safari/604.1 (compatible; KargoBot-Artemis-Mobile; +https://www.kargo.com/kargobot-artemis)" ], "description": "Ad-tech crawler analyzing pages for contextual targeting and brand safety", "addition_date": "2026/04/17", "tags": [ "advertising" ] }, { "pattern": "kazbtbot\\/", "url": "http://kazbt.com/", "instances": [ "Mozilla/5.0 (compatible; kazbtbot/0.1; +http://kazbt.com/)" ], "description": "Web crawler for content indexing and data aggregation purposes", "addition_date": "2026/04/17", "tags": [ "seo" ] }, { "pattern": "keycdn-tools\\/", "url": "https://tools.keycdn.com/", "instances": [ "keycdn-tools/speed" ], "description": "CDN performance monitoring measuring website load times and metrics", "addition_date": "2026/04/17", "tags": [ "monitoring" ] }, { "pattern": "keys-so-bot", "url": "https://datadome.co/bots/keys-so-bot/", "instances": [ "Mozilla/5.0 (keys-so-bot)" ], "description": "Automated crawler for indexing and data collection purposes", "addition_date": "2026/04/17", "tags": [ "seo" ] }, { "pattern": "kinsta-bot", "url": "https://kinsta.com/", "instances": [ "kinsta-bot" ], "description": "Hosting platform crawler for cache warming and SEO audit checks", "addition_date": "2026/04/17", "tags": [ "seo" ] }, { "pattern": "Klaviyo\\/", "url": "https://www.klaviyo.com/", "instances": [ "Klaviyo/1.0" ], "description": "Marketing automation bot fetching product data for email campaigns", "addition_date": "2026/04/17", "tags": [ "seo" ] }, { "pattern": "Kukei\\.eu-Bot\\/", "url": "https://kukei.eu/", "instances": [ "Mozilla/5.0 (compatible; Kukei.eu-Bot/0.2; +https://kukei.eu)" ], "description": "Web crawler indexing and analyzing page content for search purposes", "addition_date": "2026/04/17", "tags": [ "seo" ] }, { "pattern": "LAC_IAHarvester", "url": "https://library-archives.canada.ca/eng/services/government-canada/web-social-media-preservation-program/Pages/web-archive.aspx", "instances": [ "Mozilla/5.0 (compatible; LAC_IAHarvester/3.3.0; +https://library-archives.canada.ca/eng/services/government-canada/web-social-media-preservation-program/Pages/web-archive.aspx)" ], "description": "Canadian library web archiving bot preserving government websites", "addition_date": "2026/04/17", "tags": [ "archiver" ] }, { "pattern": "LastModBot\\/", "url": "https://last-modified.com/", "instances": [ "Mozilla/5.0 (compatible; LastModBot/0.1; +https://last-modified.com/en)" ], "description": "Web crawler detecting page freshness by analyzing last modified headers", "addition_date": "2026/04/17", "tags": [ "seo" ] }, { "pattern": "LegalMonster", "url": "https://www.legalmonster.com/", "instances": [ "LegalMonster" ], "description": "Compliance monitoring scanner detecting cookies and tracking consent", "addition_date": "2026/04/17", "tags": [ "scanner" ] }, { "pattern": "Let's Encrypt", "url": "https://letsencrypt.org/", "instances": [ "Mozilla/5.0 (compatible; Let's Encrypt validation server; +https://www.letsencrypt.org)" ], "description": "SSL certificate validation bot verifying domain control for TLS certs", "addition_date": "2026/04/17", "tags": [ "scanner" ] }, { "pattern": "Level9SearchBot\\/", "url": "https://level9.com/", "instances": [ "Mozilla/5.0 (compatible; Level9SearchBot/2-1.5)" ], "description": "Search and OSINT crawler indexing public content for search purposes", "addition_date": "2026/04/17", "tags": [ "search-engine" ] }, { "pattern": "loc\\.gov\\/programs\\/web-archiving", "url": "https://www.loc.gov/programs/web-archiving/for-site-owners", "instances": [ "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.97 Safari/537.36 (+https://www.loc.gov/programs/web-archiving/for-site-owners/)" ], "description": "Government web archiver preserving cultural content for long-term access", "addition_date": "2026/04/17", "tags": [ "archiver" ] }, { "pattern": "LinerBot\\/", "url": "https://docs.getliner.com/docs/linerbot", "instances": [ "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; LinerBot/1.0; +https://docs.getliner.com/docs/linerbot)" ], "description": "AI workspace crawler indexing web content for search and discovery", "addition_date": "2026/04/17", "tags": [ "ai-crawler" ] }, { "pattern": "LinkTiger", "url": "https://linktiger.com/", "instances": [ "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/76.0.3809.87 Safari/537.36 LinkTiger" ], "description": "Broken link detection crawler auditing website link integrity", "addition_date": "2026/04/17", "tags": [ "seo" ] }, { "pattern": "LinkAce\\/", "url": "https://www.linkace.org/", "instances": [ "LinkAce/1 (https://github.com/Kovah/LinkAce)" ], "description": "Self-hosted bookmark archive crawler preserving web content", "addition_date": "2026/04/17", "tags": [ "archiver" ] }, { "pattern": "LinksIndexerBot\\/", "url": "https://linksindexer.com/bot", "instances": [ "Mozilla/5.0 (compatible; LinksIndexerBot/1.0; +http://linksindexer.com/bot)" ], "description": "SEO crawler indexing backlinks and submitting URLs for search engines", "addition_date": "2026/04/17", "tags": [ "seo" ] }, { "pattern": "LinkWalker\\/", "url": "https://datadome.co/bots/linkwalker/", "instances": [ "Mozilla/5.0 (compatible; LinkWalker/2.0; +http://www.seventwentyfour.com)" ], "description": "Automated web crawler for content and data collection purposes", "addition_date": "2026/04/17", "tags": [ "seo" ] }, { "pattern": "LogicMonitor", "url": "https://www.logicmonitor.com/support/about-logicmonitor/overview/logicmonitor-public-ip-addresses-dns-names", "instances": [ "LogicMonitor SiteMonitor/1.0" ], "description": "Synthetic monitoring bot checking website and API availability", "addition_date": "2026/04/17", "tags": [ "monitoring" ] }, { "pattern": "LoomlyBot", "url": "https://www.loomly.com/", "instances": [ "LoomlyBot" ], "description": "Social media management metadata fetcher for link previews", "addition_date": "2026/04/17", "tags": [ "social-preview" ] }, { "pattern": "Macrobondbot", "url": "https://www.macrobond.com/", "instances": [ "Mozilla/5.0 (compatible; Macrobondbot +http://redir.macrobond.com/go/bot/)" ], "description": "Financial data crawler gathering macroeconomic time-series data", "addition_date": "2026/04/17", "tags": [ "seo" ] }, { "pattern": "MADBbot\\/", "url": "https://madb.zapto.org/bot.html", "instances": [ "MADBbot/0.1 (Gathering webpages for data analytics; https://madb.zapto.org/bot.html; ma-db-crawl@googlegroups.com)" ], "description": "Web crawler gathering data for analytics and research purposes", "addition_date": "2026/04/17", "tags": [ "seo" ] }, { "pattern": "Magellan", "url": "https://datadome.co/bots/magellan/", "instances": [ "Magellan" ], "description": "Web content indexing and competitive analysis crawler tool", "addition_date": "2026/04/17", "tags": [ "seo" ] }, { "pattern": "magicsearchdev\\/", "url": "https://datadome.co/bots/magicsearchdev/", "instances": [ "magicsearchdev/1.0" ], "description": "Web crawler for content indexing and data aggregation purposes", "addition_date": "2026/04/17", "tags": [ "seo" ] }, { "pattern": "Magnet\\.me-web\\/", "url": "https://magnet.me/bot.html", "instances": [ "Magnet.me-web/1.0 (+https://magnet.me/bot.html)" ], "description": "Graduate recruitment platform crawling job postings and employer data", "addition_date": "2026/04/17", "tags": [ "seo" ] }, { "pattern": "MainWP\\/", "url": "https://www.dshost.com.au/", "instances": [ "Mozilla/5.0 (compatible; MainWP/4.2.7.1; +http://mainwp.com)" ], "description": "WordPress management platform crawler auditing child site health", "addition_date": "2026/04/17", "tags": [ "seo" ] }, { "pattern": "Make\\/", "url": "https://www.make.com/en/", "instances": [ "Make/production" ], "description": "Automation platform crawler validating endpoints and collecting metadata", "addition_date": "2026/04/17", "tags": [ "seo" ] }, { "pattern": "ManageWP", "url": "https://managewp.com/", "instances": [ "ManageWP" ], "description": "WordPress dashboard scanner verifying site availability and health", "addition_date": "2026/04/17", "tags": [ "seo" ] }, { "pattern": "MarketGoo\\/", "url": "https://www.marketgoo.com/", "instances": [ "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_5) AppleWebKit/600.1.17 (KHTML, like Gecko) Version/7.1 Safari/537.85.10 MarketGoo/2.1" ], "description": "SEO audit spider evaluating technical search engine optimization signals", "addition_date": "2026/04/17", "tags": [ "seo" ] }, { "pattern": "MarketingMiner", "url": "https://datadome.co/bots/marketingminer-bot/", "instances": [ "Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/W.X.Y.Z Mobile Safari/537.36(compatible; MarketingMiner v2.0; +https://www.marketingminer.com)" ], "description": "Digital marketing analytics crawler gathering SEO and performance data", "addition_date": "2026/04/17", "tags": [ "seo" ] }, { "pattern": "dbot\\)", "url": "https://www.marsflag.com/ja/marsfinder/", "instances": [ "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) (dbot)" ], "description": "Link preview and content discovery metadata harvester bot", "addition_date": "2026/04/17", "tags": [ "social-preview" ] }, { "pattern": "Mattermost-Bot\\/", "url": "https://datadome.co/bots/mattermost-bot/", "instances": [ "Mattermost-Bot/1.1" ], "description": "Team communication platform fetching URLs for preview and automation", "addition_date": "2026/04/17", "tags": [ "social-preview" ] }, { "pattern": "Mavifinds", "url": "https://brandimi.com/mavifinds-bot/", "instances": [ "Mavifinds Bot" ], "description": "Shopping discovery crawler indexing product prices and availability", "addition_date": "2026/04/17", "tags": [ "seo" ] }, { "pattern": "MB-LinkChecker", "url": "https://www.marcobeierer.com/tools", "instances": [ "MB-LinkChecker" ], "description": "Website crawler for structure analysis and metadata collection", "addition_date": "2026/04/17", "tags": [ "seo" ] }, { "pattern": "MedialogiaBot", "url": "https://www.mlg.ru/", "instances": [ "Mozilla/5.0 (compatible; MedialogiaBot; +http://home.prod.mlg.ru/bots.txt)" ], "description": "Media monitoring crawler tracking news and brand mentions online", "addition_date": "2026/04/17", "tags": [ "seo" ] }, { "pattern": "MediaMonitoringBot\\/", "url": "https://mediamonitoringbot.com/", "instances": [ "MediaMonitoringBot/1.1 (+https://mediamonitoringbot.com/crawler; crawler@mediamonitoringbot.com)" ], "description": "OSINT crawler harvesting mentions from news blogs and forums", "addition_date": "2026/04/17", "tags": [ "seo" ] }, { "pattern": "MediavineMetadataParser\\/", "url": "https://radar.cloudflare.com/bots/directory/mediavine-metadata-parser/mediavine.com", "instances": [ "MediavineMetadataParser/7.7.3" ], "description": "Ad network metadata extractor gathering schema and preview data", "addition_date": "2026/04/17", "tags": [ "advertising" ] }, { "pattern": "Pywikibot\\/", "url": "https://www.mediawiki.org/wiki/Manual:Pywikibot", "instances": [ "wlc3 Pywikibot/9.0.0.dev0 (g18371) requests/2.31.0 Python/3.10.12.final.0" ], "description": "MediaWiki crawler indexing wiki content for search and analysis", "addition_date": "2026/04/17", "tags": [ "seo" ] }, { "pattern": "CentComBot\\/", "url": "https://centcom.melonmesa.com/", "instances": [ "Mozilla/5.0 (compatible; CentComBot/" ], "description": "Content indexing and dataset building crawler tool", "addition_date": "2026/04/17", "tags": [ "seo" ] }, { "pattern": "MergadoBot", "url": "https://datadome.co/bots/mergadobot/", "instances": [ "MergadoBot (+http://mergado.cz)" ], "description": "E-commerce product data feed optimization and aggregation crawler", "addition_date": "2026/04/17", "tags": [ "seo" ] }, { "pattern": "Meta-ExternalHit\\/", "url": "https://datadome.co/bots/meta-externalagent/", "instances": [ "Mozilla/5.0 (compatible; Meta-ExternalHit/1.1; +http://www.facebook.com/externalhit_uatext.php)" ], "description": "Meta platform bot fetching link previews and metadata for sharing", "addition_date": "2026/04/17", "tags": [ "social-preview" ] }, { "pattern": "Metorik", "url": "https://metorik.com/", "instances": [ "Metorik API Client/2.0.1" ], "description": "E-commerce analytics platform crawler for WooCommerce and Shopify stores", "addition_date": "2026/04/17", "tags": [ "seo" ] }, { "pattern": "MgidBot", "url": "https://www.mgid.com", "instances": [ "MgidBot 1.0" ], "description": "Native advertising platform crawler analyzing pages for targeting", "addition_date": "2026/04/17", "tags": [ "advertising" ] }, { "pattern": "Miniature\\.io\\/", "url": "https://miniature.io/", "instances": [ "Mozilla/5.0 (X11; Linux x86_64) Miniature.io/6.0 AppleWebKit/537.36 (KHTML, like Gecko) QtWebEngine/5.12.4 Chrome/69.0.3497.128 Safari/537.36" ], "description": "Web page archiving and thumbnail generation bot tool", "addition_date": "2026/04/17", "tags": [ "archiver" ] }, { "pattern": "mirrorweb\\.com", "url": "https://www.mirrorweb.com/", "instances": [ "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.97 Safari/537.36 +https://www.mirrorweb.com" ], "description": "Web archiving platform crawler capturing timestamped website snapshots", "addition_date": "2026/04/17", "tags": [ "archiver" ] }, { "pattern": "MissinglettrBot\\/", "url": "https://missinglettr.com/", "instances": [ "Mozilla/5.0 (compatible; MissinglettrBot/2.0; +http://missinglettr.com/bot/)" ], "description": "Social media automation platform crawler for blog and feed indexing", "addition_date": "2026/04/17", "tags": [ "seo" ] }, { "pattern": "crawler_eb_germany", "url": "https://datadome.co/bots/crawler_eb_germany/", "instances": [ "crawler_eb_germany_2.0" ], "description": "General web crawler for indexing and data collection purposes", "addition_date": "2026/04/17", "tags": [ "seo" ] }, { "pattern": "ModularConnector\\/", "url": "https://uniqoders.com/", "instances": [ "ModularConnector/1.0 (Linux)" ], "description": "Distributed modular crawler framework for content processing", "addition_date": "2026/04/17", "tags": [ "seo" ] }, { "pattern": "Mollie HTTP client", "url": "https://www.mollie.com/", "instances": [ "Mollie HTTP client/1.0" ], "description": "Payment service provider verifying merchant sites and compliance", "addition_date": "2026/04/17", "tags": [ "seo" ] }, { "pattern": "Monibot", "url": "https://datadome.co/bots/monibot/", "instances": [ "Monibot" ], "description": "Web crawler for competitive analysis and price monitoring", "addition_date": "2026/04/17", "tags": [ "seo" ] }, { "pattern": "monitis -", "url": "https://www.monitis.com/", "instances": [ "Mozilla/5.0 (compatible; monitis - premium monitoring service; http://www.monitis.com)" ], "description": "Synthetic monitoring bot checking website availability and performance", "addition_date": "2026/04/17", "tags": [ "monitoring" ] }, { "pattern": "MonitoRSS\\/", "url": "https://monitorss.xyz/", "instances": [ "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:89.0) Gecko/20100101 Firefox/89.0 MonitoRSS/1.0" ], "description": "Discord bot crawler fetching RSS feeds for channel notifications", "addition_date": "2026/04/17", "tags": [ "feed-reader" ] }, { "pattern": "MonSpark\\/", "url": "https://monspark.com/", "instances": [ "Mozilla/5.0+(compatible; MonSpark/1.0; http://www.monspark.com/)" ], "description": "Website monitoring service bot checking uptime and performance", "addition_date": "2026/04/17", "tags": [ "monitoring" ] }, { "pattern": "montastic-monitor", "url": "https://montastic.com/", "instances": [ "montastic-monitor www.montastic.com" ], "description": "Uptime monitoring bot tracking website availability and latency", "addition_date": "2026/04/17", "tags": [ "monitoring" ] }, { "pattern": "MonTools\\.com", "url": "https://datadome.co/bots/montools/", "instances": [ "Mozilla/5.0 (compatible; MonTools.com)" ], "description": "Web crawler for scanning and indexing website pages", "addition_date": "2026/04/17", "tags": [ "seo" ] }, { "pattern": "MotoMinerBot\\/", "url": "https://motominer.com/Bot", "instances": [ "Mozilla/5.0 (compatible; MotoMinerBot/1.0; +https://motominer.com/Bot)" ], "description": "Automotive marketplace crawler indexing vehicle listings and prices", "addition_date": "2026/04/17", "tags": [ "seo" ] }, { "pattern": "MRGbot\\/", "url": "https://www.mrg.ro/", "instances": [ "Mozilla/5.0 (compatible; MRGbot/1.0; +https://www.mrg.ro/bot.html)" ], "description": "Security testing lab crawler for anti-fraud control evaluation", "addition_date": "2026/04/17", "tags": [ "scanner" ] }, { "pattern": "MxToolbox", "url": "https://datadome.co/bots/mxtoolbox-bot/", "instances": [ "Mozilla/5.0+(compatible; MxToolbox/Beta7; http://www.mxtoolbox.com/)" ], "description": "Email delivery and reputation monitoring for organizations", "addition_date": "2026/04/17", "tags": [ "monitoring" ] }, { "pattern": "my-tiny-bot", "url": "https://datadome.co/bots/my-tiny-bot/", "instances": [ "my-tiny-bot" ], "description": "Lightweight web crawler for content indexing and analysis", "addition_date": "2026/04/17", "tags": [ "seo" ] }, { "pattern": "MyBot\\/", "url": "https://datadome.co/bots/mybot/", "instances": [ "Mozilla/5.0 (compatible; MyBot/1.0)" ], "description": "Web crawler for indexing and data aggregation purposes", "addition_date": "2026/04/17", "tags": [ "seo" ] }, { "pattern": "nbertaupete95", "url": "https://datadome.co/bots/nbertaupete95/", "instances": [ "Mozilla/5.0/Firefox/42.0 - nbertaupete95(at)gmail.com" ], "description": "Web crawler for SEO and competitive analysis purposes", "addition_date": "2026/04/17", "tags": [ "seo" ] }, { "pattern": "NetAPI", "url": "https://datadome.co/bots/netapi/", "instances": [ "NetAPI v1" ], "description": "Automated crawler interacting with websites for data collection", "addition_date": "2026/04/17", "tags": [ "seo" ] }, { "pattern": "NetpeakCheckerBot\\/", "url": "https://netpeaksoftware.com/", "instances": [ "Mozilla/5.0 (compatible; NetpeakCheckerBot/3.6; +https://netpeaksoftware.com/checker)" ], "description": "SEO analysis crawler identifying technical issues and metrics", "addition_date": "2026/04/17", "tags": [ "seo" ] }, { "pattern": "NetShelter ContentScan", "url": "https://datadome.co/bots/netshelter-contentscan/", "instances": [ "Mozilla/5.0 (Windows; U; Windows NT 6.1; en-GB; rv:1.9.2.3) Gecko/20100401 Firefox/3.6.3 (NetShelter ContentScan)" ], "description": "Media network crawler categorizing content for ad targeting", "addition_date": "2026/04/17", "tags": [ "advertising" ] }, { "pattern": "NETVIGIE", "url": "https://netvigie.com/", "instances": [ "NETVIGIE" ], "description": "European performance and uptime monitoring synthetic crawler", "addition_date": "2026/04/17", "tags": [ "monitoring" ] }, { "pattern": "NewRelicbot\\/", "url": "https://newrelic.com/", "instances": [ "Mozilla/5.0 (compatible; NewRelicbot/2.1; +http://www.newrelic.com)" ], "description": "Application monitoring bot for uptime and performance checking", "addition_date": "2026/04/17", "tags": [ "monitoring" ] }, { "pattern": "nyt_scraping", "url": "https://int.nyt.com/assets/scraping.json", "instances": [ "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/110.0.0.0 Safari/537.36 nyt_scraping/scraping@nytimes.com" ], "description": "News organization newsgathering crawler for reporting support", "addition_date": "2026/04/17", "tags": [ "seo" ] }, { "pattern": "NewsNow\\/", "url": "https://www.newsnow.co.uk/h/", "instances": [ "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT) NewsNow/1.0" ], "description": "UK news aggregation platform crawler for content discovery", "addition_date": "2026/04/17", "tags": [ "seo" ] }, { "pattern": "NLNZ_IAHarvester", "url": "https://natlib.govt.nz/publishers-and-authors/web-harvesting", "instances": [ "Mozilla/5.0 (compatible; NLNZ_IAHarvester2024/3.3.0; +https://natlib.govt.nz/publishers-and-authors/web-harvesting/domain-harvest)" ], "description": "New Zealand library digital preservation and archiving bot", "addition_date": "2026/04/17", "tags": [ "archiver" ] }, { "pattern": "NodePing", "url": "https://datadome.co/bots/nodeping/", "instances": [ "NodePing" ], "description": "Website monitoring service performing uptime checks globally", "addition_date": "2026/04/17", "tags": [ "monitoring" ] }, { "pattern": "nomore404\\.com robot", "url": "https://datadome.co/bots/nomore404-com-robot/", "instances": [ "Mozilla/5.0 (nomore404.com robot/1.1; +https://nomore404.com/)" ], "description": "Broken link detection crawler identifying HTTP 404 errors", "addition_date": "2026/04/17", "tags": [ "seo" ] }, { "pattern": "noorobot", "url": "https://noordigital.com/", "instances": [ "noorobot" ], "description": "Web scraper performing high-volume programmatic content collection", "addition_date": "2026/04/17", "tags": [ "seo" ] }, { "pattern": "Nooshub\\/", "url": "https://www.nooshub.com/", "instances": [ "Mozilla/5.0 (compatible; Nooshub/1.0; +https://www.nooshub.com/statics/bots)" ], "description": "AI feed platform crawler indexing content and RSS feeds", "addition_date": "2026/04/17", "tags": [ "seo" ] }, { "pattern": "Notabot", "url": "https://corp.helpfeel.com/ja/home", "instances": [ "Notabot" ], "description": "Content aggregation and metadata extraction crawler bot", "addition_date": "2026/04/17", "tags": [ "seo" ] }, { "pattern": "Novaact\\/", "url": "https://datadome.co/bots/novaact/", "instances": [ "Mozilla/5.0 (compatible; Novaact/1.0; +https://www.novaact.com)" ], "description": "Web crawler for data aggregation and analysis purposes", "addition_date": "2026/04/17", "tags": [ "seo" ] }, { "pattern": "Novellum", "url": "https://crawl.corp.novellum.ai/docs", "instances": [ "Novellum" ], "description": "AI data collection crawler for model training and enrichment", "addition_date": "2026/04/17", "tags": [ "ai-crawler" ] }, { "pattern": "NsToolsBot\\/", "url": "https://ns.tools/", "instances": [ "NsToolsBot/analyse" ], "description": "Network and security analysis tool crawler gathering server data", "addition_date": "2026/04/17", "tags": [ "scanner" ] }, { "pattern": "nvdorz", "url": "https://datadome.co/bots/nvdorz/", "instances": [ "nvdorz" ], "description": "Web crawler for content indexing and analysis purposes", "addition_date": "2026/04/17", "tags": [ "seo" ] }, { "pattern": "Odin;", "url": "https://docs.getodin.com/", "instances": [ "Mozilla/5.0 (compatible; Odin; https://docs.getodin.com/)" ], "description": "Automated crawler for data collection and indexing purposes", "addition_date": "2026/04/17", "tags": [ "seo" ] }, { "pattern": "Offline Explorer", "url": "https://datadome.co/bots/offline-explorer/", "instances": [ "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.0; Offline Explorer; MSIECrawler)" ], "description": "Website downloader tool for offline archival and content analysis", "addition_date": "2026/04/17", "tags": [ "archiver" ] }, { "pattern": "OhDear\\/", "url": "https://ohdear.app/docs/faq/what-is-the-oh-dear-checker", "instances": [ "Mozilla/5.0 (compatible; OhDear/1.1; +https://ohdear.app/checkerOhDear.app (+https://ohdear.app/docs/checks/uptime)" ], "description": "Website monitoring bot scanning uptime and TLS certificate validity", "addition_date": "2026/04/17", "tags": [ "monitoring" ] }, { "pattern": "Omnisend\\/", "url": "https://bots.omnisend.io/cf.txt", "instances": [ "Omnisend/1.0" ], "description": "Ecommerce marketing automation crawler fetching campaign URLs metadata", "addition_date": "2026/04/17", "tags": [ "seo" ] }, { "pattern": "Online Domain Tools", "url": "https://datadome.co/bots/online-domain-tools/", "instances": [ "Mozilla/5.0 (compatible; Online Domain Tools - Server Monitor/1.0; +http://server-monitoring.online-domain-tools.com)" ], "description": "SEO analysis crawler identifying technical issues and metrics", "addition_date": "2026/04/17", "tags": [ "seo" ] }, { "pattern": "WebCEO Online\\/", "url": "https://datadome.co/bots/online-webceo-bot/", "instances": [ "Mozilla/5.0 (compatible; WebCEO Online/1.0; +http://online.webceo.com)" ], "description": "SEO platform crawler analyzing keywords and backlinks", "addition_date": "2026/04/17", "tags": [ "seo" ] }, { "pattern": "OnlineOrNot\\.com_bot", "url": "https://onlineornot.com", "instances": [ "OnlineOrNot.com_bot_1.0_(https://onlineornot.com)" ], "description": "Web crawler for systematic content browsing and indexing", "addition_date": "2026/04/17", "tags": [ "monitoring" ] }, { "pattern": "OpenGraph\\.io\\/", "url": "https://www.opengraph.io/", "instances": [ "Mozilla/5.0 (compatible; OpenGraph.io/1.1; +https://opengraph.io/ AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.103 Safari/537.36" ], "description": "Open Graph metadata extractor for link preview generation", "addition_date": "2026/04/17", "tags": [ "social-preview" ] }, { "pattern": "OpenRSS", "url": "https://openrss.org/", "instances": [ "OpenRSS" ], "description": "Feed discovery crawler converting websites into RSS feeds", "addition_date": "2026/04/17", "tags": [ "feed-reader" ] }, { "pattern": "OpenVAS", "url": "https://openvas.org/", "instances": [ "Mozilla/5.0 [en] (X11, U; OpenVAS)" ], "description": "Vulnerability scanner crawler identifying security weaknesses", "addition_date": "2026/04/17", "tags": [ "scanner" ] }, { "pattern": "Owler \\(ows\\.eu", "url": "https://ows.eu/owler", "instances": [ "Owler (ows.eu/owler)" ], "description": "European open search crawler for decentralized indexing", "addition_date": "2026/04/17", "tags": [ "search-engine" ] }, { "pattern": "Operator\\/", "url": "https://datadome.co/bots/operator/", "instances": [ "Mozilla/5.0 (compatible; Operator/1.0; +https://www.example.com/bot.html)" ], "description": "Automated crawler performing repetitive internet tasks", "addition_date": "2026/04/17", "tags": [ "seo" ] }, { "pattern": "Orbbot\\/", "url": "https://datadome.co/bots/orbbot/", "instances": [ "Mozilla/5.0 (compatible; Orbbot/1.1;)" ], "description": "Web crawler indexing content for SEO and analysis", "addition_date": "2026/04/17", "tags": [ "seo" ] }, { "pattern": "zebra-v2-bot", "url": "https://datadome.co/bots/zebra-v2-bot/", "instances": [ "zebra-v2-bot" ], "description": "Web crawler for indexing and data collection purposes", "addition_date": "2026/04/17", "tags": [ "seo" ] }, { "pattern": "Orlo-LinkPreview\\/", "url": "https://orlo.tech/", "instances": [ "Orlo-LinkPreview/1.0" ], "description": "Social platform link preview and metadata fetcher", "addition_date": "2026/04/17", "tags": [ "social-preview" ] }, { "pattern": "Cozi-iCalendar-FeedReader", "url": "https://ourfamilywizard.com/", "instances": [ "Cozi-iCalendar-FeedReader" ], "description": "Co-parenting platform crawler fetching shared content metadata", "addition_date": "2026/04/17", "tags": [ "feed-reader" ] }, { "pattern": "OutsellURLValidator", "url": "https://www.outsell.com/", "instances": [ "OutsellURLValidator" ], "description": "Digital marketing crawler validating campaign link health", "addition_date": "2026/04/17", "tags": [ "seo" ] }, { "pattern": "Overcast\\/", "url": "https://overcast.fm/", "instances": [ "Overcast/1.0 Podcast Sync (123 subscribers; feed-id=456789; +http://overcast.fm/)" ], "description": "Podcast platform crawler discovering and refreshing feeds", "addition_date": "2026/04/17", "tags": [ "feed-reader" ] }, { "pattern": "PRTGCloudBot\\/", "url": "https://www.paessler.com/", "instances": [ "Mozilla/5.0 (compatible; PRTGCloudBot/1.0; +http://www.paessler.com/prtgcloudbot; for_[edf7e62f223b268942b7efa36b6be1e305fcdadb])" ], "description": "Paessler cloud monitoring bot checking uptime and health", "addition_date": "2026/04/17", "tags": [ "monitoring" ] }, { "pattern": "Pagespeed\\/", "url": "http://www.pagespeed.de/", "instances": [ "Mozilla/5.0 (compatible; Pagespeed/1.1 Fetcher; +http://www.pagespeed.de)" ], "description": "Performance analysis crawler assessing web page metrics", "addition_date": "2026/04/17", "tags": [ "monitoring" ] }, { "pattern": "PanguBot", "url": "https://datadome.co/bots/pangubot/", "instances": [ "Mozilla/5.0 (compatible; PanguBot +https://www.pangutech.com)" ], "description": "Chinese search engine crawler indexing websites", "addition_date": "2026/04/17", "tags": [ "search-engine" ] }, { "pattern": "Panopta", "url": "https://www.panopta.com/", "instances": [ "Panopta v1.1" ], "description": "Synthetic monitoring bot verifying uptime and transactions", "addition_date": "2026/04/17", "tags": [ "monitoring" ] }, { "pattern": "Paqlebot\\/", "url": "http://www.paqle.dk/about/paqlebot", "instances": [ "Mozilla/5.0 (compatible; Paqlebot/2.0; +http://www.paqle.dk/about/paqlebot)" ], "description": "Web crawler gathering data for analytics and research", "addition_date": "2026/04/17", "tags": [ "seo" ] }, { "pattern": "parse\\.ly scraper\\/", "url": "https://www.parse.ly/", "instances": [ "Mozilla/5.0 (compatible; parse.ly scraper/0.16; +http://parsely.com)" ], "description": "Content analytics platform crawler for metadata extraction", "addition_date": "2026/04/17", "tags": [ "seo" ] }, { "pattern": "PayPal\\/", "url": "https://www.paypal.com/ipn", "instances": [ "PayPal/AUHR-214.0-51787073PayPal IPN ( https://www.paypal.com/ipn )" ], "description": "Payment processor bot validating endpoints and compliance", "addition_date": "2026/04/17", "tags": [ "seo" ] }, { "pattern": "PDF24 URL To PDF", "url": "https://tools.pdf24.org/webpage-to-pdf", "instances": [ "PDF24 URL To PDF" ], "description": "PDF conversion crawler capturing web pages as documents", "addition_date": "2026/04/17", "tags": [ "seo" ] }, { "pattern": "PingAdmin\\.Ru\\/", "url": "http://ping-admin.ru/", "instances": [ "Mozilla/5.0 (compatible; PingAdmin.Ru/1.2; +http://pingadmin.ru/free_test/)" ], "description": "Uptime monitoring service bot checking website availability", "addition_date": "2026/04/17", "tags": [ "monitoring" ] }, { "pattern": "pingping\\.io\\/", "url": "https://pingping.io/", "instances": [ "pingping.io/1.0" ], "description": "Lightweight monitoring bot validating URL availability", "addition_date": "2026/04/17", "tags": [ "monitoring" ] }, { "pattern": "PlayStore-Google", "url": "https://support.google.com/webmasters/answer/1061943", "instances": [ "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.6478.182 Safari/537.36 PlayStore-Google" ], "description": "Google Play Store crawler indexing apps and content", "addition_date": "2026/04/17", "tags": [ "search-engine" ] }, { "pattern": "Plesk screenshot bot", "url": "https://www.plesk.com/", "instances": [ "Plesk screenshot bot https://support.plesk.com/hc/en-us/articles/10301006946066" ], "description": "Hosting platform management crawler for website auditing", "addition_date": "2026/04/17", "tags": [ "seo" ] }, { "pattern": "PocketCasts\\/", "url": "https://support.pocketcasts.com/knowledge-base/pocket-casts-feed-parser/", "instances": [ "PocketCasts/1.0 (Pocket Casts Feed Parser; +http://pocketcasts.com/)" ], "description": "Podcast platform crawler parsing feeds and episodes", "addition_date": "2026/04/17", "tags": [ "feed-reader" ] }, { "pattern": "Potions\\/", "url": "https://get-potions.com/", "instances": [ "Potions/1.0.0" ], "description": "E-commerce personalization crawler ingesting catalogs", "addition_date": "2026/04/17", "tags": [ "seo" ] }, { "pattern": "PressEngineBot", "url": "https://www.pressengine.net/", "instances": [ "PressEngineBot (+http://pressengine.net/crawl-policy)" ], "description": "Press release aggregation crawler for content syndication", "addition_date": "2026/04/17", "tags": [ "seo" ] }, { "pattern": "PricedroneShoppingBot\\/", "url": "http://pricedrone.com/robot/", "instances": [ "Mozilla/5.0 (compatible; PricedroneShoppingBot/1.0; +http://pricedrone.com/robot/)" ], "description": "Price comparison crawler aggregating product pricing", "addition_date": "2026/04/17", "tags": [ "seo" ] }, { "pattern": "PriEcoBot\\/", "url": "https://datadome.co/bots/priecobot/", "instances": [ "Mozilla/5.0 (compatible; PriEcoBot/1.0; +https://prieco.net)" ], "description": "Web crawler for content aggregation and analysis", "addition_date": "2026/04/17", "tags": [ "seo" ] }, { "pattern": "PrintFriendly\\.com", "url": "https://www.printfriendly.com/", "instances": [ "Mozilla/5.0 (X11; Linux i586; rv:31.0) Gecko/20100101 Firefox/31.0 PrintFriendly.com" ], "description": "Print optimization crawler converting pages to printable format", "addition_date": "2026/04/17", "tags": [ "seo" ] }, { "pattern": "Pro-Sitemaps\\/", "url": "https://pro-sitemaps.com/", "instances": [ "Mozilla/5.0 (compatible; Pro Sitemaps Generator; pro-sitemaps.com) Gecko Pro-Sitemaps/1.0" ], "description": "Sitemap generation crawler discovering and indexing URLs", "addition_date": "2026/04/17", "tags": [ "seo" ] }, { "pattern": "ProbelySPDR\\/", "url": "https://probely.com/sos", "instances": [ "Mozilla/5.0 (compatible; +https://probely.com/sos) ProbelySPDR/0.1.0" ], "description": "Security testing crawler identifying web vulnerabilities", "addition_date": "2026/04/17", "tags": [ "scanner" ] }, { "pattern": "ProjectShield-UrlCheck", "url": "https://projectshield.withgoogle.com/", "instances": [ "Mozilla/5.0 (compatible; ProjectShield-UrlCheck; +http://g.co/projectshield)" ], "description": "Google DDoS protection bot validating customer origins", "addition_date": "2026/04/17", "tags": [ "scanner" ] }, { "pattern": "Blackbox Exporter\\/", "url": "https://github.com/prometheus/blackbox_exporter", "instances": [ "Blackbox Exporter/0.20.0-rc.0" ], "description": "Prometheus monitoring crawler checking endpoint health", "addition_date": "2026/04/17", "tags": [ "monitoring" ] }, { "pattern": "Protopage\\/", "url": "https://datadome.co/bots/protopage/", "instances": [ "Protopage/3.0 (http://www.protopage.com)" ], "description": "Start page and RSS reader crawler aggregating content", "addition_date": "2026/04/17", "tags": [ "feed-reader" ] }, { "pattern": "PS_Daily\\/", "url": "https://datadome.co/bots/ps-daily/", "instances": [ "PS_Daily/1.0 Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/123.0.0.0 Safari/537.36" ], "description": "Web crawler for data collection and analysis", "addition_date": "2026/04/17", "tags": [ "seo" ] }, { "pattern": "pulsetic\\.com", "url": "https://pulsetic.com/", "instances": [ "pulsetic.com (+https://pulsetic.com)" ], "description": "Uptime monitoring bot tracking website availability", "addition_date": "2026/04/17", "tags": [ "monitoring" ] }, { "pattern": "PWABuilderHttpAgent", "url": "https://www.pwabuilder.com/", "instances": [ "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/96.0.4664.110 Safari/537.36 Edg/96.0.1054.57 PWABuilderHttpAgent" ], "description": "Microsoft PWA readiness evaluation crawler tool", "addition_date": "2026/04/17", "tags": [ "seo" ] }, { "pattern": "QualifiedBot\\/", "url": "https://www.qualified.com/legal/qualified-crawler-user-agent", "instances": [ "QualifiedBot/1.0", "Mozilla/5.0 (compatible; QualifiedBot/1.0; +https://www.qualified.com/legal/qualified-crawler-user-agent)" ], "description": "Conversational sales platform crawler inventorying site URLs", "addition_date": "2026/04/17", "tags": [ "seo" ] }, { "pattern": "Quantcastbot\\/", "url": "http://www.quantcast.com/bot", "instances": [ "Quantcastbot/1.0 (+http://www.quantcast.com/bot)" ], "description": "Audience analytics crawler for ad measurement and targeting", "addition_date": "2026/04/17", "tags": [ "advertising" ] }, { "pattern": "Rackspace Monitoring\\/", "url": "https://support.rackspace.com/how-to/about-the-rackspace-monitoring-agent/", "instances": [ "Rackspace Monitoring/1.1 (https://monitoring.api.rackspacecloud.com)" ], "description": "Cloud infrastructure monitoring bot checking endpoint availability", "addition_date": "2026/04/17", "tags": [ "monitoring" ] }, { "pattern": "rakutenusabot-image\\/", "url": "https://product-image.ebates.com/item-gsp/rakutenusabot.html", "instances": [ "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; rakutenusabot-image/1.0) Chrome/114.0.0.0 Safari/537.36" ], "description": "E-commerce platform crawler extracting product images", "addition_date": "2026/04/17", "tags": [ "seo" ] }, { "pattern": "top100\\.rambler\\.ru crawler", "url": "https://datadome.co/bots/rambler-bot/", "instances": [ "Mozilla/5.0 (compatible; top100.rambler.ru crawler)" ], "description": "Russian search engine crawler indexing web content", "addition_date": "2026/04/17", "tags": [ "search-engine" ] }, { "pattern": "RankurBot\\/", "url": "http://rankur.com/technology.html", "instances": [ "RankurBot/3.3 (+http://rankur.com)" ], "description": "Social media and reputation monitoring crawler collecting data", "addition_date": "2026/04/17", "tags": [ "seo" ] }, { "pattern": "RavenCrawler\\/", "url": "https://datadome.co/bots/ravencrawler/", "instances": [ "Mozilla/5.0 (compatible; RavenCrawler/2.0; +https://raventools.com/seo-website-auditor/)" ], "description": "SEO auditing crawler analyzing website optimization signals", "addition_date": "2026/04/17", "tags": [ "seo" ] }, { "pattern": "Readable\\/", "url": "https://readable.com/", "instances": [ "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36 Readable/1.1.4" ], "description": "Content analysis crawler computing readability and metrics", "addition_date": "2026/04/17", "tags": [ "seo" ] }, { "pattern": "Recurly Webhooks\\/", "url": "https://docs.recurly.com/docs/webhooks", "instances": [ "Recurly Webhooks/2.0 (+https://docs.recurly.com/push-notifications)" ], "description": "Billing platform webhook validation and monitoring bot", "addition_date": "2026/04/17", "tags": [ "seo" ] }, { "pattern": "RED\\/", "url": "https://redbot.org/", "instances": [ "RED/2.0.14 (https://redbot.org/)" ], "description": "HTTP response testing and validation analyzer", "addition_date": "2026/04/17", "tags": [ "monitoring" ] }, { "pattern": "Reelevant\\/", "url": "https://reelevant.com/", "instances": [ "Reelevant/1.0" ], "description": "Personalization platform fetching URLs for email and content", "addition_date": "2026/04/17", "tags": [ "seo" ] }, { "pattern": "remove\\.bg\\/", "url": "https://www.remove.bg/", "instances": [ "remove.bg/1.0 background removal website" ], "description": "Image editing service crawler accessing and processing images", "addition_date": "2026/04/17", "tags": [ "seo" ] }, { "pattern": "Retool\\/", "url": "https://docs.tryretool.com/docs/apis", "instances": [ "Retool/2.0 (+https://docs.tryretool.com/docs/apis)" ], "description": "Internal tools platform fetching page metadata and content", "addition_date": "2026/04/17", "tags": [ "seo" ] }, { "pattern": "RetroListeCOM\\/", "url": "https://retroliste.com/", "instances": [ "Mozilla/5.0 (compatible; RetroListeCOM/1.0)" ], "description": "Directory and price comparison crawler indexing listings", "addition_date": "2026/04/17", "tags": [ "seo" ] }, { "pattern": "RevvimGort\\/", "url": "https://revvim.com", "instances": [ "Chrome/54.0.2840.71 (compatible; RevvimGort/5.0; +http://www.revvim.com; webmaster@revvim.com)" ], "description": "SEO optimization crawler detecting performance issues", "addition_date": "2026/04/17", "tags": [ "seo" ] }, { "pattern": "reward-gateway", "url": "https://www.rewardgateway.co", "instances": [ "reward-gateway" ], "description": "Employee engagement platform crawler validating offers", "addition_date": "2026/04/17", "tags": [ "seo" ] }, { "pattern": "Riddler \\(http:\\/\\/riddler\\.io", "url": "http://riddler.io/about", "instances": [ "Riddler (http://riddler.io/about)" ], "description": "Internet scanning crawler gathering domain and IP metadata", "addition_date": "2026/04/17", "tags": [ "scanner" ] }, { "pattern": "RobotsChecker\\/", "url": "http://www.blocked.org.uk/", "instances": [ "RobotsChecker/0.6 (+http://www.blocked.org.uk)" ], "description": "Website health monitoring and content indexing crawler", "addition_date": "2026/04/17", "tags": [ "seo" ] }, { "pattern": "RSSAPI\\/", "url": "https://rssapi.net", "instances": [ "RSSAPI/2.0 (+https://rssapi.net/)" ], "description": "RSS and Atom feed crawler for content aggregation", "addition_date": "2026/04/17", "tags": [ "feed-reader" ] }, { "pattern": "rss2tg", "url": "https://rss2tg.duck.consulting", "instances": [ "Mozilla/5.0 (compatible; rss2tg bot; +http://komar.in/en/rss2tg_crawler)" ], "description": "RSS to Telegram converter fetching feed content", "addition_date": "2026/04/17", "tags": [ "feed-reader" ] }, { "pattern": "RssReaderBot", "url": "https://datadome.co/bots/rssreaderbot/", "instances": [ "RssReaderBot" ], "description": "RSS feed aggregator indexing content updates", "addition_date": "2026/04/17", "tags": [ "feed-reader" ] }, { "pattern": "s4a-probe-bot\\/", "url": "https://www.seo4ajax.com/webscraper/", "instances": [ "Mozilla/5.0 AppleWebKit (compatible; s4a-probe-bot/1.0; Fake-Googlebot; +https://www.seo4ajax.com/webscraper)" ], "description": "Web scraping and SEO analysis crawler tool", "addition_date": "2026/04/17", "tags": [ "seo" ] }, { "pattern": "SFDC-Callout\\/", "url": "https://help.salesforce.com/articleView?id=000321501&type=1&mode=1", "instances": [ "SFDC-Callout/49.0" ], "description": "Salesforce platform crawler fetching web resources", "addition_date": "2026/04/17", "tags": [ "seo" ] }, { "pattern": "page-preview-tool", "url": "https://www.salesviewer.com/en/", "instances": [ "page-preview-tool Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.132 Safari/537.36" ], "description": "B2B intelligence platform identifying visiting companies", "addition_date": "2026/04/17", "tags": [ "seo" ] }, { "pattern": "SandobaCrawler\\/", "url": "https://www.sandoba.com/en/crawler/", "instances": [ "mozilla/5.0 (compatible; SandobaCrawler/1.0; +https://www.sandoba.com/en/crawler/)" ], "description": "Web crawler for data aggregation and indexing", "addition_date": "2026/04/17", "tags": [ "seo" ] }, { "pattern": "Sansec Security Monitor\\/", "url": "https://sansec.io/monitor", "instances": [ "Mozilla/5.0 (compatible; Sansec Security Monitor/1.0; +https://sansec.io/monitor)" ], "description": "E-commerce malware detection scanner for skimmers", "addition_date": "2026/04/17", "tags": [ "scanner" ] }, { "pattern": "GIFTEDVISITOR SCAN", "url": "https://datadome.co/bots/scan/", "instances": [ "GIFTEDVISITOR SCAN" ], "description": "Web crawler for content indexing and analysis", "addition_date": "2026/04/17", "tags": [ "seo" ] }, { "pattern": "Schema-Markup-Validator", "url": "https://validator.schema.org/", "instances": [ "Mozilla/5.0 (compatible; Schema-Markup-Validator; +https://validator.schema.org/)" ], "description": "Structured data validation crawler for schema compliance", "addition_date": "2026/04/17", "tags": [ "seo" ] }, { "pattern": "Scoop\\.it\\/", "url": "https://datadome.co/bots/scoop-it/", "instances": [ "Mozilla/5.0 (compatible; Scoop.it/1.0; +http://www.scoop.it/bot)" ], "description": "Content curation platform crawler discovering relevant content", "addition_date": "2026/04/17", "tags": [ "seo" ] }, { "pattern": "ScourRSSBot\\/", "url": "https://scour.ing/bot", "instances": [ "ScourRSSBot/1.0 (+https://scour.ing/bot)" ], "description": "RSS feed discovery and monitoring crawler bot", "addition_date": "2026/04/17", "tags": [ "feed-reader" ] }, { "pattern": "ScrapeheroBot\\/", "url": "https://scrapehero.de/", "instances": [ "Mozilla/5.0 (compatible; ScrapeheroBot/1.0; +https://scrapehero.de/)" ], "description": "Web scraping service crawler for data extraction", "addition_date": "2026/04/17", "tags": [ "seo" ] }, { "pattern": "screeenly-bot", "url": "https://3.screeenly.com/ua", "instances": [ "screeenly-bot 2.0" ], "description": "Screenshot capture service crawler taking website snapshots", "addition_date": "2026/04/17", "tags": [ "seo" ] }, { "pattern": "SEBot-WA", "url": "https://help.seranking.com/en/project-tools/website-audit/overview", "instances": [ "SEBot-WA" ], "description": "SEO audit crawler performing technical analysis", "addition_date": "2026/04/17", "tags": [ "seo" ] }, { "pattern": "Searcherweb", "url": "https://datadome.co/bots/searcherweb/", "instances": [ "Searcherweb" ], "description": "Web crawler indexing content for search engines", "addition_date": "2026/04/17", "tags": [ "search-engine" ] }, { "pattern": "Searcherxweb", "url": "https://datadome.co/bots/searcherxweb/", "instances": [ "Searcherxweb" ], "description": "Web crawler for indexing and data collection", "addition_date": "2026/04/17", "tags": [ "search-engine" ] }, { "pattern": "SearchExpress", "url": "https://datadome.co/bots/searchexpress/", "instances": [ "SearchExpress" ], "description": "Search engine crawler gathering website information", "addition_date": "2026/04/17", "tags": [ "search-engine" ] }, { "pattern": "SecurityHeaders", "url": "https://securityheaders.com/", "instances": [ "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/108.0.0.0 Safari/537.36 SecurityHeaders" ], "description": "HTTP security headers analyzer and evaluator", "addition_date": "2026/04/17", "tags": [ "scanner" ] }, { "pattern": "semaltbot\\/", "url": "https://datadome.co/bots/semaltbot/", "instances": [ "semaltbot/0.1 (+http://semalt.net)" ], "description": "SEO analytics crawler collecting ranking and performance data", "addition_date": "2026/04/17", "tags": [ "seo" ] }, { "pattern": "SendGrid Event API", "url": "https://sendgrid.com/docs/for-developers/tracking-events/event/", "instances": [ "SendGrid Event API" ], "description": "Email service crawler scanning links in messages", "addition_date": "2026/04/17", "tags": [ "seo" ] }, { "pattern": "SentryUptimeBot\\/", "url": "https://docs.sentry.io/product/alerts/uptime-monitoring/troubleshooting/#verify-firewall-configuration", "instances": [ "SentryUptimeBot/1.0 (+http://docs.sentry.io/product/alerts/uptime-monitoring/)" ], "description": "Application monitoring bot verifying URL availability", "addition_date": "2026/04/17", "tags": [ "monitoring" ] }, { "pattern": "seo-audit-check-bot\\/", "url": "https://www.webceo.com/webceo-bots.htm", "instances": [ "Mozilla/5.0 (compatible; seo-audit-check-bot/1.0)" ], "description": "SEO auditing crawler evaluating optimization signals", "addition_date": "2026/04/17", "tags": [ "seo" ] }, { "pattern": "s4a\\/", "url": "https://www.seo4ajax.com/", "instances": [ "Mozilla/5.0 AppleWebKit (compatible; s4a/1.0; +https://www.seo4ajax.com/webscraper)" ], "description": "JavaScript rendering and SEO prerendering service", "addition_date": "2026/04/17", "tags": [ "seo" ] }, { "pattern": "ClarityBot\\/", "url": "https://www.seoclarity.net/bot.html", "instances": [ "Mozilla/5.0 (compatible; ClarityBot/9.0; +https://www.seoclarity.net/bot.html)" ], "description": "SEO analytics crawler providing optimization insights", "addition_date": "2026/04/17", "tags": [ "seo" ] }, { "pattern": "SeoSiteCheckup", "url": "https://datadome.co/bots/seositecheckup/", "instances": [ "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.108 Safari/537.36 SeoSiteCheckup (https://seositecheckup.com)" ], "description": "SEO site analysis crawler evaluating search performance", "addition_date": "2026/04/17", "tags": [ "seo" ] }, { "pattern": "SeoulBot", "url": "https://datadome.co/bots/seoulbot/", "instances": [ "SeoulBot" ], "description": "Web crawler for SEO and competitive analysis", "addition_date": "2026/04/17", "tags": [ "seo" ] }, { "pattern": "SERPtimizerBot", "url": "http://serptimizer.com/serptimizer-bot", "instances": [ "Mozilla/5.0 (compatible; SERPtimizerBot; +http://serptimizer.com/serptimizer-bot)" ], "description": "SEO analysis crawler gathering data for optimization", "addition_date": "2026/04/17", "tags": [ "seo" ] }, { "pattern": "Server Density Service Monitoring", "url": "https://www.stackpath.com/", "instances": [ "Server Density Service Monitoring v2Server Density Agent" ], "description": "Cloud monitoring bot checking uptime and performance", "addition_date": "2026/04/17", "tags": [ "monitoring" ] }, { "pattern": "ServerHunterSpider\\/", "url": "https://www.serverhunter.com/", "instances": [ "Mozilla/5.0 (compatible; ServerHunterSpider/1.1; +https://www.serverhunter.com/spider/)" ], "description": "Hosting and VPS price comparison crawler", "addition_date": "2026/04/17", "tags": [ "seo" ] }, { "pattern": "SeznamHomepageCrawler\\/", "url": "http://napoveda.seznam.cz/en/seznambot-intro/", "instances": [ "SeznamHomepageCrawler/v1.0.6" ], "description": "Czech search engine crawler indexing web pages", "addition_date": "2026/04/17", "tags": [ "search-engine" ] }, { "pattern": "Shopify-Captain-Hook", "url": "https://shopify.dev/docs/apps/build/webhooks", "instances": [ "Shopify-Captain-Hook" ], "description": "E-commerce platform crawler for webhook verification", "addition_date": "2026/04/17", "tags": [ "seo" ] }, { "pattern": "Shortwave Image Fetcher", "url": "https://www.shortwave.com/", "instances": [ "Shortwave Image Fetcher" ], "description": "Email client image proxy and link preview fetcher", "addition_date": "2026/04/17", "tags": [ "social-preview" ] }, { "pattern": "linkReader\\/", "url": "https://sider.ai/", "instances": [ "linkReader/1.0 (+https://gochitchat.ai)" ], "description": "AI assistant crawler for content indexing", "addition_date": "2026/04/17", "tags": [ "ai-crawler" ] }, { "pattern": "Sidetrade indexer bot", "url": "https://datadome.co/bots/sidetrade-crawler/", "instances": [ "Sidetrade indexer bot" ], "description": "Financial analytics crawler gathering business data", "addition_date": "2026/04/17", "tags": [ "seo" ] }, { "pattern": "Silk\\/", "url": "https://www.useragentstring.com/pages/silk/", "instances": [ "Silk/1.0 Also seen as: silk/1.0 (+http://www.slider.com/silk.htm)/3.7" ], "description": "DMOZ directory crawler indexing web content", "addition_date": "2026/04/17", "tags": [ "seo" ] }, { "pattern": "SinceraSyntheticUser\\/", "url": "https://datadome.co/bots/sincera-bot/", "instances": [ "Mozilla/5.0 (compatible; SinceraSyntheticUser/1.0; +http://app.sincera.io/bots)" ], "description": "Web data aggregation and analytics crawler", "addition_date": "2026/04/17", "tags": [ "seo" ] }, { "pattern": "Optimizer\\)", "url": "https://www.sistrix.com/faq/uptime", "instances": [ "Mozilla/5.0 (compatible; Optimizer)" ], "description": "SEO analysis and website optimization crawler", "addition_date": "2026/04/17", "tags": [ "seo" ] }, { "pattern": "Site24x7", "url": "https://www.site24x7.com/", "instances": [ "Site24x7" ], "description": "Cloud monitoring service checking website health", "addition_date": "2026/04/17", "tags": [ "monitoring" ] }, { "pattern": "SiteAuditBot\\/", "url": "https://datadome.co/bots/siteauditbot/", "instances": [ "Mozilla/5.0 (compatible; SiteAuditBot/1.1; +https://siteauditbot.com)" ], "description": "Website SEO auditing and link checking crawler", "addition_date": "2026/04/17", "tags": [ "seo" ] }, { "pattern": "SiteCheck-sitecrawl", "url": "https://support.siteimprove.com/hc/en-gb/articles/206345523-What-IP-addresses-and-user-agents-are-used-by-Siteimprove-", "instances": [ "Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; Trident/6.0) SiteCheck-sitecrawl" ], "description": "Digital governance platform auditing web quality", "addition_date": "2026/04/17", "tags": [ "seo" ] }, { "pattern": "SiteScoreBot", "url": "https://datadome.co/bots/sitescorebot/", "instances": [ "SiteScoreBot v20210315 - https://sitescore.ai" ], "description": "Website performance and security evaluation crawler", "addition_date": "2026/04/17", "tags": [ "seo" ] }, { "pattern": "SiteSearch360\\/", "url": "https://www.sitesearch360.com/", "instances": [ "Mozilla/5.0 (compatible; SiteSearch360/1.0; +https://sitesearch360.com/)" ], "description": "On-site search platform crawler building indices", "addition_date": "2026/04/17", "tags": [ "search-engine" ] }, { "pattern": "SiteUptime\\.com", "url": "https://datadome.co/bots/siteuptime-com/", "instances": [ "SiteUptime.com" ], "description": "Uptime monitoring service checking website availability", "addition_date": "2026/04/17", "tags": [ "monitoring" ] }, { "pattern": "Konturbot\\/", "url": "https://datadome.co/bots/skb-kontur-bot/", "instances": [ "Mozilla/5.0 (compatible; Konturbot/1.2; +http://kontur.ru; cargo@kontur.ru)" ], "description": "Russian business intelligence and analytics crawler", "addition_date": "2026/04/17", "tags": [ "seo" ] }, { "pattern": "SkroutzBot", "url": "https://www.skroutz.gr/", "instances": [ "SkroutzBot v1.0" ], "description": "Price comparison platform crawler aggregating products", "addition_date": "2026/04/17", "tags": [ "seo" ] }, { "pattern": "SkyworkSpider", "url": "https://datadome.co/bots/skyworkspider/", "instances": [ "SkyworkSpider" ], "description": "Web crawler for content indexing and analysis", "addition_date": "2026/04/17", "tags": [ "seo" ] }, { "pattern": "SlickBot\\/", "url": "https://www.slickstream.com/", "instances": [ "Mozilla/5.0 (compatible; SlickBot/1.0; +http://slickstream.com)" ], "description": "Content recommendation and search indexing crawler", "addition_date": "2026/04/17", "tags": [ "seo" ] }, { "pattern": "SmartologyBot\\/", "url": "https://smartology.net/smartologybot/", "instances": [ "SmartologyBot/1.0 (+http://www.smartology.net/smartologybot)" ], "description": "Contextual advertising crawler for brand safety", "addition_date": "2026/04/17", "tags": [ "advertising" ] }, { "pattern": "SnapURLPreview\\/", "url": "https://business.snapchat.com/legal/snapchat-automated-crawler", "instances": [ "Mozilla/5.0 (compatible; SnapURLPreview/1.0; +https://www.snapchat.com)" ], "description": "Social platform URL preview and metadata fetcher", "addition_date": "2026/04/17", "tags": [ "social-preview" ] }, { "pattern": "SnapchatAds\\/", "url": "https://businesshelp.snapchat.com/s/article/adsbot-crawler", "instances": [ "Mozilla/5.0 (compatible; SnapchatAds/1.0; +https://businesshelp.snapchat.com/s/article/adsbot-crawler)" ], "description": "Advertising platform crawler for ad verification", "addition_date": "2026/04/17", "tags": [ "advertising" ] }, { "pattern": "Snipcart\\/", "url": "https://snipcart.com/", "instances": [ "Snipcart/1.0" ], "description": "E-commerce shopping cart crawler validating products", "addition_date": "2026/04/17", "tags": [ "seo" ] }, { "pattern": "solarwinds\\/", "url": "https://documentation.solarwinds.com/en/success_center/observability/content/get-started/dem_getting_started_guide.htm", "instances": [ "solarwinds/1.0 ( www.solarwinds.com/solarwinds-observability)" ], "description": "Observability platform monitoring website performance", "addition_date": "2026/04/17", "tags": [ "monitoring" ] }, { "pattern": "Sora POS\\/", "url": "https://www.sora-caisse.com/sora.pdf", "instances": [ "Sora POS/1.0 (Sora Websoft)" ], "description": "Point-of-sale system crawler for transaction data", "addition_date": "2026/04/17", "tags": [ "seo" ] }, { "pattern": "SparkShipping", "url": "https://www.sparkshipping.com/", "instances": [ "SparkShipping" ], "description": "E-commerce data crawler normalizing product catalogs", "addition_date": "2026/04/17", "tags": [ "seo" ] }, { "pattern": "SparkPost", "url": "https://www.sparkpost.com/", "instances": [ "SparkPost" ], "description": "Email service crawler validating links and tracking", "addition_date": "2026/04/17", "tags": [ "seo" ] }, { "pattern": "Spawning-AI", "url": "https://datadome.co/bots/spawning-bot/", "instances": [ "Spawning-AI" ], "description": "Web crawler indexing content for AI models", "addition_date": "2026/04/17", "tags": [ "ai-crawler" ] }, { "pattern": "IDG\\/EU", "url": "https://datadome.co/bots/spaziodati-bot/", "instances": [ "IDG/EU (http://spaziodati.eu/)" ], "description": "Semantic data and business intelligence crawler", "addition_date": "2026/04/17", "tags": [ "seo" ] }, { "pattern": "Specificfeeds", "url": "https://follow.it/", "instances": [ "FeedBurner/1.0 (http://www.FeedBurner.com) Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.132 Safari/537.36 Specificfeeds" ], "description": "Content aggregation crawler for personalized feeds", "addition_date": "2026/04/17", "tags": [ "feed-reader" ] }, { "pattern": "Spectate\\/", "url": "https://docs.spectate.net/faq/uptime-monitor-bot", "instances": [ "Spectate/1.0 (+https://docs.spectate.net/faq/uptime-monitor-bot)" ], "description": "Content monitoring and uptime checking crawler", "addition_date": "2026/04/17", "tags": [ "monitoring" ] }, { "pattern": "SpiderLing", "url": "http://nlp.fi.muni.cz/projects/biwec/", "instances": [ "Mozilla/5.0 (compatible; SpiderLing (a SPIDER for LINGustic research); +http://nlp.fi.muni.cz/projects/biwec/)" ], "description": "Linguistic research crawler building language corpora", "addition_date": "2026/04/17", "tags": [ "academic" ] }, { "pattern": "splash Version\\/", "url": "https://www.zyte.com/splash/", "instances": [ "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/602.1 (KHTML, like Gecko) splash Version/10.0 Safari/602.1" ], "description": "Headless browser for web scraping and testing", "addition_date": "2026/04/17", "tags": [ "browser-automation" ] }, { "pattern": "Rigor\\)", "url": "https://www.splunk.com/", "instances": [ "Mozilla/5.0 (X11; Linux x86_64; Rigor) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.75 Safari/537.36" ], "description": "Splunk synthetic monitoring for performance checks", "addition_date": "2026/04/17", "tags": [ "monitoring" ] }, { "pattern": "TwinWaveScanner", "url": "https://www.splunk.com/en_us/products/attack-analyzer.html", "instances": [ "TwinWaveScanner" ], "description": "Security threat analysis and phishing detection bot", "addition_date": "2026/04/17", "tags": [ "scanner" ] }, { "pattern": "SSL Labs \\(https:\\/\\/www\\.ssllabs\\.com", "url": "https://www.ssllabs.com/", "instances": [ "SSL Labs (https://www.ssllabs.com/about/assessment.html)" ], "description": "TLS certificate and encryption configuration analyzer", "addition_date": "2026/04/17", "tags": [ "scanner" ] }, { "pattern": "SSSSBot\\/", "url": "https://datadome.co/bots/ssssbot/", "instances": [ "Mozilla/5.0 (compatible; SSSSBot/0.0.1; +http://s-ans.xyz:80/)" ], "description": "Web crawler for content indexing and analysis", "addition_date": "2026/04/17", "tags": [ "seo" ] }, { "pattern": "Stape\\/", "url": "https://stape.io/helpdesk/documentation/stape-scanner", "instances": [ "Stape/1.0.0Stape" ], "description": "Server-side tagging platform tag verification bot", "addition_date": "2026/04/17", "tags": [ "seo" ] }, { "pattern": "StartpagePrivateImageProxy\\/", "url": "https://datadome.co/bots/startpage-bot/", "instances": [ "StartpagePrivateImageProxy/2.0 (https://www.startpage.com/; support@startpage.com) requests/2.25.1" ], "description": "Privacy-focused search engine crawler indexing pages", "addition_date": "2026/04/17", "tags": [ "search-engine" ] }, { "pattern": "Statabot\\/", "url": "https://www.stata.com/support/statabot", "instances": [ "Statabot/1.0 https://www.stata.com/support/statabot" ], "description": "Analytics and market intelligence web crawler", "addition_date": "2026/04/17", "tags": [ "seo" ] }, { "pattern": "StatistikAustria\\/", "url": "https://www.statistik.at/ueber-uns/innovationen-und-experimentelle-statistik/einsatz-von-kassenscannerdaten-und-webscraping-in-der-preisstatistik", "instances": [ "Mozilla/5.0 (compatible; StatistikAustria/1.0; +http://www.statistik.gv.at)" ], "description": "National statistics office crawler for price data", "addition_date": "2026/04/17", "tags": [ "seo" ] }, { "pattern": "StatsDroneBot", "url": "https://statsdrone.com/statsdrone-bot-documentation/", "instances": [ "StatsDroneBot (https://statsdrone.com/statsdrone-bot-documentation/) Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36" ], "description": "Affiliate marketing link validation and monitoring bot", "addition_date": "2026/04/17", "tags": [ "seo" ] }, { "pattern": "Stripe\\/", "url": "https://stripe.com/docs/webhooks", "instances": [ "Stripe/1.0 (+https://stripe.com/docs/webhooks)" ], "description": "Payment processor crawler verifying merchant sites and collecting metadata signals", "addition_date": "2026/04/17", "tags": [ "seo" ] }, { "pattern": "Sucuri", "url": "https://blog.sucuri.net/2012/10/ask-sucuri-how-does-sitecheck-work.html", "instances": [ "Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US; rv:1.9.2) Gecko/20100115 Firefox/3.6 MSIE 7.0; Sucuri Integrity Monitor/2.4" ], "description": "Security scanner identifying malware blacklist status and checking plugin vulnerability compliance", "addition_date": "2026/04/17", "tags": [ "scanner" ] }, { "pattern": "Svix-Webhooks\\/", "url": "https://docs.svix.com/receiving/source-ips", "instances": [ "Svix-Webhooks/1.65.0 (sender-9YMgn; +https://www.svix.com/http-sender/)Webhooks/1.65.0 (sender-9YMgn)" ], "description": "Webhooks infrastructure provider crawler cataloging vendor sites and delivery practices", "addition_date": "2026/04/17", "tags": [ "seo" ] }, { "pattern": "SwifteqLinkChecker", "url": "https://www.swifteq.com/", "instances": [ "SwifteqLinkChecker" ], "description": "Web crawler verifying link availability status redirects and basic page content validation", "addition_date": "2026/04/17", "tags": [ "seo" ] }, { "pattern": "Swisscows", "url": "https://swisscows.com/", "instances": [ "Swisscows Favicons" ], "description": "Privacy-focused search engine crawler indexing web pages without storing personal data", "addition_date": "2026/04/17", "tags": [ "search-engine" ] }, { "pattern": "Datadog Synthetic", "url": "https://docs.datadoghq.com/synthetics/", "instances": [ "Datadog Synthetic" ], "description": "Synthetic monitoring bot testing website APIs and collecting uptime performance data", "addition_date": "2026/04/17", "tags": [ "monitoring" ] }, { "pattern": "TactiScout\\/", "url": "http://find-it.world/TempCrawl/Crawltheque.php", "instances": [ "TactiScout/recruit (+http://find-it.world/TempCrawl/Crawltheque.php)" ], "description": "Data collection crawler gathering competitive intelligence and monitoring market trends analysis", "addition_date": "2026/04/17", "tags": [ "seo" ] }, { "pattern": "tchelebi\\/", "url": "https://tchelebi.io/", "instances": [ "Mozilla/5.0 (compatible; tchelebi/1.0; +http://tchelebi.io)" ], "description": "Web crawling tool indexing content for SEO data aggregation and competitive analysis purposes", "addition_date": "2026/04/17", "tags": [ "seo" ] }, { "pattern": "bitdiscovery", "url": "https://www.tenable.com/products/tenable-asm", "instances": [ "bitdiscovery" ], "description": "Security vulnerability scanner identifying weaknesses and compliance issues in web infrastructure", "addition_date": "2026/04/17", "tags": [ "scanner" ] }, { "pattern": "Test Certificate Info", "url": "https://datadome.co/bots/test-certificate-info/", "instances": [ "Test Certificate Info" ], "description": "SSL TLS certificate analyzer assessing website security encryption and expiration status", "addition_date": "2026/04/17", "tags": [ "scanner" ] }, { "pattern": "Testcrawler", "url": "https://datadome.co/bots/testcrawler/", "instances": [ "Testcrawler" ], "description": "Testing crawler evaluating website performance security vulnerabilities and SEO optimization", "addition_date": "2026/04/17", "tags": [ "seo" ] }, { "pattern": "test-bot", "url": "https://datadome.co/bots/test-bot/", "instances": [ "test-bot" ], "description": "Automated testing script monitoring uptime and gathering performance analysis data", "addition_date": "2026/04/17", "tags": [ "seo" ] }, { "pattern": "TestLocally\\/", "url": "https://testlocal.ly/", "instances": [ "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.6422.26 Safari/537.36 TestLocally/1.0" ], "description": "Development testing bot simulating user interactions and verifying website functionality behavior", "addition_date": "2026/04/17", "tags": [ "browser-automation" ] }, { "pattern": "TestURI", "url": "https://datadome.co/bots/testuri-crawler/", "instances": [ "Mozilla/5.0 (compatible; TestURI; +http://testuri.org/)" ], "description": "Web crawler testing performance security compliance and assessing website configuration", "addition_date": "2026/04/17", "tags": [ "seo" ] }, { "pattern": "TextRazor", "url": "https://datadome.co/bots/textrazor-crawler/", "instances": [ "TextRazor Downloader (https://www.textrazor.com)" ], "description": "NLP service crawler collecting web data for text analysis entity extraction processing", "addition_date": "2026/04/17", "tags": [ "seo" ] }, { "pattern": "The Knowledge AI", "url": "https://datadome.co/bots/the-knowledge-ai/", "instances": [ "The Knowledge AI" ], "description": "AI data collection crawler indexing content and enhancing machine learning models", "addition_date": "2026/04/17", "tags": [ "ai-crawler" ] }, { "pattern": "TheInternetSearchx", "url": "https://datadome.co/bots/theinternetsearch/", "instances": [ "TheInternetSearchx" ], "description": "Search engine crawler indexing websites and improving search result database coverage", "addition_date": "2026/04/17", "tags": [ "search-engine" ] }, { "pattern": "thesis-research-bot", "url": "https://datadome.co/bots/thesis-research-bot/", "instances": [ "thesis-research-bot" ], "description": "Academic research crawler collecting web data trends and analyzing website structures systematically", "addition_date": "2026/04/17", "tags": [ "academic" ] }, { "pattern": "Trellis-Services", "url": "https://www.mediavine.com/", "instances": [ "Trellis-Services" ], "description": "Content scraper and fetching service crawler for programmatic large-scale web page retrieval", "addition_date": "2026/04/17", "tags": [ "seo" ] }, { "pattern": "trentwil\\.es", "url": "https://trentwil.es/domains.html", "instances": [ "domain research project (+https://trentwil.es/domains.html)" ], "description": "Web crawler indexing content for data aggregation SEO analysis and competitive intelligence", "addition_date": "2026/04/17", "tags": [ "seo" ] }, { "pattern": "Trustly\\/", "url": "https://www.trustly.net/", "instances": [ "Trustly/1" ], "description": "Open banking crawler automating user sessions for account data and payment initiation", "addition_date": "2026/04/17", "tags": [ "seo" ] }, { "pattern": "TTD-Content", "url": "https://www.thetradedesk.com/us/ttd-content", "instances": [ "Mozilla/5.0 (compatible; TTD-Content; +https://www.thetradedesk.com/general/ttd-content)" ], "description": "Ad network content crawler analyzing pages for contextual advertising and brand safety", "addition_date": "2026/04/17", "tags": [ "advertising" ] }, { "pattern": "Tweakers", "url": "https://tweakers.net/", "instances": [ "Tweakers Image Proxy https://tweakers.net" ], "description": "Dutch technology website crawler aggregating product reviews and IT content data", "addition_date": "2026/04/17", "tags": [ "seo" ] }, { "pattern": "TwilioProxy\\/", "url": "https://www.upday.com/", "instances": [ "TwilioProxy/1.1" ], "description": "Communication platform fetcher retrieving media links and previews for messaging services", "addition_date": "2026/04/17", "tags": [ "social-preview" ] }, { "pattern": "UASlinkChecker\\/", "url": "https://udger.com/support/UASlinkChecker", "instances": [ "Mozilla/5.0 (compatible; UASlinkChecker/2.1; +https://udger.com/support/UASlinkChecker)" ], "description": "Link checking crawler validating URL availability and HTTP status across websites", "addition_date": "2026/04/17", "tags": [ "seo" ] }, { "pattern": "hgfAlphaXCrawl\\/", "url": "https://datadome.co/bots/uni-passau_bot/", "instances": [ "hgfAlphaXCrawl/1.0 (+https://www.fim.uni-passau.de/data-science/forschung/open-search)" ], "description": "University research crawler collecting data for academic projects and information retrieval", "addition_date": "2026/04/17", "tags": [ "academic" ] }, { "pattern": "Unshorten\\.It\\!", "url": "https://unshorten.it/", "instances": [ "Unshorten.It!/1.0 (https://unshorten.it/)" ], "description": "URL expansion crawler revealing shortened link destinations for cybersecurity threat intelligence", "addition_date": "2026/04/17", "tags": [ "scanner" ] }, { "pattern": "updown\\.io", "url": "https://updown.io", "instances": [ "updown.io daemon 2.11" ], "description": "Website monitoring crawler checking availability and performance metrics from locations", "addition_date": "2026/04/17", "tags": [ "monitoring" ] }, { "pattern": "Uptime\\/", "url": "https://uptime.com/", "instances": [ "Mozilla/5.0 (compatible; Uptime/1.0; http://uptime.com)" ], "description": "Synthetic monitoring bot verifying site availability and transaction health globally", "addition_date": "2026/04/17", "tags": [ "monitoring" ] }, { "pattern": "uptimedoctor", "url": "http://uptimestatistics.com/", "instances": [ "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/64.0.3282.140 Safari/537.36 Edge/17.17134 www.uptimedoctor.com (username slowmail)" ], "description": "Uptime monitoring probe verifying website availability and SSL certificate health", "addition_date": "2026/04/17", "tags": [ "monitoring" ] }, { "pattern": "Uptimia", "url": "https://datadome.co/bots/uptimia/", "instances": [ "Mozilla/5.0 (compatible; Uptimia; www.uptimia.com)" ], "description": "Platform crawler monitoring website performance uptime and simulating user interactions", "addition_date": "2026/04/17", "tags": [ "monitoring" ] }, { "pattern": "uptrends", "url": "https://www.uptrends.com/", "instances": [ "uptrends" ], "description": "Synthetic monitoring agent testing transaction availability and performance from locations", "addition_date": "2026/04/17", "tags": [ "monitoring" ] }, { "pattern": "Urlcheckr\\/", "url": "https://www.urlcheckr.com/", "instances": [ "Urlcheckr/2.0" ], "description": "Dead link checker scanner identifying broken hyperlinks and inaccessible web pages", "addition_date": "2026/04/17", "tags": [ "seo" ] }, { "pattern": "URLSuMaBot", "url": "https://www.urlsuma.de/bot.aspx", "instances": [ "Mozilla / 5.0(Windows NT 10.0; Win64; x64) AppleWebKit / 537.36(KHTML, like Gecko; compatible; URLSuMaBot / 1.0; +https://www.urlsuma.de/bot.aspx) Chrome / 70.0.3538.77 Safari / 537.36" ], "description": "Web crawler systematically indexing and collecting data from websites for analysis", "addition_date": "2026/04/17", "tags": [ "seo" ] }, { "pattern": "useeBookChecker\\/", "url": "https://datadome.co/bots/useebookchecker/", "instances": [ "Mozilla/5.0 (compatible; useeBookChecker/0.2; +http://usee.pl/useeBookChecker.html)" ], "description": "Ebook metadata scanner identifying and cataloging digital publication availability data", "addition_date": "2026/04/17", "tags": [ "seo" ] }, { "pattern": "Vagabondo\\/", "url": "https://datadome.co/bots/vagabondo-bot/", "instances": [ "Mozilla/4.0 (compatible; Vagabondo/4.0Beta; webcrawler at wise-guys dot nl; http://webagent.wise-guys.nl/; http://www.wise-guys.nl/)", "Mozilla/4.0 (compatible; Vagabondo/2.2; webcrawler at wise-guys dot nl)", "Mozilla/5.0 (compatible; Vagabondo/2.1; webcrawler at wise-guys dot nl)", "Mozilla/3.0 (Vagabondo/2.0 MT; webcrawler@NOSPAMexperimental.net; http://aanmelden.ilse.nl/?aanmeld_mode=webhints)" ], "description": "Legacy Dutch web crawler indexing mobile pages for regional search and content", "addition_date": "2026/04/17", "tags": [ "seo" ] }, { "pattern": "VaultPress", "url": "https://vaultpress.com/", "instances": [ "VaultPress" ], "description": "WordPress backup and security scanner performing malware scanning and integrity", "addition_date": "2026/04/17", "tags": [ "scanner" ] }, { "pattern": "videootvBot", "url": "https://videoo.tv/", "instances": [ "Mozilla/5.0 (compatible; videootvBot; +https://www.videoo.tv)" ], "description": "Video aggregator crawler discovering indexing video content and stream endpoints", "addition_date": "2026/04/17", "tags": [ "seo" ] }, { "pattern": "VsuSearchSpider\\/", "url": "https://datadome.co/bots/vsusearchspider/", "instances": [ "VsuSearchSpider/1.0" ], "description": "Web crawler indexing and gathering information from websites for search purposes", "addition_date": "2026/04/17", "tags": [ "search-engine" ] }, { "pattern": "vu-server-health-scanner\\/", "url": "http://130.37.198.75/index.html", "instances": [ "Mozilla/5.0 (compatible;vu-server-health-scanner/1.0;https://130.37.198.75/index.html)" ], "description": "Academic crawler indexing web content for research data collection and analysis", "addition_date": "2026/04/17", "tags": [ "academic" ] }, { "pattern": "WARDBot\\/", "url": "https://ward.ai/robot", "instances": [ "Mozilla/5.0 (compatible; WARDBot/1.0; http://ward.ai/robot)" ], "description": "Web automation crawler extracting website data for reconnaissance and analysis", "addition_date": "2026/04/17", "tags": [ "seo" ] }, { "pattern": "WebsiteOps", "url": "https://watchful.net/faqs/technical-support/how-do-i-whitelist-the-watchful-ip-address", "instances": [ "WebsiteOps (hello@websiteops.io)" ], "description": "Web crawling client monitoring uptime changes and analyzing website data", "addition_date": "2026/04/17", "tags": [ "monitoring" ] }, { "pattern": "WatchMouse", "url": "https://asm.saas.broadcom.com/", "instances": [ "WatchMouse (http://watchmouse.com/ ; HQ)" ], "description": "Web performance monitoring tool assessing website availability from global testing locations", "addition_date": "2026/04/17", "tags": [ "monitoring" ] }, { "pattern": "Web Measure\\/", "url": "https://webresearch.eecs.umich.edu/overview-of-web-measurements/", "instances": [ "Web Measure/1.0 (https://webresearch.eecs.umich.edu/overview-of-web-measurements/) Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.130 Safari/537.36" ], "description": "Web analytics crawler gathering data for performance optimization and competitive analysis", "addition_date": "2026/04/17", "tags": [ "academic" ] }, { "pattern": "Webflow", "url": "https://datadome.co/bots/webflow-bot/", "instances": [ "Webflow 1.0 - Site Screenshot" ], "description": "Platform crawler rendering and indexing Webflow hosted websites for optimization", "addition_date": "2026/04/17", "tags": [ "seo" ] }, { "pattern": "webgains-bot", "url": "https://datadome.co/bots/webgains-bot/", "instances": [ "webgains-bot (https://www.webgains.com/public/en/)" ], "description": "Affiliate marketing crawler extracting product listings and pricing information", "addition_date": "2026/04/17", "tags": [ "advertising" ] }, { "pattern": "webprosbot\\/", "url": "https://datadome.co/bots/webprosbot/", "instances": [ "webprosbot/2.0 (+mailto:abuse-6337@webpros.com)" ], "description": "SEO crawler analyzing websites for optimization content and competitive research", "addition_date": "2026/04/17", "tags": [ "seo" ] }, { "pattern": "websitepulse", "url": "https://www.websitepulse.com/kb/websitepulse", "instances": [ "websitepulse checker/3.0 (compatible; MSIE 5.5; Netscape 4.75; Linux)" ], "description": "Synthetic monitoring agent measuring availability performance and content integrity", "addition_date": "2026/04/17", "tags": [ "monitoring" ] }, { "pattern": "WebSniffer\\/", "url": "http://websniffer.com/", "instances": [ "WebSniffer/1.1 (+http://websniffer.com/)" ], "description": "Web analyzer retrieving HTTP headers and HTML content for diagnostic purposes", "addition_date": "2026/04/17", "tags": [ "seo" ] }, { "pattern": "WSM\\/", "url": "https://webspidermount.com/", "instances": [ "Mozilla/5.0 (compatible; WSM/2.0; +https://webspidermount.com/)" ], "description": "Web scraper crawler for content collection and data aggregation purposes", "addition_date": "2026/04/17", "tags": [ "seo" ] }, { "pattern": "WebwikiBot\\/", "url": "https://www.webwiki.com/", "instances": [ "Mozilla/5.0 (compatible; WebwikiBot/2.1; +https://www.webwiki.com)" ], "description": "Directory crawler collecting website metadata and content for comprehensive indexing", "addition_date": "2026/04/17", "tags": [ "seo" ] }, { "pattern": "WEDOS OnLine", "url": "https://www.wedos.online", "instances": [ "WEDOS OnLine monitoring; https://www.wedos.online/" ], "description": "Website monitoring service checking availability DNS and SSL certificate status", "addition_date": "2026/04/17", "tags": [ "monitoring" ] }, { "pattern": "WhatsMyIP\\.org", "url": "http://whatsmyip.org/ua", "instances": [ "Mozilla/5.0 (WhatsMyIP.org Text_to_Code_Ratio_Tool) http://whatsmyip.org/ua" ], "description": "IP address analysis tool gathering network configuration and geolocation data", "addition_date": "2026/04/17", "tags": [ "seo" ] }, { "pattern": "WhatWeb\\/", "url": "https://www.whatweb.net/", "instances": [ "WhatWeb/0.5.5" ], "description": "Web technology scanner identifying frameworks servers and CMS configurations", "addition_date": "2026/04/17", "tags": [ "scanner" ] }, { "pattern": "Wheregoes\\.com", "url": "https://wheregoes.com/", "instances": [ "Wheregoes.com Redirect Checker/1.0" ], "description": "URL redirect analyzer mapping redirect chains and HTTP forwarding paths", "addition_date": "2026/04/17", "tags": [ "seo" ] }, { "pattern": "wheresitup\\.com\\/", "url": "https://wheresitup.com/", "instances": [ "wheresitup.com/1.1" ], "description": "Availability monitor crawler tracking website uptime and performance metrics", "addition_date": "2026/04/17", "tags": [ "monitoring" ] }, { "pattern": "Citoid", "url": "https://www.mediawiki.org/wiki/Citoid", "instances": [ "Citoid (Wikimedia tool; learn more at https://www.mediawiki.org/wiki/Citoid)" ], "description": "Wikipedia citation generator extracting metadata for automatic reference creation", "addition_date": "2026/04/17", "tags": [ "academic" ] }, { "pattern": "WireReaderBot\\/", "url": "https://datadome.co/bots/wirereaderbot/", "instances": [ "Mozilla/5.0 (compatible; WireReaderBot/1.0; +https://wirereader.app)" ], "description": "Web crawler indexing content for data aggregation and competitive analysis", "addition_date": "2026/04/17", "tags": [ "seo" ] }, { "pattern": "ZoteroTranslationServer\\/WMF", "url": "https://wikitech.wikimedia.org/wiki/Zotero", "instances": [ "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/110.0.0.0 Safari/537.36 ZoteroTranslationServer/WMF (mailto:noc@wikimedia.org)" ], "description": "Citation server extracting metadata from URLs for Wikipedia reference generation", "addition_date": "2026/04/17", "tags": [ "academic" ] }, { "pattern": "wmtips\\.com\\/", "url": "http://www.wmtips.com/tools/", "instances": [ "Mozilla/5.0 (compatible; wmtips.com/2.0; +http://www.wmtips.com/tools/)" ], "description": "SEO analysis crawler scanning websites for optimization issues and metrics", "addition_date": "2026/04/17", "tags": [ "seo" ] }, { "pattern": "WordCountBot\\/", "url": "https://weglot.com/", "instances": [ "Mozilla/5.0 (compatible; WordCountBot/0.1;)" ], "description": "Content analysis tool computing word counts keyword density and readability", "addition_date": "2026/04/17", "tags": [ "seo" ] }, { "pattern": "Wordup-1", "url": "https://datadome.co/bots/wordup-1/", "instances": [ "Wordup-1" ], "description": "Web crawler indexing and analyzing content for search and data analytics", "addition_date": "2026/04/17", "tags": [ "seo" ] }, { "pattern": "workona-favicon-service\\/", "url": "https://datadome.co/bots/workona-bot/", "instances": [ "workona-favicon-service/1.0.0" ], "description": "Productivity platform crawler organizing web content and managing bookmarks", "addition_date": "2026/04/17", "tags": [ "seo" ] }, { "pattern": "Indy Library", "url": "https://secure.ogone.com/", "instances": [ "Mozilla/3.0 (compatible; Indy Library)" ], "description": "Payment processor crawler verifying merchant sites and payment flow transactions", "addition_date": "2026/04/17", "tags": [ "seo" ] }, { "pattern": "WJHRO\\/", "url": "https://docs.worldpay.com/apis", "instances": [ "WJHRO/1.0 (WorldPay Java HTTP Request Object)" ], "description": "Payment platform crawler testing integrations and verifying checkout flow functionality", "addition_date": "2026/04/17", "tags": [ "seo" ] }, { "pattern": "WormlyBot", "url": "https://www.wormly.com/help/server-monitoring/website", "instances": [ "Mozilla/5.0 (compatible; WormlyBot; +http://wormly.com)" ], "description": "Synthetic monitoring crawler testing site APIs and measuring uptime performance", "addition_date": "2026/04/17", "tags": [ "monitoring" ] }, { "pattern": "WovnCrawler\\/", "url": "https://support.wovn.io/hc/ja/articles/360043165091", "instances": [ "WovnCrawler/1.0" ], "description": "Localization platform crawler extracting content and generating translated versions", "addition_date": "2026/04/17", "tags": [ "seo" ] }, { "pattern": "wowLink Crawler\\/", "url": "https://datadome.co/bots/wowlink-crawler/", "instances": [ "wowLink Crawler/1.0" ], "description": "Link management crawler indexing and analyzing web content for optimization", "addition_date": "2026/04/17", "tags": [ "seo" ] }, { "pattern": "WP Time Capsule", "url": "https://docs.wptimecapsule.com/", "instances": [ "WP Time Capsule API/1.0 ( https://cron.wptimecapsule.com/; https://service.wptimecapsule.com/ )" ], "description": "WordPress backup service crawler detecting changes and performing incremental backups", "addition_date": "2026/04/17", "tags": [ "seo" ] }, { "pattern": "WPUmbrella", "url": "https://wp-umbrella.com/bot", "instances": [ "WPUmbrella" ], "description": "WordPress monitor scanner assessing availability and tracking plugin updates", "addition_date": "2026/04/17", "tags": [ "monitoring" ] }, { "pattern": "wpbot\\/", "url": "https://datadome.co/bots/wpbot/", "instances": [ "Mozilla/5.0 (compatible; wpbot/1.1; +https://forms.gle/ajBaxygz9jSR8p8G9)" ], "description": "Web crawler indexing websites and analyzing content for search optimization", "addition_date": "2026/04/17", "tags": [ "seo" ] }, { "pattern": "WPMU DEV Broken Link Checker", "url": "https://wpmudev.com/docs/hub-2-0/broken-link-checker-2/", "instances": [ "WPMU DEV Broken Link Checker Spider" ], "description": "Link auditor identifying dead links bad anchors and HTTP errors for maintenance", "addition_date": "2026/04/17", "tags": [ "seo" ] }, { "pattern": "WPMUDEV Uptime Monitor", "url": "https://wpmudev.com/monitor/", "instances": [ "WPMUDEV Uptime Monitor 5.0 (https://wpmudev.com)" ], "description": "HTTP probe checking website availability status codes and SSL certificate health", "addition_date": "2026/04/17", "tags": [ "monitoring" ] }, { "pattern": "WPSec\\/", "url": "https://wpsec.com/", "instances": [ "Mozilla/5.0 (compatible; WPSec/1.3; +https://wpsec.com)" ], "description": "WordPress security scanner identifying vulnerabilities outdated plugins and misconfigurations", "addition_date": "2026/04/17", "tags": [ "scanner" ] }, { "pattern": "WRTNBot", "url": "https://datadome.co/bots/wrtnbot/", "instances": [ "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; WRTNBot; +https://wrtn.ai/wrtnbot)" ], "description": "Web crawler indexing content for data aggregation and market analysis purposes", "addition_date": "2026/04/17", "tags": [ "seo" ] }, { "pattern": "abuse\\.xmco\\.fr", "url": "https://abuse.xmco.fr/", "instances": [ "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:103.0) Gecko/20100101 Firefox/103.0 abuse.xmco.fr" ], "description": "Cybersecurity crawler scanning websites for vulnerabilities and threat analysis signals", "addition_date": "2026/04/17", "tags": [ "scanner" ] }, { "pattern": "XY-Archive-Compliance", "url": "https://xy-archive.helpscoutdocs.com/article/61-does-xy-archive-have-a-dedicated-ip-address", "instances": [ "Mozilla/5.0 (compatible; XY-Archive-Compliance-Crawler; +https://archive.xyplanningnetwork.com/)", "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/127.0.0.0 Safari/537.36 (compatible; XY-Archive-Compliance-Archiver; +https://archive.xyplanningnetwork.com/)" ], "description": "Archiving compliance crawler collecting content for legal retention and eDiscovery", "addition_date": "2026/04/17", "tags": [ "archiver" ] }, { "pattern": "Yahoo Ad monitoring", "url": "https://developer.yahoo.com/api/", "instances": [ "Mozilla/5.0 (compatible; Yahoo Ad monitoring; https://help.yahoo.com/kb/yahoo-ad-monitoring-SLN24857.html)" ], "description": "Ad ecosystem crawler auditing quality safety and policy compliance across inventory", "addition_date": "2026/04/17", "tags": [ "advertising" ] }, { "pattern": "YahooMailProxy", "url": "https://help.yahoo.com/kb/yahoo-mail-proxy-SLN28749.html", "instances": [ "YahooMailProxy; https://help.yahoo.com/kb/yahoo-mail-proxy-SLN28749.html" ], "description": "Email proxy crawler fetching URLs and resources for security scanning and rendering", "addition_date": "2026/04/17", "tags": [ "social-preview" ] }, { "pattern": "YahooCacheSystem", "url": "https://developer.yahoo.com/oauth2/guide/", "instances": [ "YahooCacheSystem; YahooWebServiceClient" ], "description": "Content fetcher crawler caching web pages and thumbnails for Yahoo properties", "addition_date": "2026/04/17", "tags": [ "seo" ] }, { "pattern": "YLT Chrome", "url": "http://yellowlab.tools/", "instances": [ "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_6) AppleWebKit/537.36 (KHTML, like Gecko) YLT Chrome/85.0.4183.121 Safari/537.36" ], "description": "Performance analyzer evaluating front-end quality and optimization recommendations", "addition_date": "2026/04/17", "tags": [ "monitoring" ] }, { "pattern": "YokoyGroupAG\\/", "url": "https://yokoy.io/", "instances": [ "YokoyGroupAG/1.0" ], "description": "Expense management platform webhook client delivering events and verifying endpoints", "addition_date": "2026/04/17", "tags": [ "seo" ] }, { "pattern": "Yuuperbot", "url": "https://datadome.co/bots/yuuperbot/", "instances": [ "Yuuperbot" ], "description": "Web crawler indexing content for data aggregation market analysis and intelligence", "addition_date": "2026/04/17", "tags": [ "seo" ] }, { "pattern": "Zapier", "url": "https://zapier.com/", "instances": [ "Zapier" ], "description": "Automation client fetching web pages metadata and content for workflow integration", "addition_date": "2026/04/17", "tags": [ "seo" ] }, { "pattern": "Zendesk Webhook", "url": "https://support.zendesk.com/", "instances": [ "Zendesk Webhook" ], "description": "Support platform service validating webhook endpoints and fetching URL metadata", "addition_date": "2026/04/17", "tags": [ "seo" ] }, { "pattern": "Zombiebot\\/", "url": "http://www.zombiedomain.net/robot/", "instances": [ "Mozilla/5.0 (Windows NT 6.3; Trident/7.0; rv 11.0) like Gecko (compatible; Zombiebot/2.1; +http://www.zombiedomain.net/robot/)" ], "description": "Web crawler designed to index content for SEO analysis and competitive intelligence", "addition_date": "2026/04/17", "tags": [ "seo" ] }, { "pattern": "zzhbot", "url": "https://datadome.co/bots/zzhbot/", "instances": [ "zzhbot" ], "description": "Web crawler indexing and analyzing content for search engine and data aggregation", "addition_date": "2026/04/17", "tags": [ "seo" ] }, { "pattern": "Penthouse Critical Path CSS Generator", "url": "https://criticalcss.com/", "instances": [ "Penthouse Critical Path CSS Generator" ], "description": "CSS above-the-fold extractor generating critical path stylesheets", "addition_date": "2026/04/17", "tags": [ "seo" ] }, { "pattern": "Google-AdWords-Express", "url": "https://developers.google.com/search/docs/crawling-indexing/google-user-triggered-fetchers#googleproducer", "instances": [ "Google-AdWords-Express" ], "description": "Google Ads Express fetching advertiser landing pages for review", "addition_date": "2026/04/17", "tags": [ "advertising" ] }, { "pattern": "Notion\\/", "url": "https://www.iubenda.com/", "instances": [ "Notion/1.0 (https://notion.so; team@makenotion.com)" ], "description": "Notion workspace crawler fetching URLs for link previews", "addition_date": "2026/04/17", "tags": [ "social-preview" ] }, { "pattern": "SSL Labs$", "url": "https://www.qualys.com/apps/pci-compliance/", "instances": [ "SSL Labs" ], "description": "Qualys SSL Labs scanner testing TLS configuration and certificate grades", "addition_date": "2026/04/17", "tags": [ "scanner" ] }, { "pattern": "Skroutz ImageBot", "url": "https://www.skroutz.gr/", "instances": [ "Skroutz ImageBot v1" ], "description": "Greek price comparison platform crawler fetching product images", "addition_date": "2026/04/17", "tags": [ "seo" ] }, { "pattern": "Tumblr\\/", "url": "https://automattic.com/", "instances": [ "Tumblr/14.0.835.186" ], "description": "Tumblr social blogging platform crawler fetching link previews", "addition_date": "2026/04/17", "tags": [ "social-preview" ] }, { "pattern": "upday\\/", "url": "https://www.upday.com/", "instances": [ "Mozilla/5.0 (compatible; upday/1.0; +upday)" ], "description": "Samsung news aggregator crawler fetching articles for daily briefings", "addition_date": "2026/04/17", "tags": [ "feed-reader" ] }, { "pattern": "watchTowr", "addition_date": "2026/04/23", "url": "https://watchtowr.com", "instances": [ "Mozilla/5.0 (watchTowr; Windows NT 10.0; Win64; x64; rv:84.0) Gecko/20100101 Firefox/84.0" ], "description": "watchTowr is an external attack surface management tool that scans assets and generates findings against those assets", "tags": [ "scanner" ] }, { "pattern": "PRTG Network Monitor", "addition_date": "2026/05/18", "url": "https://www.paessler.com/manuals/prtg/http_transaction_sensor", "instances": [ "Mozilla/5.0 (compatible; PRTG Network Monitor (www.paessler.com ); Windows)" ], "description": "PRTG HTTP Transaction Sensor", "tags": [ "monitoring" ] } ]