[ { "id": "google-crawler", "categories": [ "google", "search-engine" ], "pattern": { "accepted": [ "Googlebot\\/" ], "forbidden": [] }, "url": "http://www.google.com/bot.html", "verification": [ { "type": "cidr", "sources": [ { "type": "http-json", "url": "https://developers.google.com/static/search/apis/ipranges/googlebot.json", "selector": "$.prefixes[*][\\\"ipv6Prefix\\\",\\\"ipv4Prefix\\\"]" } ] }, { "type": "dns", "masks": [ "crawl-***-***-***-***.googlebot.com", "geo-crawl-***-***-***-***.geo.googlebot.com" ] } ], "instances": { "accepted": [ "Googlebot/2.1 (+http://www.google.com/bot.html)", "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)", "Mozilla/5.0 (iPhone; CPU iPhone OS 6_0 like Mac OS X) AppleWebKit/536.26 (KHTML, like Gecko) Version/6.0 Mobile/10A5376e Safari/8536.25 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)", "Mozilla/5.0 (iPhone; CPU iPhone OS 8_3 like Mac OS X) AppleWebKit/537.36 (KHTML, like Gecko) Version/8.0 Mobile/12F70 Safari/600.1.4 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)", "Mozilla/5.0 (iPhone; CPU iPhone OS 8_3 like Mac OS X) AppleWebKit/600.1.4 (KHTML, like Gecko) Version/8.0 Mobile/12F70 Safari/600.1.4 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)", "Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/27.0.1453 Mobile Safari/537.36 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)", "Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2272.96 Mobile Safari/537.36 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)", "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Googlebot/2.1; +http://www.google.com/bot.html) Safari/537.36" ], "rejected": [] }, "aliases": [ "GoogleBot" ] }, { "id": "google-crawler-mobile", "categories": [ "google", "search-engine" ], "pattern": { "accepted": [ "Googlebot-Mobile" ], "forbidden": [] }, "verification": [], "instances": { "accepted": [ "DoCoMo/2.0 N905i(c100;TB;W24H16) (compatible; Googlebot-Mobile/2.1; +http://www.google.com/bot.html)", "Mozilla/5.0 (iPhone; CPU iPhone OS 6_0 like Mac OS X) AppleWebKit/536.26 (KHTML, like Gecko) Version/6.0 Mobile/10A5376e Safari/8536.25 (compatible; Googlebot-Mobile/2.1; +http://www.google.com/bot.html)", "Mozilla/5.0 (iPhone; U; CPU iPhone OS 4_1 like Mac OS X; en-us) AppleWebKit/532.9 (KHTML, like Gecko) Version/4.0.5 Mobile/8B117 Safari/6531.22.7 (compatible; Googlebot-Mobile/2.1; +http://www.google.com/bot.html)", "Nokia6820/2.0 (4.83) Profile/MIDP-1.0 Configuration/CLDC-1.0 (compatible; Googlebot-Mobile/2.1; +http://www.google.com/bot.html)", "SAMSUNG-SGH-E250/1.0 Profile/MIDP-2.0 Configuration/CLDC-1.1 UP.Browser/6.2.3.3.c.1.101 (GUI) MMP/2.0 (compatible; Googlebot-Mobile/2.1; +http://www.google.com/bot.html)" ], "rejected": [] } }, { "id": "google-crawler-image", "categories": [ "google", "search-engine" ], "pattern": { "accepted": [ "Googlebot-Image" ], "forbidden": [] }, "verification": [ { "type": "cidr", "sources": [ { "type": "http-json", "url": "https://developers.google.com/static/search/apis/ipranges/googlebot.json", "selector": "$.prefixes[*][\\\"ipv6Prefix\\\",\\\"ipv4Prefix\\\"]" } ] }, { "type": "dns", "masks": [ "crawl-***-***-***-***.googlebot.com", "geo-crawl-***-***-***-***.geo.googlebot.com" ] } ], "instances": { "accepted": [ "Googlebot-Image/1.0" ], "rejected": [] }, "aliases": [ "GoogleBotImage" ] }, { "id": "google-crawler-news", "categories": [ "google", "search-engine" ], "pattern": { "accepted": [ "Googlebot-News" ], "forbidden": [] }, "verification": [ { "type": "cidr", "sources": [ { "type": "http-json", "url": "https://developers.google.com/static/search/apis/ipranges/googlebot.json", "selector": "$.prefixes[*][\\\"ipv6Prefix\\\",\\\"ipv4Prefix\\\"]" } ] }, { "type": "dns", "masks": [ "crawl-***-***-***-***.googlebot.com", "geo-crawl-***-***-***-***.geo.googlebot.com" ] } ], "instances": { "accepted": [ "Googlebot-News" ], "rejected": [] } }, { "id": "google-crawler-video", "categories": [ "google", "search-engine" ], "pattern": { "accepted": [ "Googlebot-Video" ], "forbidden": [] }, "verification": [ { "type": "cidr", "sources": [ { "type": "http-json", "url": "https://developers.google.com/static/search/apis/ipranges/googlebot.json", "selector": "$.prefixes[*][\\\"ipv6Prefix\\\",\\\"ipv4Prefix\\\"]" } ] }, { "type": "dns", "masks": [ "crawl-***-***-***-***.googlebot.com", "geo-crawl-***-***-***-***.geo.googlebot.com" ] } ], "instances": { "accepted": [ "Googlebot-Video/1.0" ], "rejected": [] } }, { "id": "google-adsbot", "categories": [ "google", "advertising" ], "pattern": { "accepted": [ "AdsBot-Google([^-]|$)" ], "forbidden": [] }, "url": "https://support.google.com/webmasters/answer/1061943?hl=en", "verification": [ { "type": "cidr", "sources": [ { "type": "http-json", "url": "https://developers.google.com/static/search/apis/ipranges/special-crawlers.json", "selector": "$.prefixes[*][\\\"ipv6Prefix\\\",\\\"ipv4Prefix\\\"]" } ] }, { "type": "dns", "masks": [ "rate-limited-proxy-***-***-***-***.google.com" ] } ], "instances": { "accepted": [ "AdsBot-Google (+http://www.google.com/adsbot.html)" ], "rejected": [] }, "aliases": [ "AdsBotGoogle" ] }, { "id": "google-adsbot-mobile", "categories": [ "google", "advertising" ], "pattern": { "accepted": [ "AdsBot-Google-Mobile" ], "forbidden": [] }, "addition_date": "2017/08/21", "url": "https://support.google.com/adwords/answer/2404197", "verification": [ { "type": "cidr", "sources": [ { "type": "http-json", "url": "https://developers.google.com/static/search/apis/ipranges/special-crawlers.json", "selector": "$.prefixes[*][\\\"ipv6Prefix\\\",\\\"ipv4Prefix\\\"]" } ] }, { "type": "dns", "masks": [ "rate-limited-proxy-***-***-***-***.google.com" ] } ], "instances": { "accepted": [ "AdsBot-Google-Mobile-Apps", "Mozilla/5.0 (Linux; Android 5.0; SM-G920A) AppleWebKit (KHTML, like Gecko) Chrome Mobile Safari (compatible; AdsBot-Google-Mobile; +http://www.google.com/mobile/adsbot.html)", "Mozilla/5.0 (iPhone; CPU iPhone OS 9_1 like Mac OS X) AppleWebKit/601.1.46 (KHTML, like Gecko) Version/9.0 Mobile/13B143 Safari/601.1 (compatible; AdsBot-Google-Mobile; +http://www.google.com/mobile/adsbot.html)" ], "rejected": [] }, "aliases": [ "GoogleAdsBot" ] }, { "id": "google-ads-conversions", "categories": [ "google", "advertising" ], "pattern": { "accepted": [ "Google-Ads-Conversions" ], "forbidden": [] }, "url": "https://developers.google.com/google-ads/api/docs/conversions/upload-online", "verification": [], "instances": { "accepted": [ "Mozilla/5.0 AppleWebKit/537.36 Chrome/139.0.7258.127 Safari/537.36 Google-Ads-Conversions" ], "rejected": [] } }, { "id": "google-feedfetcher", "categories": [ "google", "feedfetcher" ], "pattern": { "accepted": [ "Feedfetcher-Google" ], "forbidden": [] }, "addition_date": "2018/06/27", "url": "https://support.google.com/webmasters/answer/178852", "verification": [ { "type": "cidr", "sources": [ { "type": "http-json", "url": "https://developers.google.com/static/search/apis/ipranges/user-triggered-fetchers.json", "selector": "$.prefixes[*][\\\"ipv6Prefix\\\",\\\"ipv4Prefix\\\"]" }, { "type": "http-json", "url": "https://developers.google.com/static/search/apis/ipranges/user-triggered-fetchers-google.json", "selector": "$.prefixes[*][\\\"ipv6Prefix\\\",\\\"ipv4Prefix\\\"]" } ] }, { "type": "dns", "masks": [ "***-***-***-***.gae.googleusercontent.com", "google-proxy-***-***-***-***.google.com" ] } ], "instances": { "accepted": [ "Feedfetcher-Google; (+http://www.google.com/feedfetcher.html; 1 subscribers; feed-id=728742641706423)" ], "rejected": [] } }, { "id": "google-adsense", "categories": [ "google", "advertising" ], "pattern": { "accepted": [ "Mediapartners-Google" ], "forbidden": [] }, "url": "https://support.google.com/webmasters/answer/1061943?hl=en", "verification": [ { "type": "cidr", "sources": [ { "type": "http-json", "url": "https://developers.google.com/static/search/apis/ipranges/special-crawlers.json", "selector": "$.prefixes[*][\\\"ipv6Prefix\\\",\\\"ipv4Prefix\\\"]" } ] }, { "type": "dns", "masks": [ "rate-limited-proxy-***-***-***-***.google.com" ] } ], "instances": { "accepted": [ "Mediapartners-Google", "Mozilla/5.0 (compatible; MSIE or Firefox mutant; not on Windows server;) Daumoa/4.0 (Following Mediapartners-Google)", "Mozilla/5.0 (iPhone; U; CPU iPhone OS 10_0 like Mac OS X; en-us) AppleWebKit/602.1.38 (KHTML, like Gecko) Version/10.0 Mobile/14A5297c Safari/602.1 (compatible; Mediapartners-Google/2.1; +http://www.google.com/bot.html)", "Mozilla/5.0 (iPhone; U; CPU iPhone OS 4_1 like Mac OS X; en-us) AppleWebKit/532.9 (KHTML, like Gecko) Version/4.0.5 Mobile/8B117 Safari/6531.22.7 (compatible; Mediapartners-Google/2.1; +http://www.google.com/bot.html)" ], "rejected": [] }, "aliases": [ "MediapartnersGoogle" ] }, { "id": "google-adsense-googlebot", "categories": [ "google", "advertising" ], "pattern": { "accepted": [ "Mediapartners \\(Googlebot\\)" ], "forbidden": [] }, "addition_date": "2017/08/08", "url": "https://support.google.com/webmasters/answer/1061943?hl=en", "verification": [], "instances": { "accepted": [], "rejected": [] } }, { "id": "google-push-notifications", "categories": [ "google" ], "pattern": { "accepted": [ "APIs-Google" ], "forbidden": [] }, "addition_date": "2017/08/08", "url": "https://support.google.com/webmasters/answer/1061943?hl=en", "verification": [ { "type": "cidr", "sources": [ { "type": "http-json", "url": "https://developers.google.com/static/search/apis/ipranges/special-crawlers.json", "selector": "$.prefixes[*][\\\"ipv6Prefix\\\",\\\"ipv4Prefix\\\"]" } ] }, { "type": "dns", "masks": [ "rate-limited-proxy-***-***-***-***.google.com" ] } ], "instances": { "accepted": [ "APIs-Google (+https://developers.google.com/webmasters/APIs-Google.html)" ], "rejected": [] }, "aliases": [ "APIsGoogle" ] }, { "id": "google-inspection-tool", "categories": [ "google", "search-engine" ], "pattern": { "accepted": [ "Google-InspectionTool" ], "forbidden": [] }, "url": "https://developers.google.com/search/docs/crawling-indexing/overview-google-crawlers", "verification": [ { "type": "cidr", "sources": [ { "type": "http-json", "url": "https://developers.google.com/static/search/apis/ipranges/googlebot.json", "selector": "$.prefixes[*][\\\"ipv6Prefix\\\",\\\"ipv4Prefix\\\"]" } ] }, { "type": "dns", "masks": [ "crawl-***-***-***-***.googlebot.com", "geo-crawl-***-***-***-***.geo.googlebot.com" ] } ], "instances": { "accepted": [ "Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/W.X.Y.Z Mobile Safari/537.36 (compatible; Google-InspectionTool/1.0)", "Mozilla/5.0 (compatible; Google-InspectionTool/1.0)" ], "rejected": [] } }, { "id": "google-crawler-store", "categories": [ "google", "search-engine" ], "pattern": { "accepted": [ "Storebot-Google" ], "forbidden": [] }, "url": "https://developers.google.com/search/docs/crawling-indexing/overview-google-crawlers", "verification": [ { "type": "cidr", "sources": [ { "type": "http-json", "url": "https://developers.google.com/static/search/apis/ipranges/googlebot.json", "selector": "$.prefixes[*][\\\"ipv6Prefix\\\",\\\"ipv4Prefix\\\"]" } ] }, { "type": "dns", "masks": [ "crawl-***-***-***-***.googlebot.com", "geo-crawl-***-***-***-***.geo.googlebot.com" ] } ], "instances": { "accepted": [ "Mozilla/5.0 (X11; Linux x86_64; Storebot-Google/1.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.88 Safari/537.36", "Mozilla/5.0 (Linux; Android 8.0; Pixel 2 Build/OPD3.170816.012; Storebot-Google/1.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.138 Mobile Safari/537.36" ], "rejected": [] }, "aliases": [ "StorebotGoogle" ] }, { "id": "google-crawler-other", "categories": [ "google", "search-engine" ], "pattern": { "accepted": [ "GoogleOther" ], "forbidden": [] }, "url": "https://developers.google.com/search/docs/crawling-indexing/overview-google-crawlers", "verification": [ { "type": "cidr", "sources": [ { "type": "http-json", "url": "https://developers.google.com/static/search/apis/ipranges/googlebot.json", "selector": "$.prefixes[*][\\\"ipv6Prefix\\\",\\\"ipv4Prefix\\\"]" } ] }, { "type": "dns", "masks": [ "crawl-***-***-***-***.googlebot.com", "geo-crawl-***-***-***-***.geo.googlebot.com" ] } ], "instances": { "accepted": [ "GoogleOther" ], "rejected": [] } }, { "id": "bing-crawler", "categories": [ "microsoft", "search-engine" ], "pattern": { "accepted": [ "bingbot" ], "forbidden": [] }, "url": "http://www.bing.com/bingbot.htm", "verification": [ { "type": "cidr", "sources": [ { "type": "http-json", "url": "https://www.bing.com/toolbox/bingbot.json", "selector": "$.prefixes[*][\\\"ipv6Prefix\\\",\\\"ipv4Prefix\\\"]" } ] }, { "type": "dns", "masks": [ "@.search.msn.com" ] } ], "instances": { "accepted": [ "Mozilla/5.0 (Windows Phone 8.1; ARM; Trident/7.0; Touch; rv:11.0; IEMobile/11.0; NOKIA; Lumia 530) like Gecko (compatible; adidxbot/2.0; +http://www.bing.com/bingbot.htm)", "Mozilla/5.0 (compatible; adidxbot/2.0; http://www.bing.com/bingbot.htm)", "Mozilla/5.0 (compatible; adidxbot/2.0; +http://www.bing.com/bingbot.htm)", "Mozilla/5.0 (compatible; bingbot/2.0; http://www.bing.com/bingbot.htm)", "Mozilla/5.0 (compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm", "Mozilla/5.0 (compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm)", "Mozilla/5.0 (compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm) SitemapProbe", "Mozilla/5.0 (iPhone; CPU iPhone OS 7_0 like Mac OS X) AppleWebKit/537.51.1 (KHTML, like Gecko) Version/7.0 Mobile/11A465 Safari/9537.53 (compatible; adidxbot/2.0; http://www.bing.com/bingbot.htm)", "Mozilla/5.0 (iPhone; CPU iPhone OS 7_0 like Mac OS X) AppleWebKit/537.51.1 (KHTML, like Gecko) Version/7.0 Mobile/11A465 Safari/9537.53 (compatible; adidxbot/2.0; +http://www.bing.com/bingbot.htm)", "Mozilla/5.0 (iPhone; CPU iPhone OS 7_0 like Mac OS X) AppleWebKit/537.51.1 (KHTML, like Gecko) Version/7.0 Mobile/11A465 Safari/9537.53 (compatible; bingbot/2.0; http://www.bing.com/bingbot.htm)", "Mozilla/5.0 (iPhone; CPU iPhone OS 7_0 like Mac OS X) AppleWebKit/537.51.1 (KHTML, like Gecko) Version/7.0 Mobile/11A465 Safari/9537.53 (compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm)", "Mozilla/5.0 (seoanalyzer; compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm)", "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm) Safari/537.36", "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm) Chrome/103.0.5060.134 Safari/537.36" ], "rejected": [] }, "aliases": [ "BingBot" ] }, { "id": "yahoo-crawler", "categories": [ "yahoo", "search-engine" ], "pattern": { "accepted": [ "Slurp" ], "forbidden": [] }, "url": "http://help.yahoo.com/help/us/ysearch/slurp", "verification": [], "instances": { "accepted": [ "Mozilla/5.0 (compatible; Yahoo! Slurp/3.0; http://help.yahoo.com/help/us/ysearch/slurp)", "Mozilla/5.0 (compatible; Yahoo! Slurp; http://help.yahoo.com/help/us/ysearch/slurp)", "Mozilla/5.0 (compatible; Yahoo! Slurp China; http://misc.yahoo.com.cn/help.html)" ], "rejected": [] }, "aliases": [ "SlurpBot" ] }, { "id": "wget", "categories": [ "tool" ], "pattern": { "accepted": [ "[wW]get" ], "forbidden": [] }, "verification": [], "instances": { "accepted": [ "WGETbot/1.0 (+http://wget.alanreed.org)", "Wget/1.14 (linux-gnu)", "Wget/1.20.3 (linux-gnu)" ], "rejected": [] } }, { "id": "linkedin-crawler", "categories": [ "social" ], "pattern": { "accepted": [ "LinkedInBot" ], "forbidden": [] }, "verification": [], "instances": { "accepted": [ "LinkedInBot/1.0 (compatible; Mozilla/5.0; Jakarta Commons-HttpClient/3.1 +http://www.linkedin.com)", "LinkedInBot/1.0 (compatible; Mozilla/5.0; Jakarta Commons-HttpClient/4.3 +http://www.linkedin.com)", "LinkedInBot/1.0 (compatible; Mozilla/5.0; Apache-HttpClient +http://www.linkedin.com)" ], "rejected": [] }, "aliases": [ "LinkedInBot" ] }, { "id": "python-urllib", "categories": [ "programmatic" ], "pattern": { "accepted": [ "Python-urllib" ], "forbidden": [] }, "verification": [], "instances": { "accepted": [ "Python-urllib/1.17", "Python-urllib/2.5", "Python-urllib/2.6", "Python-urllib/2.7", "Python-urllib/3.1", "Python-urllib/3.2", "Python-urllib/3.3", "Python-urllib/3.4", "Python-urllib/3.5", "Python-urllib/3.6", "Python-urllib/3.7" ], "rejected": [] } }, { "id": "python-requests", "categories": [ "programmatic" ], "pattern": { "accepted": [ "python-requests" ], "forbidden": [] }, "addition_date": "2018/05/27", "verification": [], "instances": { "accepted": [ "python-requests/2.9.2", "python-requests/2.11.1", "python-requests/2.18.4", "python-requests/2.19.1", "python-requests/2.20.0", "python-requests/2.21.0", "python-requests/2.22.0" ], "rejected": [] } }, { "id": "python-aiohttp", "categories": [ "programmatic" ], "pattern": { "accepted": [ "aiohttp" ], "forbidden": [] }, "addition_date": "2019/12/23", "verification": [], "instances": { "accepted": [ "Python/3.9 aiohttp/3.7.3", "Python/3.8 aiohttp/3.7.2", "Python/3.7 aiohttp/3.6.2a2" ], "rejected": [] }, "url": "https://docs.aiohttp.org/en/stable/" }, { "id": "python-httpx", "categories": [ "programmatic" ], "pattern": { "accepted": [ "httpx" ], "forbidden": [] }, "addition_date": "2019/12/23", "verification": [], "instances": { "accepted": [ "python-httpx/0.16.1", "python-httpx/0.13.0.dev1" ], "rejected": [] }, "url": "https://www.python-httpx.org" }, { "id": "perl-libwww", "categories": [ "programmatic" ], "pattern": { "accepted": [ "libwww-perl" ], "forbidden": [] }, "verification": [], "instances": { "accepted": [ "2Bone_LinkChecker/1.0 libwww-perl/6.03", "2Bone_LinkChkr/1.0 libwww-perl/6.03", "amibot - http://www.amidalla.de - tech@amidalla.com libwww-perl/5.831" ], "rejected": [] } }, { "id": "java-httpunit", "categories": [ "programmatic" ], "pattern": { "accepted": [ "httpunit" ], "forbidden": [] }, "verification": [], "instances": { "accepted": [ "httpunit/1.x" ], "rejected": [] } }, { "id": "nutch", "categories": [ "tool" ], "pattern": { "accepted": [ "Nutch" ], "forbidden": [] }, "verification": [], "instances": { "accepted": [ "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/605.1.16 (KHTML, like Gecko; compatible; Friendly_Crawler/2.0) Chrome/120.0.6099.217 Safari/605.1.15/Nutch-1.20-SNAPSHOT", "NutchCVS/0.7.1 (Nutch; http://lucene.apache.org/nutch/bot.html; nutch-agent@lucene.apache.org)", "istellabot-nutch/Nutch-1.10" ], "rejected": [] } }, { "id": "go-http", "categories": [ "programmatic" ], "pattern": { "accepted": [ "Go-http-client" ], "forbidden": [] }, "addition_date": "2016/03/26", "url": "https://golang.org/pkg/net/http/", "verification": [], "instances": { "accepted": [ "Go-http-client/1.1", "Go-http-client/2.0" ], "rejected": [] } }, { "id": "php-phpcrawl", "categories": [ "programmatic" ], "pattern": { "accepted": [ "phpcrawl" ], "forbidden": [] }, "addition_date": "2012/09/17", "url": "http://phpcrawl.cuab.de/", "verification": [], "instances": { "accepted": [ "phpcrawl" ], "rejected": [] } }, { "id": "msn-crawler", "categories": [ "microsoft", "advertising" ], "pattern": { "accepted": [ "msnbot" ], "forbidden": [] }, "url": "http://search.msn.com/msnbot.htm", "verification": [], "instances": { "accepted": [ "adidxbot/1.1 (+http://search.msn.com/msnbot.htm)", "adidxbot/2.0 (+http://search.msn.com/msnbot.htm)", "librabot/1.0 (+http://search.msn.com/msnbot.htm)", "librabot/2.0 (+http://search.msn.com/msnbot.htm)", "msnbot-NewsBlogs/2.0b (+http://search.msn.com/msnbot.htm)", "msnbot-UDiscovery/2.0b (+http://search.msn.com/msnbot.htm)", "msnbot-media/1.0 (+http://search.msn.com/msnbot.htm)", "msnbot-media/1.1 (+http://search.msn.com/msnbot.htm)", "msnbot-media/2.0b (+http://search.msn.com/msnbot.htm)", "msnbot/1.0 (+http://search.msn.com/msnbot.htm)", "msnbot/1.1 (+http://search.msn.com/msnbot.htm)", "msnbot/2.0b (+http://search.msn.com/msnbot.htm)", "msnbot/2.0b (+http://search.msn.com/msnbot.htm).", "msnbot/2.0b (+http://search.msn.com/msnbot.htm)._" ], "rejected": [] }, "aliases": [ "MsnBot" ] }, { "id": "jyxo-crawler", "categories": [ "unknown" ], "pattern": { "accepted": [ "jyxobot" ], "forbidden": [] }, "verification": [], "instances": { "accepted": [], "rejected": [] } }, { "id": "fast-crawler", "categories": [ "unknown" ], "pattern": { "accepted": [ "FAST-WebCrawler" ], "forbidden": [] }, "verification": [], "instances": { "accepted": [ "FAST-WebCrawler/3.6/FirstPage (atw-crawler at fast dot no;http://fast.no/support/crawler.asp)", "FAST-WebCrawler/3.7 (atw-crawler at fast dot no; http://fast.no/support/crawler.asp)", "FAST-WebCrawler/3.7/FirstPage (atw-crawler at fast dot no;http://fast.no/support/crawler.asp)", "FAST-WebCrawler/3.8" ], "rejected": [] } }, { "id": "fast-crawler-enterprise", "categories": [ "unknown" ], "pattern": { "accepted": [ "FAST Enterprise Crawler" ], "forbidden": [] }, "verification": [], "instances": { "accepted": [ "FAST Enterprise Crawler 6 / Scirus scirus-crawler@fast.no; http://www.scirus.com/srsapp/contactus/", "FAST Enterprise Crawler 6 used by Schibsted (webcrawl@schibstedsok.no)" ], "rejected": [] } }, { "id": "biglotron", "categories": [ "unknown" ], "pattern": { "accepted": [ "BIGLOTRON" ], "forbidden": [] }, "verification": [], "instances": { "accepted": [ "BIGLOTRON (Beta 2;GNU/Linux)" ], "rejected": [] } }, { "id": "ask-crawler", "categories": [ "search-engine" ], "pattern": { "accepted": [ "Teoma" ], "forbidden": [] }, "verification": [], "instances": { "accepted": [ "Mozilla/2.0 (compatible; Ask Jeeves/Teoma; +http://sp.ask.com/docs/about/tech_crawling.html)", "Mozilla/2.0 (compatible; Ask Jeeves/Teoma; +http://about.ask.com/en/docs/about/webmasters.shtml)" ], "rejected": [] }, "url": "http://about.ask.com/en/docs/about/webmasters.shtml" }, { "id": "convera-crawler", "categories": [ "unknown" ], "pattern": { "accepted": [ "convera" ], "forbidden": [] }, "verification": [], "instances": { "accepted": [ "ConveraCrawler/0.9e (+http://ews.converasearch.com/crawl.htm)" ], "rejected": [] }, "url": "http://ews.converasearch.com/crawl.htm" }, { "id": "seekbot-crawler", "categories": [ "unknown" ], "pattern": { "accepted": [ "seekbot" ], "forbidden": [] }, "verification": [], "instances": { "accepted": [ "Seekbot/1.0 (http://www.seekbot.net/bot.html) RobotsTxtFetcher/1.2" ], "rejected": [] }, "url": "http://www.seekbot.net/bot.html" }, { "id": "gigablast-crawler", "categories": [ "unknown" ], "pattern": { "accepted": [ "Gigabot" ], "forbidden": [] }, "verification": [], "instances": { "accepted": [ "Gigabot/1.0", "Gigabot/2.0 (http://www.gigablast.com/spider.html)" ], "rejected": [] }, "url": "http://www.gigablast.com/spider.html" }, { "id": "gigablast-crawler-oss", "categories": [ "tool", "programmatic" ], "pattern": { "accepted": [ "Gigablast" ], "forbidden": [] }, "verification": [], "instances": { "accepted": [ "GigablastOpenSource/1.0" ], "rejected": [] }, "url": "https://github.com/gigablast/open-source-search-engine" }, { "id": "amazon-alexa-crawler", "categories": [ "amazon" ], "pattern": { "accepted": [ "exabot" ], "forbidden": [] }, "verification": [], "instances": { "accepted": [ "Mozilla/5.0 (compatible; Alexabot/1.0; +http://www.alexa.com/help/certifyscan; certifyscan@alexa.com)", "Mozilla/5.0 (compatible; Exabot PyExalead/3.0; +http://www.exabot.com/go/robot)", "Mozilla/5.0 (compatible; Exabot-Images/3.0; +http://www.exabot.com/go/robot)", "Mozilla/5.0 (compatible; Exabot/3.0 (BiggerBetter); +http://www.exabot.com/go/robot)", "Mozilla/5.0 (compatible; Exabot/3.0; +http://www.exabot.com/go/robot)", "Mozilla/5.0 (compatible; Exabot/3.0; http://www.exabot.com/go/robot)" ], "rejected": [] } }, { "id": "ia-archiver", "categories": [ "unknown" ], "pattern": { "accepted": [ "ia_archiver" ], "forbidden": [] }, "verification": [], "instances": { "accepted": [ "ia_archiver (+http://www.alexa.com/site/help/webmasters; crawler@alexa.com)", "ia_archiver-web.archive.org" ], "rejected": [] } }, { "id": "ginger-crawler", "categories": [ "unknown" ], "pattern": { "accepted": [ "GingerCrawler" ], "forbidden": [] }, "verification": [], "instances": { "accepted": [ "GingerCrawler/1.0 (Language Assistant for Dyslexics; www.gingersoftware.com/crawler_agent.htm; support at ginger software dot com)" ], "rejected": [] } }, { "id": "webmon", "categories": [ "unknown" ], "pattern": { "accepted": [ "webmon " ], "forbidden": [] }, "verification": [], "instances": { "accepted": [], "rejected": [] } }, { "id": "httrack", "categories": [ "unknown" ], "pattern": { "accepted": [ "HTTrack" ], "forbidden": [] }, "verification": [], "instances": { "accepted": [ "Mozilla/4.5 (compatible; HTTrack 3.0x; Windows 98)" ], "rejected": [] } }, { "id": "grub", "categories": [ "unknown" ], "pattern": { "accepted": [ "grub\\.org" ], "forbidden": [] }, "verification": [], "instances": { "accepted": [ "Mozilla/4.0 (compatible; grub-client-0.3.0; Crawl your own stuff with http://grub.org)", "Mozilla/4.0 (compatible; grub-client-1.0.4; Crawl your own stuff with http://grub.org)", "Mozilla/4.0 (compatible; grub-client-1.0.5; Crawl your own stuff with http://grub.org)", "Mozilla/4.0 (compatible; grub-client-1.0.6; Crawl your own stuff with http://grub.org)", "Mozilla/4.0 (compatible; grub-client-1.0.7; Crawl your own stuff with http://grub.org)", "Mozilla/4.0 (compatible; grub-client-1.1.1; Crawl your own stuff with http://grub.org)", "Mozilla/4.0 (compatible; grub-client-1.2.1; Crawl your own stuff with http://grub.org)", "Mozilla/4.0 (compatible; grub-client-1.3.1; Crawl your own stuff with http://grub.org)", "Mozilla/4.0 (compatible; grub-client-1.3.7; Crawl your own stuff with http://grub.org)", "Mozilla/4.0 (compatible; grub-client-1.4.3; Crawl your own stuff with http://grub.org)", "Mozilla/4.0 (compatible; grub-client-1.5.3; Crawl your own stuff with http://grub.org)" ], "rejected": [] } }, { "id": "usine-nouvelle-crawler", "categories": [ "unknown" ], "pattern": { "accepted": [ "UsineNouvelleCrawler" ], "forbidden": [] }, "verification": [], "instances": { "accepted": [], "rejected": [] } }, { "id": "antibot", "categories": [ "unknown" ], "pattern": { "accepted": [ "antibot" ], "forbidden": [] }, "verification": [], "instances": { "accepted": [], "rejected": [] } }, { "id": "loop-crawler", "categories": [ "unknown" ], "pattern": { "accepted": [ "netresearchserver" ], "forbidden": [] }, "verification": [], "instances": { "accepted": [], "rejected": [] } }, { "id": "entireweb-crawler", "categories": [ "search-engine" ], "pattern": { "accepted": [ "speedy" ], "forbidden": [] }, "verification": [], "instances": { "accepted": [ "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US) Speedy Spider (http://www.entireweb.com/about/search_tech/speedy_spider/)", "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US) Speedy Spider for SpeedyAds (http://www.entireweb.com/about/search_tech/speedy_spider/)", "Mozilla/5.0 (compatible; Speedy Spider; http://www.entireweb.com/about/search_tech/speedy_spider/)", "Speedy Spider (Entireweb; Beta/1.2; http://www.entireweb.com/about/search_tech/speedyspider/)", "Speedy Spider (http://www.entireweb.com/about/search_tech/speedy_spider/)" ], "rejected": [] } }, { "id": "fluffy", "categories": [ "unknown" ], "pattern": { "accepted": [ "fluffy" ], "forbidden": [] }, "verification": [], "instances": { "accepted": [], "rejected": [] } }, { "id": "leipzig-findlinks", "categories": [ "academic" ], "pattern": { "accepted": [ "findlink" ], "forbidden": [] }, "verification": [], "instances": { "accepted": [ "findlinks/1.0 (+http://wortschatz.uni-leipzig.de/findlinks/)", "findlinks/1.1.3-beta8 (+http://wortschatz.uni-leipzig.de/findlinks/)", "findlinks/1.1.3-beta9 (+http://wortschatz.uni-leipzig.de/findlinks/)", "findlinks/1.1.5-beta7 (+http://wortschatz.uni-leipzig.de/findlinks/)", "findlinks/1.1.6-beta1 (+http://wortschatz.uni-leipzig.de/findlinks/)", "findlinks/1.1.6-beta1 (+http://wortschatz.uni-leipzig.de/findlinks/; YaCy 0.1; yacy.net)", "findlinks/1.1.6-beta2 (+http://wortschatz.uni-leipzig.de/findlinks/)", "findlinks/1.1.6-beta3 (+http://wortschatz.uni-leipzig.de/findlinks/)", "findlinks/1.1.6-beta4 (+http://wortschatz.uni-leipzig.de/findlinks/)", "findlinks/1.1.6-beta5 (+http://wortschatz.uni-leipzig.de/findlinks/)", "findlinks/1.1.6-beta6 (+http://wortschatz.uni-leipzig.de/findlinks/)", "findlinks/2.0 (+http://wortschatz.uni-leipzig.de/findlinks/)", "findlinks/2.0.1 (+http://wortschatz.uni-leipzig.de/findlinks/)", "findlinks/2.0.2 (+http://wortschatz.uni-leipzig.de/findlinks/)", "findlinks/2.0.4 (+http://wortschatz.uni-leipzig.de/findlinks/)", "findlinks/2.0.5 (+http://wortschatz.uni-leipzig.de/findlinks/)", "findlinks/2.0.9 (+http://wortschatz.uni-leipzig.de/findlinks/)", "findlinks/2.1 (+http://wortschatz.uni-leipzig.de/findlinks/)", "findlinks/2.1.3 (+http://wortschatz.uni-leipzig.de/findlinks/)", "findlinks/2.1.5 (+http://wortschatz.uni-leipzig.de/findlinks/)", "findlinks/2.2 (+http://wortschatz.uni-leipzig.de/findlinks/)", "findlinks/2.5 (+http://wortschatz.uni-leipzig.de/findlinks/)", "findlinks/2.6 (+http://wortschatz.uni-leipzig.de/findlinks/)" ], "rejected": [] } }, { "id": "microsoft-research-crawler", "categories": [ "microsoft" ], "pattern": { "accepted": [ "msrbot" ], "forbidden": [] }, "verification": [], "instances": { "accepted": [], "rejected": [] } }, { "id": "panscient-crawler", "categories": [ "unknown" ], "pattern": { "accepted": [ "panscient" ], "forbidden": [] }, "verification": [], "instances": { "accepted": [ "panscient.com" ], "rejected": [] } }, { "id": "yacy-crawler", "categories": [ "search-engine" ], "pattern": { "accepted": [ "yacybot" ], "forbidden": [] }, "verification": [], "instances": { "accepted": [ "yacybot (/global; amd64 FreeBSD 10.3-RELEASE; java 1.8.0_77; GMT/en) http://yacy.net/bot.html", "yacybot (/global; amd64 FreeBSD 10.3-RELEASE-p7; java 1.7.0_95; GMT/en) http://yacy.net/bot.html", "yacybot (-global; amd64 FreeBSD 9.2-RELEASE-p10; java 1.7.0_65; Europe/en) http://yacy.net/bot.html", "yacybot (/global; amd64 Linux 2.6.32-042stab093.4; java 1.7.0_65; Etc/en) http://yacy.net/bot.html", "yacybot (/global; amd64 Linux 2.6.32-042stab094.8; java 1.7.0_79; America/en) http://yacy.net/bot.html", "yacybot (/global; amd64 Linux 2.6.32-042stab108.8; java 1.7.0_91; America/en) http://yacy.net/bot.html", "yacybot (-global; amd64 Linux 2.6.32-042stab111.11; java 1.7.0_79; Europe/en) http://yacy.net/bot.html", "yacybot (-global; amd64 Linux 2.6.32-042stab116.1; java 1.7.0_79; Europe/en) http://yacy.net/bot.html", "yacybot (/global; amd64 Linux 2.6.32-573.3.1.el6.x86_64; java 1.7.0_85; Europe/en) http://yacy.net/bot.html", "yacybot (-global; amd64 Linux 3.10.0-229.4.2.el7.x86_64; java 1.7.0_79; Europe/en) http://yacy.net/bot.html", "yacybot (-global; amd64 Linux 3.10.0-229.4.2.el7.x86_64; java 1.8.0_45; Europe/en) http://yacy.net/bot.html", "yacybot (/global; amd64 Linux 3.10.0-229.7.2.el7.x86_64; java 1.8.0_45; Europe/en) http://yacy.net/bot.html", "yacybot (/global; amd64 Linux 3.10.0-327.22.2.el7.x86_64; java 1.7.0_101; Etc/en) http://yacy.net/bot.html", "yacybot (/global; amd64 Linux 3.11.10-21-desktop; java 1.7.0_51; America/en) http://yacy.net/bot.html", "yacybot (/global; amd64 Linux 3.12.1; java 1.7.0_65; Europe/en) http://yacy.net/bot.html", "yacybot (/global; amd64 Linux 3.13.0-042stab093.4; java 1.7.0_79; Europe/de) http://yacy.net/bot.html", "yacybot (/global; amd64 Linux 3.13.0-042stab093.4; java 1.7.0_79; Europe/en) http://yacy.net/bot.html", "yacybot (/global; amd64 Linux 3.13.0-45-generic; java 1.7.0_75; Europe/en) http://yacy.net/bot.html", "yacybot (-global; amd64 Linux 3.13.0-61-generic; java 1.7.0_79; Europe/en) http://yacy.net/bot.html", "yacybot (/global; amd64 Linux 3.13.0-74-generic; java 1.7.0_91; Europe/en) http://yacy.net/bot.html", "yacybot (/global; amd64 Linux 3.13.0-83-generic; java 1.7.0_95; Europe/de) http://yacy.net/bot.html", "yacybot (/global; amd64 Linux 3.13.0-83-generic; java 1.7.0_95; Europe/en) http://yacy.net/bot.html", "yacybot (/global; amd64 Linux 3.13.0-85-generic; java 1.7.0_101; Europe/en) http://yacy.net/bot.html", "yacybot (/global; amd64 Linux 3.13.0-85-generic; java 1.7.0_95; Europe/en) http://yacy.net/bot.html", "yacybot (/global; amd64 Linux 3.13.0-88-generic; java 1.7.0_101; Europe/en) http://yacy.net/bot.html", "yacybot (/global; amd64 Linux 3.14-0.bpo.1-amd64; java 1.7.0_55; Europe/de) http://yacy.net/bot.html", "yacybot (/global; amd64 Linux 3.14.32-xxxx-grs-ipv6-64; java 1.7.0_75; Europe/en) http://yacy.net/bot.html", "yacybot (-global; amd64 Linux 3.14.32-xxxx-grs-ipv6-64; java 1.8.0_111; Europe/de) http://yacy.net/bot.html", "yacybot (/global; amd64 Linux 3.16.0-4-amd64; java 1.7.0_111; Europe/de) http://yacy.net/bot.html", "yacybot (/global; amd64 Linux 3.16.0-4-amd64; java 1.7.0_75; America/en) http://yacy.net/bot.html", "yacybot (-global; amd64 Linux 3.16.0-4-amd64; java 1.7.0_75; Europe/en) http://yacy.net/bot.html", "yacybot (/global; amd64 Linux 3.16.0-4-amd64; java 1.7.0_75; Europe/en) http://yacy.net/bot.html", "yacybot (/global; amd64 Linux 3.16.0-4-amd64; java 1.7.0_79; Europe/de) http://yacy.net/bot.html", "yacybot (/global; amd64 Linux 3.16.0-4-amd64; java 1.7.0_79; Europe/en) http://yacy.net/bot.html", "yacybot (/global; amd64 Linux 3.16.0-4-amd64; java 1.7.0_91; Europe/de) http://yacy.net/bot.html", "yacybot (/global; amd64 Linux 3.16.0-4-amd64; java 1.7.0_95; Europe/en) http://yacy.net/bot.html", "yacybot (/global; amd64 Linux 3.16.0-4-amd64; java 1.8.0_111; Europe/en) http://yacy.net/bot.html", "yacybot (/global; amd64 Linux 3.16-0.bpo.2-amd64; java 1.7.0_65; Europe/en) http://yacy.net/bot.html", "yacybot (-global; amd64 Linux 3.19.0-15-generic; java 1.8.0_45-internal; Europe/de) http://yacy.net/bot.html", "yacybot (-global; amd64 Linux 3.2.0-4-amd64; java 1.7.0_65; Europe/en) http://yacy.net/bot.html", "yacybot (-global; amd64 Linux 3.2.0-4-amd64; java 1.7.0_67; Europe/en) http://yacy.net/bot.html", "yacybot (-global; amd64 Linux 4.4.0-57-generic; java 9-internal; Europe/en) http://yacy.net/bot.html", "yacybot (-global; amd64 Windows 8.1 6.3; java 1.7.0_55; Europe/de) http://yacy.net/bot.html", "yacybot (-global; amd64 Windows 8 6.2; java 1.7.0_55; Europe/de) http://yacy.net/bot.html", "yacybot (-global; amd64 Linux 5.2.8-Jinsol; java 12.0.2; Europe/en) http://yacy.net/bot.html", "yacybot (-global; amd64 Linux 5.2.9-Jinsol; java 12.0.2; Europe/en) http://yacy.net/bot.html", "yacybot (-global; amd64 Linux 5.2.11-Jinsol; java 12.0.2; Europe/en) http://yacy.net/bot.html" ], "rejected": [] } }, { "id": "ai-search-bot", "categories": [ "ai" ], "pattern": { "accepted": [ "AISearchBot" ], "forbidden": [] }, "verification": [], "instances": { "accepted": [], "rejected": [] } }, { "id": "verisign-ips-agent", "categories": [ "unknown" ], "pattern": { "accepted": [ "ips-agent" ], "forbidden": [] }, "verification": [], "instances": { "accepted": [ "BlackBerry9000/4.6.0.167 Profile/MIDP-2.0 Configuration/CLDC-1.1 VendorID/102 ips-agent", "Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.7.12; ips-agent) Gecko/20050922 Fedora/1.0.7-1.1.fc4 Firefox/1.0.7", "Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.9.1.3; ips-agent) Gecko/20090824 Fedora/1.0.7-1.1.fc4 Firefox/3.5.3", "Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.9.2.24; ips-agent) Gecko/20111107 Ubuntu/10.04 (lucid) Firefox/3.6.24", "Mozilla/5.0 (X11; Ubuntu; Linux i686; rv:14.0; ips-agent) Gecko/20100101 Firefox/14.0.1" ], "rejected": [] } }, { "id": "tagoo-crawler", "categories": [ "unknown" ], "pattern": { "accepted": [ "tagoobot" ], "forbidden": [] }, "verification": [], "instances": { "accepted": [], "rejected": [] } }, { "id": "majestic-crawler", "categories": [ "search-engine" ], "pattern": { "accepted": [ "MJ12bot" ], "forbidden": [] }, "verification": [], "instances": { "accepted": [ "MJ12bot/v1.2.0 (http://majestic12.co.uk/bot.php?+)", "Mozilla/5.0 (compatible; MJ12bot/v1.2.1; http://www.majestic12.co.uk/bot.php?+)", "Mozilla/5.0 (compatible; MJ12bot/v1.2.3; http://www.majestic12.co.uk/bot.php?+)", "Mozilla/5.0 (compatible; MJ12bot/v1.2.4; http://www.majestic12.co.uk/bot.php?+)", "Mozilla/5.0 (compatible; MJ12bot/v1.2.5; http://www.majestic12.co.uk/bot.php?+)", "Mozilla/5.0 (compatible; MJ12bot/v1.3.0; http://www.majestic12.co.uk/bot.php?+)", "Mozilla/5.0 (compatible; MJ12bot/v1.3.1; http://www.majestic12.co.uk/bot.php?+)", "Mozilla/5.0 (compatible; MJ12bot/v1.3.2; http://www.majestic12.co.uk/bot.php?+)", "Mozilla/5.0 (compatible; MJ12bot/v1.3.3; http://www.majestic12.co.uk/bot.php?+)", "Mozilla/5.0 (compatible; MJ12bot/v1.4.0; http://www.majestic12.co.uk/bot.php?+)", "Mozilla/5.0 (compatible; MJ12bot/v1.4.1; http://www.majestic12.co.uk/bot.php?+)", "Mozilla/5.0 (compatible; MJ12bot/v1.4.2; http://www.majestic12.co.uk/bot.php?+)", "Mozilla/5.0 (compatible; MJ12bot/v1.4.3; http://www.majestic12.co.uk/bot.php?+)", "Mozilla/5.0 (compatible; MJ12bot/v1.4.4 (domain ownership verifier); http://www.majestic12.co.uk/bot.php?+)", "Mozilla/5.0 (compatible; MJ12bot/v1.4.4; http://www.majestic12.co.uk/bot.php?+)", "Mozilla/5.0 (compatible; MJ12bot/v1.4.5; http://www.majestic12.co.uk/bot.php?+)", "Mozilla/5.0 (compatible; MJ12bot/v1.4.6; http://mj12bot.com/)", "Mozilla/5.0 (compatible; MJ12bot/v1.4.7; http://mj12bot.com/)", "Mozilla/5.0 (compatible; MJ12bot/v1.4.7; http://www.majestic12.co.uk/bot.php?+)", "Mozilla/5.0 (compatible; MJ12bot/v1.4.8; http://mj12bot.com/)" ], "rejected": [] } }, { "id": "worio-crawler", "categories": [ "unknown" ], "pattern": { "accepted": [ "woriobot" ], "forbidden": [] }, "verification": [], "instances": { "accepted": [ "Mozilla/5.0 (compatible; woriobot +http://worio.com)", "Mozilla/5.0 (compatible; woriobot support [at] zite [dot] com +http://zite.com)" ], "rejected": [] } }, { "id": "yanga-crawler", "categories": [ "unknown" ], "pattern": { "accepted": [ "yanga" ], "forbidden": [] }, "verification": [], "instances": { "accepted": [ "Yanga WorldSearch Bot v1.1/beta (http://www.yanga.co.uk/)" ], "rejected": [] } }, { "id": "buzzstream-crawler", "categories": [ "unknown" ], "pattern": { "accepted": [ "buzzbot" ], "forbidden": [] }, "verification": [], "instances": { "accepted": [ "Buzzbot/1.0 (Buzzbot; http://www.buzzstream.com; buzzbot@buzzstream.com)" ], "rejected": [] } }, { "id": "metadatalabs-crawler", "categories": [ "unknown" ], "pattern": { "accepted": [ "mlbot" ], "forbidden": [] }, "verification": [], "instances": { "accepted": [ "MLBot (www.metadatalabs.com/mlbot)" ], "rejected": [] } }, { "id": "yandex-crawler", "categories": [ "search-engine" ], "pattern": { "accepted": [ "yandex\\.com\\/bots" ], "forbidden": [] }, "url": "https://yandex.ru/support/webmaster/robot-workings/check-yandex-robots.html#robot-in-logs", "verification": [ { "type": "dns", "masks": [ "@.yandex.ru", "@.yandex.com", "@.yandex.net" ] } ], "instances": { "accepted": [ "Mozilla/5.0 (compatible; YandexBot/3.0; +http://yandex.com/bots)", "Mozilla/5.0 (compatible; YandexBot/3.0; MirrorDetector; +http://yandex.com/bots)", "Mozilla/5.0 (iPhone; CPU iPhone OS 8_1 like Mac OS X) AppleWebKit/600.1.4 (KHTML, like Gecko) Version/8.0 Mobile/12B411 Safari/600.1.4 (compatible; YandexBot/3.0; +http://yandex.com/bots)", "Mozilla/5.0 (compatible; YandexImages/3.0; +http://yandex.com/bots)", "Mozilla/5.0 (compatible; YandexAccessibilityBot/3.0; +http://yandex.com/bots", "Mozilla/5.0 (compatible; YandexUserproxy; robot; +http://yandex.com/bots", "Mozilla/5.0 (iPhone; CPU iPhone OS 8_1 like Mac OS X) AppleWebKit/600.1.4 (KHTML, like Gecko) Version/8.0 Mobile/12B411 Safari/600.1.4 (compatible; YandexMobileBot/3.0; +http://yandex.com/bots)", "Mozilla/5.0 (compatible; YandexMetrika/2.0; +http://yandex.com/bots)", "Mozilla/5.0 (compatible; YandexMetrika/2.0; +http://yandex.com/bots yabs01)", "Mozilla/5.0 (compatible; YandexMetrika/3.0; +http://yandex.com/bots)", "Mozilla/5.0 (compatible; YandexMetrika/4.0; +http://yandex.com/bots)", "Mozilla/5.0 (compatible; YandexTurbo/1.0; +http://yandex.com/bots)", "Mozilla/5.0 (compatible; YandexVideoParser/1.0; +http://yandex.com/bots)", "Mozilla/5.0 (compatible; YandexVideo/3.0; +http://yandex.com/bots)", "Mozilla/5.0 (compatible; YandexImageResizer/2.0; +http://yandex.com/bots)", "Mozilla/5.0 (compatible; YandexAdNet/1.0; +http://yandex.com/bots)", "Mozilla/5.0 (compatible; YandexBlogs/0.99; robot; +http://yandex.com/bots)", "Mozilla/5.0 (compatible; YandexCalendar/1.0; +http://yandex.com/bots)", "Mozilla/5.0 (compatible; YandexDirect/3.0; +http://yandex.com/bots)", "Mozilla/5.0 (compatible; YandexDirectDyn/1.0; +http://yandex.com/bots", "Mozilla/5.0 (compatible; YandexFavicons/1.0; +http://yandex.com/bots)", "Mozilla/5.0 (compatible; YaDirectFetcher/1.0; Dyatel; +http://yandex.com/bots)", "Mozilla/5.0 (compatible; YandexForDomain/1.0; +http://yandex.com/bots)", "Mozilla/5.0 (compatible; YandexMarket/1.0; +http://yandex.com/bots)", "Mozilla/5.0 (compatible; YandexMarket/2.0; +http://yandex.com/bots)", "Mozilla/5.0 (compatible; YandexMedia/3.0; +http://yandex.com/bots)", "Mozilla/5.0 (compatible; YandexMobileScreenShotBot/1.0; +http://yandex.com/bots)", "Mozilla/5.0 (compatible; YandexNews/4.0; +http://yandex.com/bots)", "Mozilla/5.0 (compatible; YandexOntoDB/1.0; +http://yandex.com/bots)", "Mozilla/5.0 (compatible; YandexOntoDBAPI/1.0; +http://yandex.com/bots)", "Mozilla/5.0 (compatible; YandexPagechecker/1.0; +http://yandex.com/bots)", "Mozilla/5.0 (compatible; YandexPartner/3.0; +http://yandex.com/bots)", "Mozilla/5.0 (compatible; YandexRCA/1.0; +http://yandex.com/bots)", "Mozilla/5.0 (compatible; YandexSearchShop/1.0; +http://yandex.com/bots)", "Mozilla/5.0 (compatible; YandexSitelinks; Dyatel; +http://yandex.com/bots)", "Mozilla/5.0 (compatible; YandexSpravBot/1.0; +http://yandex.com/bots)", "Mozilla/5.0 (compatible; YandexTracker/1.0; +http://yandex.com/bots)", "Mozilla/5.0 (compatible; YandexVertis/3.0; +http://yandex.com/bots)", "Mozilla/5.0 (compatible; YandexVerticals/1.0; +http://yandex.com/bots)", "Mozilla/5.0 (compatible; YandexWebmaster/2.0; +http://yandex.com/bots)", "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2228.0 Safari/537.36 (compatible; YandexScreenshotBot/3.0; +http://yandex.com/bots)" ], "rejected": [] }, "addition_date": "2015/04/14", "aliases": [ "YandexBot" ] }, { "id": "pure-crawler", "categories": [ "unknown" ], "pattern": { "accepted": [ "purebot" ], "forbidden": [] }, "addition_date": "2010/01/19", "verification": [], "instances": { "accepted": [], "rejected": [] } }, { "id": "linguee-crawler", "categories": [ "search-engine" ], "pattern": { "accepted": [ "Linguee Bot" ], "forbidden": [] }, "addition_date": "2010/01/26", "url": "http://www.linguee.com/bot", "verification": [], "instances": { "accepted": [ "Linguee Bot (http://www.linguee.com/bot)", "Linguee Bot (http://www.linguee.com/bot; bot@linguee.com)" ], "rejected": [] } }, { "id": "cyberpatrol-crawler", "categories": [ "unknown" ], "pattern": { "accepted": [ "CyberPatrol" ], "forbidden": [] }, "addition_date": "2010/02/11", "url": "http://www.cyberpatrol.com/cyberpatrolcrawler.asp", "verification": [], "instances": { "accepted": [ "CyberPatrol SiteCat Webbot (http://www.cyberpatrol.com/cyberpatrolcrawler.asp)" ], "rejected": [] } }, { "id": "orange-ftgroup-crawler", "categories": [ "unknown" ], "pattern": { "accepted": [ "voilabot" ], "forbidden": [] }, "addition_date": "2010/05/18", "verification": [], "instances": { "accepted": [ "Mozilla/5.0 (Windows NT 5.1; U; Win64; fr; rv:1.8.1) VoilaBot BETA 1.2 (support.voilabot@orange-ftgroup.com)", "Mozilla/5.0 (Windows; U; Windows NT 5.1; fr; rv:1.8.1) VoilaBot BETA 1.2 (support.voilabot@orange-ftgroup.com)" ], "rejected": [] } }, { "id": "baidu-crawler", "categories": [ "unknown" ], "pattern": { "accepted": [ "Baiduspider" ], "forbidden": [] }, "addition_date": "2010/07/15", "url": "http://www.baidu.jp/spider/", "verification": [ { "type": "dns", "masks": [ "@.baidu.jp", "@.baidu.com" ] } ], "instances": { "accepted": [ "Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)", "Mozilla/5.0 (compatible; Baiduspider-render/2.0; +http://www.baidu.com/search/spider.html)" ], "rejected": [] }, "aliases": [ "BaiduSpider" ] }, { "id": "citeseerx-crawler", "categories": [ "unknown" ], "pattern": { "accepted": [ "citeseerxbot" ], "forbidden": [] }, "addition_date": "2010/07/17", "verification": [], "instances": { "accepted": [], "rejected": [] } }, { "id": "seoprofiler-crawler", "categories": [ "unknown" ], "pattern": { "accepted": [ "spbot" ], "forbidden": [] }, "addition_date": "2010/07/31", "url": "http://www.seoprofiler.com/bot", "verification": [], "instances": { "accepted": [ "Mozilla/5.0 (compatible; spbot/1.0; +http://www.seoprofiler.com/bot/ )", "Mozilla/5.0 (compatible; spbot/1.1; +http://www.seoprofiler.com/bot/ )", "Mozilla/5.0 (compatible; spbot/1.2; +http://www.seoprofiler.com/bot/ )", "Mozilla/5.0 (compatible; spbot/2.0.1; +http://www.seoprofiler.com/bot/ )", "Mozilla/5.0 (compatible; spbot/2.0.2; +http://www.seoprofiler.com/bot/ )", "Mozilla/5.0 (compatible; spbot/2.0.3; +http://www.seoprofiler.com/bot/ )", "Mozilla/5.0 (compatible; spbot/2.0.4; +http://www.seoprofiler.com/bot )", "Mozilla/5.0 (compatible; spbot/2.0; +http://www.seoprofiler.com/bot/ )", "Mozilla/5.0 (compatible; spbot/2.1; +http://www.seoprofiler.com/bot )", "Mozilla/5.0 (compatible; spbot/3.0; +http://www.seoprofiler.com/bot )", "Mozilla/5.0 (compatible; spbot/3.1; +http://www.seoprofiler.com/bot )", "Mozilla/5.0 (compatible; spbot/4.0.1; +http://www.seoprofiler.com/bot )", "Mozilla/5.0 (compatible; spbot/4.0.2; +http://www.seoprofiler.com/bot )", "Mozilla/5.0 (compatible; spbot/4.0.3; +http://www.seoprofiler.com/bot )", "Mozilla/5.0 (compatible; spbot/4.0.4; +http://www.seoprofiler.com/bot )", "Mozilla/5.0 (compatible; spbot/4.0.5; +http://www.seoprofiler.com/bot )", "Mozilla/5.0 (compatible; spbot/4.0.6; +http://www.seoprofiler.com/bot )", "Mozilla/5.0 (compatible; spbot/4.0.7; +http://OpenLinkProfiler.org/bot )", "Mozilla/5.0 (compatible; spbot/4.0.7; +https://www.seoprofiler.com/bot )", "Mozilla/5.0 (compatible; spbot/4.0.8; +http://OpenLinkProfiler.org/bot )", "Mozilla/5.0 (compatible; spbot/4.0.9; +http://OpenLinkProfiler.org/bot )", "Mozilla/5.0 (compatible; spbot/4.0; +http://www.seoprofiler.com/bot )", "Mozilla/5.0 (compatible; spbot/4.0a; +http://www.seoprofiler.com/bot )", "Mozilla/5.0 (compatible; spbot/4.0b; +http://www.seoprofiler.com/bot )", "Mozilla/5.0 (compatible; spbot/4.1.0; +http://OpenLinkProfiler.org/bot )", "Mozilla/5.0 (compatible; spbot/4.2.0; +http://OpenLinkProfiler.org/bot )", "Mozilla/5.0 (compatible; spbot/4.3.0; +http://OpenLinkProfiler.org/bot )", "Mozilla/5.0 (compatible; spbot/4.4.0; +http://OpenLinkProfiler.org/bot )", "Mozilla/5.0 (compatible; spbot/4.4.1; +http://OpenLinkProfiler.org/bot )", "Mozilla/5.0 (compatible; spbot/4.4.2; +http://OpenLinkProfiler.org/bot )", "Mozilla/5.0 (compatible; spbot/5.0.1; +http://OpenLinkProfiler.org/bot )", "Mozilla/5.0 (compatible; spbot/5.0.2; +http://OpenLinkProfiler.org/bot )", "Mozilla/5.0 (compatible; spbot/5.0.3; +http://OpenLinkProfiler.org/bot )", "Mozilla/5.0 (compatible; spbot/5.0; +http://OpenLinkProfiler.org/bot )" ], "rejected": [] } }, { "id": "twenga-crawler", "categories": [ "unknown" ], "pattern": { "accepted": [ "twengabot" ], "forbidden": [] }, "addition_date": "2010/08/03", "url": "http://www.twenga.com/bot.html", "verification": [], "instances": { "accepted": [], "rejected": [] } }, { "id": "postrank-crawler", "categories": [ "unknown" ], "pattern": { "accepted": [ "postrank" ], "forbidden": [] }, "addition_date": "2010/08/03", "url": "http://www.postrank.com", "verification": [], "instances": { "accepted": [ "PostRank/2.0 (postrank.com)", "PostRank/2.0 (postrank.com; 1 subscribers)" ], "rejected": [] } }, { "id": "turnitin-crawler", "categories": [ "academic", "ai" ], "pattern": { "accepted": [ "Turnitin" ], "forbidden": [] }, "addition_date": "2010/09/26", "url": "http://www.turnitin.com", "verification": [], "instances": { "accepted": [ "TurnitinBot (https://turnitin.com/robot/crawlerinfo.html)", "Turnitin (https://bit.ly/2UvnfoQ)" ], "rejected": [] } }, { "id": "scribd-crawler", "categories": [ "unknown" ], "pattern": { "accepted": [ "scribdbot" ], "forbidden": [] }, "addition_date": "2010/09/28", "url": "http://www.scribd.com", "verification": [], "instances": { "accepted": [], "rejected": [] } }, { "id": "page-to-rss", "categories": [ "unknown" ], "pattern": { "accepted": [ "page2rss" ], "forbidden": [] }, "addition_date": "2010/10/07", "url": "http://www.page2rss.com", "verification": [], "instances": { "accepted": [ "Mozilla/5.0 (compatible; Page2RSS/0.7; +http://page2rss.com/)" ], "rejected": [] } }, { "id": "sitebot-crawler", "categories": [ "unknown" ], "pattern": { "accepted": [ "sitebot" ], "forbidden": [] }, "addition_date": "2010/12/15", "url": "http://www.sitebot.org", "verification": [], "instances": { "accepted": [ "Mozilla/5.0 (compatible; Whoiswebsitebot/0.1; +http://www.whoiswebsite.net)" ], "rejected": [] } }, { "id": "linkdex-crawler", "categories": [ "unknown" ], "pattern": { "accepted": [ "linkdex" ], "forbidden": [] }, "addition_date": "2011/01/06", "url": "http://www.linkdex.com", "verification": [], "instances": { "accepted": [ "Mozilla/5.0 (compatible; linkdexbot/2.0; +http://www.linkdex.com/about/bots/)", "Mozilla/5.0 (compatible; linkdexbot/2.0; +http://www.linkdex.com/bots/)", "Mozilla/5.0 (compatible; linkdexbot/2.1; +http://www.linkdex.com/about/bots/)", "Mozilla/5.0 (compatible; linkdexbot/2.1; +http://www.linkdex.com/bots/)", "Mozilla/5.0 (compatible; linkdexbot/2.2; +http://www.linkdex.com/bots/)", "linkdex.com/v2.0", "linkdexbot/Nutch-1.0-dev (http://www.linkdex.com/; crawl at linkdex dot com)" ], "rejected": [] } }, { "id": "bing-ads", "categories": [ "microsoft", "advertising" ], "pattern": { "accepted": [ "Adidxbot" ], "forbidden": [] }, "url": "https://www.bing.com/webmasters/help/which-crawlers-does-bing-use-8c184ec0", "verification": [ { "type": "cidr", "sources": [ { "type": "http-json", "url": "https://www.bing.com/toolbox/bingbot.json", "selector": "$.prefixes[*][\\\"ipv6Prefix\\\",\\\"ipv4Prefix\\\"]" } ] }, { "type": "dns", "masks": [ "@.search.msn.com" ] } ], "instances": { "accepted": [], "rejected": [] } }, { "id": "dotnetdotcom-crawler", "categories": [ "unknown" ], "pattern": { "accepted": [ "ezooms" ], "forbidden": [] }, "addition_date": "2011/04/27", "url": "http://www.phpbb.com/community/viewtopic.php?f=64&t=935605&start=450#p12948289", "verification": [], "instances": { "accepted": [ "Mozilla/5.0 (compatible; Ezooms/1.0; ezooms.bot@gmail.com)" ], "rejected": [] } }, { "id": "moz-crawler", "categories": [ "unknown" ], "pattern": { "accepted": [ "dotbot" ], "forbidden": [] }, "addition_date": "2011/04/27", "verification": [], "instances": { "accepted": [ "Mozilla/5.0 (compatible; DotBot/1.1; http://www.opensiteexplorer.org/dotbot, help@moz.com)", "dotbot" ], "rejected": [] } }, { "id": "mailru-crawler", "categories": [ "unknown" ], "pattern": { "accepted": [ "Mail\\.RU_Bot" ], "forbidden": [] }, "addition_date": "2011/04/27", "verification": [], "instances": { "accepted": [ "Mozilla/5.0 (compatible; Linux x86_64; Mail.RU_Bot/2.0; +http://go.mail.ru/help/robots)", "Mozilla/5.0 (compatible; Linux x86_64; Mail.RU_Bot/2.0; +http://go.mail.ru/", "Mozilla/5.0 (compatible; Mail.RU_Bot/2.0; +http://go.mail.ru/", "Mozilla/5.0 (compatible; Linux x86_64; Mail.RU_Bot/Robots/2.0; +http://go.mail.ru/help/robots)" ], "rejected": [] }, "aliases": [ "MailRuBot" ] }, { "id": "discoveryengine-crawler", "categories": [ "unknown" ], "pattern": { "accepted": [ "discobot" ], "forbidden": [] }, "addition_date": "2011/05/03", "url": "http://discoveryengine.com/discobot.html", "verification": [], "instances": { "accepted": [ "Mozilla/5.0 (compatible; discobot/1.0; +http://discoveryengine.com/discobot.html)", "Mozilla/5.0 (compatible; discobot/2.0; +http://discoveryengine.com/discobot.html)", "mozilla/5.0 (compatible; discobot/1.1; +http://discoveryengine.com/discobot.html)" ], "rejected": [] } }, { "id": "internetarchive-crawler-oss", "categories": [ "archive", "tool" ], "pattern": { "accepted": [ "heritrix" ], "forbidden": [] }, "addition_date": "2011/06/21", "url": "https://github.com/internetarchive/heritrix3/wiki", "verification": [], "instances": { "accepted": [ "Mozilla/5.0 (compatible; heritrix/1.12.1 +http://www.webarchiv.cz)", "Mozilla/5.0 (compatible; heritrix/1.12.1b +http://netarkivet.dk/website/info.html)", "Mozilla/5.0 (compatible; heritrix/1.14.2 +http://rjpower.org)", "Mozilla/5.0 (compatible; heritrix/1.14.2 +http://www.webarchiv.cz)", "Mozilla/5.0 (compatible; heritrix/1.14.3 +http://archive.org)", "Mozilla/5.0 (compatible; heritrix/1.14.3 +http://www.accelobot.com)", "Mozilla/5.0 (compatible; heritrix/1.14.3 +http://www.webarchiv.cz)", "Mozilla/5.0 (compatible; heritrix/1.14.3.r6601 +http://www.buddybuzz.net/yptrino)", "Mozilla/5.0 (compatible; heritrix/1.14.4 +http://parsijoo.ir)", "Mozilla/5.0 (compatible; heritrix/1.14.4 +http://www.exif-search.com)", "Mozilla/5.0 (compatible; heritrix/2.0.2 +http://aihit.com)", "Mozilla/5.0 (compatible; heritrix/2.0.2 +http://seekda.com)", "Mozilla/5.0 (compatible; heritrix/3.0.0-SNAPSHOT-20091120.021634 +http://crawler.archive.org)", "Mozilla/5.0 (compatible; heritrix/3.1.0-RC1 +http://boston.lti.cs.cmu.edu/crawler_12/)", "Mozilla/5.0 (compatible; heritrix/3.1.1 +http://places.tomtom.com/crawlerinfo)", "Mozilla/5.0 (compatible; heritrix/3.1.1 +http://www.mixdata.com)", "Mozilla/5.0 (compatible; heritrix/3.1.1; UniLeipzigASV +http://corpora.informatik.uni-leipzig.de/crawler_faq.html)", "Mozilla/5.0 (compatible; heritrix/3.2.0 +http://www.crim.ca)", "Mozilla/5.0 (compatible; heritrix/3.2.0 +http://www.exif-search.com)", "Mozilla/5.0 (compatible; heritrix/3.2.0 +http://www.mixdata.com)", "Mozilla/5.0 (compatible; heritrix/3.3.0-SNAPSHOT-20160309-0050; UniLeipzigASV +http://corpora.informatik.uni-leipzig.de/crawler_faq.html)", "Mozilla/5.0 (compatible; sukibot_heritrix/3.1.1 +http://suki.ling.helsinki.fi/eng/webmasters.html)" ], "rejected": [] } }, { "id": "findthatfile-crawler", "categories": [ "unknown" ], "pattern": { "accepted": [ "findthatfile" ], "forbidden": [] }, "addition_date": "2011/06/21", "url": "http://www.findthatfile.com/", "verification": [], "instances": { "accepted": [], "rejected": [] } }, { "id": "europarchive-crawler", "categories": [ "unknown" ], "pattern": { "accepted": [ "europarchive\\.org" ], "forbidden": [] }, "addition_date": "2011/06/21", "url": "", "verification": [], "instances": { "accepted": [ "Mozilla/5.0 (compatible; MSIE 7.0 +http://www.europarchive.org)" ], "rejected": [] } }, { "id": "nerdbynature-crawler", "categories": [ "unknown" ], "pattern": { "accepted": [ "NerdByNature\\.Bot" ], "forbidden": [] }, "addition_date": "2011/07/12", "url": "http://www.nerdbynature.net/bot", "verification": [], "instances": { "accepted": [ "Mozilla/5.0 (compatible; NerdByNature.Bot; http://www.nerdbynature.net/bot)" ], "rejected": [] } }, { "id": "sistrix-crawler", "categories": [ "optimizer" ], "pattern": { "accepted": [ "(sistrix|SISTRIX) [cC]rawler" ], "forbidden": [] }, "addition_date": "2011/08/02", "url": "https://www.sistrix.com/tutorials/crawling-errors-in-the-optimizer/", "verification": [], "instances": { "accepted": [ "Mozilla/5.0 (compatible; SISTRIX Crawler; http://crawler.sistrix.net/)" ], "rejected": [] } }, { "id": "ahrefs-crawler", "categories": [ "search-engine" ], "pattern": { "accepted": [ "AhrefsBot\\/" ], "forbidden": [] }, "addition_date": "2011/08/28", "verification": [ { "type": "cidr", "sources": [ { "type": "http-json", "url": "https://api.ahrefs.com/v3/public/crawler-ip-ranges", "selector": "$.prefixes[*].ipv4Prefix" } ] } ], "instances": { "accepted": [ "Mozilla/5.0 (compatible; AhrefsBot/6.1; +http://ahrefs.com/robot/)", "Mozilla/5.0 (compatible; AhrefsBot/5.2; News; +http://ahrefs.com/robot/)", "Mozilla/5.0 (compatible; AhrefsBot/5.2; +http://ahrefs.com/robot/)", "Mozilla/5.0 (compatible; AhrefsBot/6.1; News; +http://ahrefs.com/robot/)", "Mozilla/5.0 (compatible; AhrefsBot/7.0; +http://ahrefs.com/robot/)" ], "rejected": [] }, "aliases": [ "AhrefsBot" ] }, { "id": "fuelbot", "categories": [ "unknown" ], "pattern": { "accepted": [ "fuelbot" ], "forbidden": [] }, "addition_date": "2018/06/28", "verification": [], "instances": { "accepted": [ "fuelbot" ], "rejected": [] } }, { "id": "leadcrunch-crawler", "categories": [ "ai" ], "pattern": { "accepted": [ "CrunchBot" ], "forbidden": [] }, "addition_date": "2018/06/28", "verification": [], "instances": { "accepted": [ "CrunchBot/1.0 (+http://www.leadcrunch.com/crunchbot)" ], "rejected": [] } }, { "id": "indeed-crawler", "categories": [ "unknown" ], "pattern": { "accepted": [ "IndeedBot" ], "forbidden": [] }, "addition_date": "2018/06/28", "verification": [], "instances": { "accepted": [ "Mozilla/5.0 (Windows NT 6.1; rv:38.0) Gecko/20100101 Firefox/38.0 (IndeedBot 1.1)" ], "rejected": [] } }, { "id": "mappydata-crawler", "categories": [ "unknown" ], "pattern": { "accepted": [ "mappydata" ], "forbidden": [] }, "addition_date": "2018/06/28", "verification": [], "instances": { "accepted": [ "Mozilla/5.0 (compatible; Mappy/1.0; +http://mappydata.net/bot/)" ], "rejected": [] } }, { "id": "woorank-crawler", "categories": [ "unknown" ], "pattern": { "accepted": [ "woobot" ], "forbidden": [] }, "addition_date": "2018/06/28", "verification": [], "instances": { "accepted": [ "woobot" ], "rejected": [] } }, { "id": "zoominfo-crawler", "categories": [ "unknown" ], "pattern": { "accepted": [ "ZoominfoBot" ], "forbidden": [] }, "addition_date": "2018/06/28", "verification": [], "instances": { "accepted": [ "ZoominfoBot (zoominfobot at zoominfo dot com)" ], "rejected": [] } }, { "id": "privacyaware-crawler", "categories": [ "unknown" ], "pattern": { "accepted": [ "PrivacyAwareBot" ], "forbidden": [] }, "addition_date": "2018/06/28", "verification": [], "instances": { "accepted": [ "Mozilla/5.0 (compatible; PrivacyAwareBot/1.1; +http://www.privacyaware.org)" ], "rejected": [] } }, { "id": "multiviewbot", "categories": [ "unknown" ], "pattern": { "accepted": [ "Multiviewbot" ], "forbidden": [] }, "addition_date": "2018/06/28", "verification": [], "instances": { "accepted": [ "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Multiviewbot" ], "rejected": [] } }, { "id": "swimgbot", "categories": [ "unknown" ], "pattern": { "accepted": [ "SWIMGBot" ], "forbidden": [] }, "addition_date": "2018/06/28", "verification": [], "instances": { "accepted": [ "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/45.0.2454.101 Safari/537.36 SWIMGBot" ], "rejected": [] } }, { "id": "grob-crawler", "categories": [ "unknown" ], "pattern": { "accepted": [ "Grobbot" ], "forbidden": [] }, "addition_date": "2018/06/28", "verification": [], "instances": { "accepted": [ "Mozilla/5.0 (compatible; Grobbot/2.2; +https://grob.it)" ], "rejected": [] } }, { "id": "eright-crawler", "categories": [ "unknown" ], "pattern": { "accepted": [ "eright" ], "forbidden": [] }, "addition_date": "2018/06/28", "verification": [], "instances": { "accepted": [ "Mozilla/5.0 (compatible; eright/1.0; +bot@eright.com)" ], "rejected": [] } }, { "id": "apercite-crawler", "categories": [ "unknown" ], "pattern": { "accepted": [ "Apercite" ], "forbidden": [] }, "addition_date": "2018/06/28", "verification": [], "instances": { "accepted": [ "Mozilla/5.0 (compatible; Apercite; +http://www.apercite.fr/robot/index.html)" ], "rejected": [] } }, { "id": "semanticaudience-crawler", "categories": [ "unknown" ], "pattern": { "accepted": [ "semanticbot" ], "forbidden": [] }, "addition_date": "2018/06/28", "verification": [], "instances": { "accepted": [ "semanticbot", "semanticbot (info@semanticaudience.com)" ], "rejected": [] } }, { "id": "aboundex-crawler", "categories": [ "unknown" ], "pattern": { "accepted": [ "Aboundex" ], "forbidden": [] }, "addition_date": "2011/09/28", "url": "http://www.aboundex.com/crawler/", "verification": [], "instances": { "accepted": [ "Aboundex/0.2 (http://www.aboundex.com/crawler/)", "Aboundex/0.3 (http://www.aboundex.com/crawler/)" ], "rejected": [] } }, { "id": "domaincrawler-crawler", "categories": [ "unknown" ], "pattern": { "accepted": [ "domaincrawler" ], "forbidden": [] }, "addition_date": "2011/10/21", "verification": [], "instances": { "accepted": [ "CipaCrawler/3.0 (info@domaincrawler.com; http://www.domaincrawler.com/www.example.com)" ], "rejected": [] } }, { "id": "warebay-crawler", "categories": [ "unknown" ], "pattern": { "accepted": [ "wbsearchbot" ], "forbidden": [] }, "addition_date": "2011/12/21", "url": "http://www.warebay.com/bot.html", "verification": [], "instances": { "accepted": [], "rejected": [] } }, { "id": "summify-crawler", "categories": [ "unknown" ], "pattern": { "accepted": [ "summify" ], "forbidden": [] }, "addition_date": "2012/01/04", "url": "http://summify.com", "verification": [], "instances": { "accepted": [ "Summify (Summify/1.0.1; +http://summify.com)" ], "rejected": [] } }, { "id": "commoncrawl-crawler", "categories": [ "academic", "ai", "archive" ], "pattern": { "accepted": [ "CCBot" ], "forbidden": [] }, "addition_date": "2012/02/05", "url": "http://www.commoncrawl.org/bot.html", "verification": [], "instances": { "accepted": [ "CCBot/2.0 (http://commoncrawl.org/faq/)", "CCBot/2.0 (https://commoncrawl.org/faq/)" ], "rejected": [] }, "aliases": [ "CommonCrawlBot" ] }, { "id": "edister-crawler", "categories": [ "unknown" ], "pattern": { "accepted": [ "edisterbot" ], "forbidden": [] }, "addition_date": "2012/02/25", "verification": [], "instances": { "accepted": [], "rejected": [] } }, { "id": "seznam-crawler", "categories": [ "unknown" ], "pattern": { "accepted": [ "SeznamBot" ], "forbidden": [] }, "addition_date": "2012/03/14", "verification": [], "instances": { "accepted": [ "Mozilla/5.0 (compatible; SeznamBot/3.2-test1-1; +http://napoveda.seznam.cz/en/seznambot-intro/)", "Mozilla/5.0 (compatible; SeznamBot/3.2-test1; +http://napoveda.seznam.cz/en/seznambot-intro/)", "Mozilla/5.0 (compatible; SeznamBot/3.2-test2; +http://napoveda.seznam.cz/en/seznambot-intro/)", "Mozilla/5.0 (compatible; SeznamBot/3.2-test4; +http://napoveda.seznam.cz/en/seznambot-intro/)", "Mozilla/5.0 (compatible; SeznamBot/3.2; +http://napoveda.seznam.cz/en/seznambot-intro/)", "Mozilla/5.0 (compatible; SeznamBot/4.0; +http://napoveda.seznam.cz/seznambot-intro/)" ], "rejected": [] } }, { "id": "ec2linkfinder", "categories": [ "unknown" ], "pattern": { "accepted": [ "ec2linkfinder" ], "forbidden": [] }, "addition_date": "2012/03/22", "verification": [], "instances": { "accepted": [ "ec2linkfinder" ], "rejected": [] } }, { "id": "gslfbot", "categories": [ "unknown" ], "pattern": { "accepted": [ "gslfbot" ], "forbidden": [] }, "addition_date": "2012/04/03", "verification": [], "instances": { "accepted": [], "rejected": [] } }, { "id": "aihit-crawler", "categories": [ "ai" ], "pattern": { "accepted": [ "aiHitBot" ], "forbidden": [] }, "addition_date": "2012/04/16", "verification": [], "instances": { "accepted": [ "Mozilla/5.0 (compatible; aiHitBot/2.9; +https://www.aihitdata.com/about)" ], "rejected": [] } }, { "id": "intelium-crawler", "categories": [ "unknown" ], "pattern": { "accepted": [ "intelium_bot" ], "forbidden": [] }, "addition_date": "2012/05/07", "verification": [], "instances": { "accepted": [], "rejected": [] } }, { "id": "facebook-share-crawler", "categories": [ "ai", "meta", "preview", "social" ], "pattern": { "accepted": [ "facebookexternalhit" ], "forbidden": [ "Twitterbot", "Facebot" ] }, "addition_date": "2012/05/07", "verification": [ { "type": "cidr", "sources": [ { "type": "http-csv", "url": "http://www.facebook.com/peering/geofeed" } ] } ], "instances": { "accepted": [ "facebookexternalhit/1.0 (+http://www.facebook.com/externalhit_uatext.php)", "facebookexternalhit/1.1", "facebookexternalhit/1.1 (+http://www.facebook.com/externalhit_uatext.php)" ], "rejected": [ "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_1) AppleWebKit/601.2.4 (KHTML, like Gecko) Version/9.0.1 Safari/601.2.4 facebookexternalhit/1.1 Facebot Twitterbot/1.0" ] }, "url": "https://developers.facebook.com/docs/sharing/webmasters/crawler/", "aliases": [ "Facebookexternalhit" ] }, { "id": "naver-crawler", "categories": [ "unknown" ], "pattern": { "accepted": [ "Yeti" ], "forbidden": [] }, "addition_date": "2012/05/07", "url": "http://naver.me/bot", "verification": [], "instances": { "accepted": [ "Mozilla/5.0 (compatible; Yeti/1.1; +http://naver.me/bot)" ], "rejected": [] } }, { "id": "retrevo-page-analyzer", "categories": [ "unknown" ], "pattern": { "accepted": [ "RetrevoPageAnalyzer" ], "forbidden": [] }, "addition_date": "2012/05/07", "verification": [], "instances": { "accepted": [ "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; RetrevoPageAnalyzer; +http://www.retrevo.com/content/about-us)" ], "rejected": [] } }, { "id": "lb-spider", "categories": [ "unknown" ], "pattern": { "accepted": [ "lb-spider" ], "forbidden": [] }, "addition_date": "2012/05/07", "verification": [], "instances": { "accepted": [], "rejected": [] } }, { "id": "sogou-crawler", "categories": [ "unknown" ], "pattern": { "accepted": [ "Sogou" ], "forbidden": [] }, "addition_date": "2012/05/13", "url": "http://www.sogou.com/docs/help/webmasters.htm#07", "verification": [], "instances": { "accepted": [ "Sogou News Spider/4.0(+http://www.sogou.com/docs/help/webmasters.htm#07)", "Sogou Pic Spider/3.0(+http://www.sogou.com/docs/help/webmasters.htm#07)", "Sogou web spider/4.0(+http://www.sogou.com/docs/help/webmasters.htm#07)" ], "rejected": [] } }, { "id": "lssbot", "categories": [ "unknown" ], "pattern": { "accepted": [ "lssbot" ], "forbidden": [] }, "addition_date": "2012/05/15", "url": "https://www.lssbot.com/", "verification": [], "instances": { "accepted": [], "rejected": [] } }, { "id": "careerx-crawler", "categories": [ "unknown" ], "pattern": { "accepted": [ "careerbot" ], "forbidden": [] }, "addition_date": "2012/05/23", "url": "http://www.career-x.de/bot.html", "verification": [], "instances": { "accepted": [], "rejected": [] } }, { "id": "wotbox-crawler", "categories": [ "unknown" ], "pattern": { "accepted": [ "wotbox" ], "forbidden": [] }, "addition_date": "2012/06/12", "url": "http://www.wotbox.com", "verification": [], "instances": { "accepted": [ "Wotbox/2.0 (bot@wotbox.com; http://www.wotbox.com)", "Wotbox/2.01 (+http://www.wotbox.com/bot/)" ], "rejected": [] } }, { "id": "wocodi-crawler", "categories": [ "unknown" ], "pattern": { "accepted": [ "wocbot" ], "forbidden": [] }, "addition_date": "2012/07/25", "url": "http://www.wocodi.com/crawler", "verification": [], "instances": { "accepted": [], "rejected": [] } }, { "id": "goo-crawler", "categories": [ "unknown" ], "pattern": { "accepted": [ "ichiro" ], "forbidden": [] }, "addition_date": "2012/08/28", "url": "http://help.goo.ne.jp/help/article/1142", "verification": [], "instances": { "accepted": [ "DoCoMo/2.0 P900i(c100;TB;W24H11) (compatible; ichiro/mobile goo; +http://help.goo.ne.jp/help/article/1142/)", "DoCoMo/2.0 P900i(c100;TB;W24H11) (compatible; ichiro/mobile goo; +http://search.goo.ne.jp/option/use/sub4/sub4-1/)", "DoCoMo/2.0 P900i(c100;TB;W24H11) (compatible; ichiro/mobile goo;+http://search.goo.ne.jp/option/use/sub4/sub4-1/)", "DoCoMo/2.0 P900i(c100;TB;W24H11)(compatible; ichiro/mobile goo;+http://help.goo.ne.jp/door/crawler.html)", "DoCoMo/2.0 P901i(c100;TB;W24H11) (compatible; ichiro/mobile goo; +http://help.goo.ne.jp/door/crawler.html)", "KDDI-CA31 UP.Browser/6.2.0.7.3.129 (GUI) MMP/2.0 (compatible; ichiro/mobile goo; +http://help.goo.ne.jp/help/article/1142/)", "KDDI-CA31 UP.Browser/6.2.0.7.3.129 (GUI) MMP/2.0 (compatible; ichiro/mobile goo; +http://search.goo.ne.jp/option/use/sub4/sub4-1/)", "KDDI-CA31 UP.Browser/6.2.0.7.3.129 (GUI) MMP/2.0 (compatible; ichiro/mobile goo;+http://search.goo.ne.jp/option/use/sub4/sub4-1/)", "ichiro/2.0 (http://help.goo.ne.jp/door/crawler.html)", "ichiro/2.0 (ichiro@nttr.co.jp)", "ichiro/3.0 (http://help.goo.ne.jp/door/crawler.html)", "ichiro/3.0 (http://help.goo.ne.jp/help/article/1142)", "ichiro/3.0 (http://search.goo.ne.jp/option/use/sub4/sub4-1/)", "ichiro/4.0 (http://help.goo.ne.jp/door/crawler.html)", "ichiro/5.0 (http://help.goo.ne.jp/door/crawler.html)" ], "rejected": [] } }, { "id": "duckduckgo-crawler", "categories": [ "search-engine" ], "pattern": { "accepted": [ "DuckDuckBot" ], "forbidden": [] }, "addition_date": "2012/09/19", "url": "http://duckduckgo.com/duckduckbot.html", "verification": [], "instances": { "accepted": [ "DuckDuckBot/1.0; (+http://duckduckgo.com/duckduckbot.html)", "DuckDuckBot/1.1; (+http://duckduckgo.com/duckduckbot.html)", "Mozilla/5.0 (compatible; DuckDuckBot-Https/1.1; https://duckduckgo.com/duckduckbot)" ], "rejected": [] }, "aliases": [ "DuckDuckBot", "DuckDuckGoBot" ] }, { "id": "lssbot-rocket", "categories": [ "unknown" ], "pattern": { "accepted": [ "lssrocketcrawler" ], "forbidden": [] }, "addition_date": "2012/09/24", "verification": [], "instances": { "accepted": [], "rejected": [] } }, { "id": "arocom-crawler", "categories": [ "unknown" ], "pattern": { "accepted": [ "drupact" ], "forbidden": [] }, "addition_date": "2012/09/27", "url": "http://www.arocom.de/drupact", "verification": [], "instances": { "accepted": [ "drupact/0.7; http://www.arocom.de/drupact" ], "rejected": [] } }, { "id": "webcompany-crawler", "categories": [ "unknown" ], "pattern": { "accepted": [ "webcompanycrawler" ], "forbidden": [] }, "addition_date": "2012/10/03", "verification": [], "instances": { "accepted": [], "rejected": [] } }, { "id": "acoon-crawler", "categories": [ "unknown" ], "pattern": { "accepted": [ "acoonbot" ], "forbidden": [] }, "addition_date": "2012/10/07", "url": "http://www.acoon.de/robot.asp", "verification": [], "instances": { "accepted": [], "rejected": [] } }, { "id": "openindex-crawler", "categories": [ "unknown" ], "pattern": { "accepted": [ "openindexspider" ], "forbidden": [] }, "addition_date": "2012/10/26", "url": "http://www.openindex.io/en/webmasters/spider.html", "verification": [], "instances": { "accepted": [], "rejected": [] } }, { "id": "gnam-gnam-spider", "categories": [ "unknown" ], "pattern": { "accepted": [ "gnam gnam spider" ], "forbidden": [] }, "addition_date": "2012/10/31", "verification": [], "instances": { "accepted": [], "rejected": [] } }, { "id": "webarchive-crawler", "categories": [ "archive" ], "pattern": { "accepted": [ "web-archive-net\\.com\\.bot" ], "forbidden": [] }, "verification": [], "instances": { "accepted": [], "rejected": [] } }, { "id": "backlinktest-crawler", "categories": [ "unknown" ], "pattern": { "accepted": [ "backlinkcrawler" ], "forbidden": [] }, "addition_date": "2013/01/04", "url": "http://www.backlinktest.com/crawler.html", "verification": [], "instances": { "accepted": [], "rejected": [] } }, { "id": "coccoc-crawler", "categories": [ "unknown" ], "pattern": { "accepted": [ "coccoc" ], "forbidden": [] }, "addition_date": "2013/01/04", "url": "http://help.coccoc.vn/", "verification": [], "instances": { "accepted": [ "Mozilla/5.0 (compatible; coccoc/1.0; +http://help.coccoc.com/)", "Mozilla/5.0 (compatible; coccoc/1.0; +http://help.coccoc.com/searchengine)", "Mozilla/5.0 (compatible; coccocbot-image/1.0; +http://help.coccoc.com/searchengine)", "Mozilla/5.0 (compatible; coccocbot-web/1.0; +http://help.coccoc.com/searchengine)", "Mozilla/5.0 (compatible; image.coccoc/1.0; +http://help.coccoc.com/)", "Mozilla/5.0 (compatible; imagecoccoc/1.0; +http://help.coccoc.com/)", "Mozilla/5.0 (compatible; imagecoccoc/1.0; +http://help.coccoc.com/searchengine)", "coccoc", "coccoc/1.0 ()", "coccoc/1.0 (http://help.coccoc.com/)", "coccoc/1.0 (http://help.coccoc.vn/)" ], "rejected": [] } }, { "id": "integromedb-crawler", "categories": [ "unknown" ], "pattern": { "accepted": [ "integromedb" ], "forbidden": [] }, "addition_date": "2013/01/10", "url": "http://www.integromedb.org/Crawler", "verification": [], "instances": { "accepted": [ "www.integromedb.org/Crawler" ], "rejected": [] } }, { "id": "content-crawler-spider", "categories": [ "unknown" ], "pattern": { "accepted": [ "content crawler spider" ], "forbidden": [] }, "addition_date": "2013/01/11", "verification": [], "instances": { "accepted": [], "rejected": [] } }, { "id": "toplist-crawler", "categories": [ "unknown" ], "pattern": { "accepted": [ "toplistbot" ], "forbidden": [] }, "addition_date": "2013/02/05", "verification": [], "instances": { "accepted": [], "rejected": [] } }, { "id": "it2media-crawler", "categories": [ "unknown" ], "pattern": { "accepted": [ "it2media-domain-crawler" ], "forbidden": [] }, "addition_date": "2013/03/12", "verification": [], "instances": { "accepted": [ "it2media-domain-crawler/1.0 on crawler-prod.it2media.de", "it2media-domain-crawler/2.0" ], "rejected": [] } }, { "id": "ip-web-crawler", "categories": [ "unknown" ], "pattern": { "accepted": [ "ip-web-crawler\\.com" ], "forbidden": [] }, "addition_date": "2013/03/22", "verification": [], "instances": { "accepted": [], "rejected": [] } }, { "id": "siteexplorer-crawler", "categories": [ "unknown" ], "pattern": { "accepted": [ "siteexplorer\\.info" ], "forbidden": [] }, "addition_date": "2013/05/01", "verification": [], "instances": { "accepted": [ "Mozilla/5.0 (compatible; SiteExplorer/1.0b; +http://siteexplorer.info/)", "Mozilla/5.0 (compatible; SiteExplorer/1.1b; +http://siteexplorer.info/Backlink-Checker-Spider/)" ], "rejected": [] } }, { "id": "elisabot", "categories": [ "unknown" ], "pattern": { "accepted": [ "elisabot" ], "forbidden": [] }, "addition_date": "2013/06/27", "verification": [], "instances": { "accepted": [], "rejected": [] } }, { "id": "proximic-crawler", "categories": [ "unknown" ], "pattern": { "accepted": [ "proximic" ], "forbidden": [] }, "addition_date": "2013/09/12", "url": "http://www.proximic.com/info/spider.php", "verification": [], "instances": { "accepted": [ "Mozilla/5.0 (compatible; proximic; +http://www.proximic.com)", "Mozilla/5.0 (compatible; proximic; +http://www.proximic.com/info/spider.php)" ], "rejected": [] } }, { "id": "changedetection-crawler", "categories": [ "monitor" ], "pattern": { "accepted": [ "changedetection" ], "forbidden": [] }, "addition_date": "2013/09/13", "url": "https://visualping.io/", "verification": [], "instances": { "accepted": [ "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; http://www.changedetection.com/bot.html )" ], "rejected": [] } }, { "id": "ara-crawler", "categories": [ "unknown" ], "pattern": { "accepted": [ "arabot" ], "forbidden": [] }, "addition_date": "2013/10/09", "verification": [], "instances": { "accepted": [], "rejected": [] } }, { "id": "wesee-crawler", "categories": [ "unknown" ], "pattern": { "accepted": [ "WeSEE:Search" ], "forbidden": [] }, "addition_date": "2013/11/18", "verification": [], "instances": { "accepted": [ "WeSEE:Search", "WeSEE:Search/0.1 (Alpha, http://www.wesee.com/en/support/bot/)" ], "rejected": [] } }, { "id": "niki-bot", "categories": [ "unknown" ], "pattern": { "accepted": [ "niki-bot" ], "forbidden": [] }, "addition_date": "2014/01/01", "verification": [], "instances": { "accepted": [], "rejected": [] } }, { "id": "crystalsemantics-crawler", "categories": [ "unknown" ], "pattern": { "accepted": [ "CrystalSemanticsBot" ], "forbidden": [] }, "addition_date": "2014/02/17", "url": "http://www.crystalsemantics.com/user-agent/", "verification": [], "instances": { "accepted": [], "rejected": [] } }, { "id": "moz-site-audit", "categories": [ "optimizer" ], "pattern": { "accepted": [ "rogerbot" ], "forbidden": [] }, "addition_date": "2014/02/28", "url": "https://moz.com/help/moz-procedures/crawlers/rogerbot", "verification": [], "instances": { "accepted": [ "Mozilla/5.0 (compatible; rogerBot/1.0; UrlCrawler; http://www.seomoz.org/dp/rogerbot)", "rogerbot/1.0 (http://moz.com/help/pro/what-is-rogerbot-, rogerbot-crawler+partager@moz.com)", "rogerbot/1.0 (http://moz.com/help/pro/what-is-rogerbot-, rogerbot-crawler+shiny@moz.com)", "rogerbot/1.0 (http://moz.com/help/pro/what-is-rogerbot-, rogerbot-wherecat@moz.com", "rogerbot/1.0 (http://moz.com/help/pro/what-is-rogerbot-, rogerbot-wherecat@moz.com)", "rogerbot/1.0 (http://www.moz.com/dp/rogerbot, rogerbot-crawler@moz.com)", "rogerbot/1.0 (http://www.seomoz.org/dp/rogerbot, rogerbot-crawler+shiny@seomoz.org)", "rogerbot/1.0 (http://www.seomoz.org/dp/rogerbot, rogerbot-crawler@seomoz.org)", "rogerbot/1.0 (http://www.seomoz.org/dp/rogerbot, rogerbot-wherecat@moz.com)", "rogerbot/1.1 (http://moz.com/help/guides/search-overview/crawl-diagnostics#more-help, rogerbot-crawler+pr2-crawler-05@moz.com)", "rogerbot/1.1 (http://moz.com/help/guides/search-overview/crawl-diagnostics#more-help, rogerbot-crawler+pr4-crawler-11@moz.com)", "rogerbot/1.1 (http://moz.com/help/guides/search-overview/crawl-diagnostics#more-help, rogerbot-crawler+pr4-crawler-15@moz.com)", "rogerbot/1.2 (http://moz.com/help/pro/what-is-rogerbot-, rogerbot-crawler+phaser-testing-crawler-01@moz.com)" ], "rejected": [] }, "aliases": [ "RogerBot" ] }, { "id": "haosou-crawler", "categories": [ "unknown" ], "pattern": { "accepted": [ "360Spider" ], "forbidden": [] }, "addition_date": "2014/03/14", "url": "http://needs-be.blogspot.co.uk/2013/02/how-to-block-spider360.html", "verification": [], "instances": { "accepted": [ "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/21.0.1180.89 Safari/537.1; 360Spider", "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/21.0.1180.89 Safari/537.1; 360Spider(compatible; HaosouSpider; http://www.haosou.com/help/help_3_2.html)", "Mozilla/5.0 (Windows NT 6.2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/31.0.1650.63 Safari/537.36 QIHU 360SE; 360Spider", "Mozilla/5.0 (Windows; U; Windows NT 5.1; zh-CN; ) Firefox/1.5.0.11; 360Spider", "Mozilla/5.0 (Windows; U; Windows NT 5.1; zh-CN; rv:1.8.0.11) Firefox/1.5.0.11; 360Spider", "Mozilla/5.0 (Windows; U; Windows NT 5.1; zh-CN; rv:1.8.0.11) Firefox/1.5.0.11 360Spider;", "Mozilla/5.0 (Windows; U; Windows NT 5.1; zh-CN; rv:1.8.0.11) Gecko/20070312 Firefox/1.5.0.11; 360Spider", "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0); 360Spider", "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0); 360Spider(compatible; HaosouSpider; http://www.haosou.com/help/help_3_2.html)", "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/50.0.2661.102 Safari/537.36; 360Spider" ], "rejected": [] } }, { "id": "picsearch-crawler", "categories": [ "unknown" ], "pattern": { "accepted": [ "psbot" ], "forbidden": [] }, "addition_date": "2014/03/31", "url": "http://www.picsearch.com/bot.html", "verification": [], "instances": { "accepted": [ "psbot-image (+http://www.picsearch.com/bot.html)", "psbot-page (+http://www.picsearch.com/bot.html)", "psbot/0.1 (+http://www.picsearch.com/bot.html)" ], "rejected": [] } }, { "id": "scan-interfax-crawler", "categories": [ "unknown" ], "pattern": { "accepted": [ "InterfaxScanBot" ], "forbidden": [] }, "addition_date": "2014/03/31", "url": "http://scan-interfax.ru", "verification": [], "instances": { "accepted": [], "rejected": [] } }, { "id": "creativecommons-crawler", "categories": [ "unknown" ], "pattern": { "accepted": [ "CC Metadata Scaper" ], "forbidden": [] }, "addition_date": "2014/04/01", "url": "http://wiki.creativecommons.org/Metadata_Scraper", "verification": [], "instances": { "accepted": [ "CC Metadata Scaper http://wiki.creativecommons.org/Metadata_Scraper" ], "rejected": [] } }, { "id": "g00g1e-crawler", "categories": [ "unknown" ], "pattern": { "accepted": [ "g00g1e\\.net" ], "forbidden": [] }, "addition_date": "2014/04/01", "url": "http://www.g00g1e.net/", "verification": [], "instances": { "accepted": [], "rejected": [] } }, { "id": "grapeshot-crawler", "categories": [ "unknown" ], "pattern": { "accepted": [ "GrapeshotCrawler" ], "forbidden": [] }, "addition_date": "2014/04/01", "url": "http://www.grapeshot.co.uk/crawler.php", "verification": [], "instances": { "accepted": [ "Mozilla/5.0 (compatible; GrapeshotCrawler/2.0; +http://www.grapeshot.co.uk/crawler.php)" ], "rejected": [] } }, { "id": "profound-crawler", "categories": [ "unknown" ], "pattern": { "accepted": [ "urlappendbot" ], "forbidden": [] }, "addition_date": "2014/05/10", "url": "http://www.profound.net/urlappendbot.html", "verification": [], "instances": { "accepted": [ "Mozilla/5.0 (compatible; URLAppendBot/1.0; +http://www.profound.net/urlappendbot.html)" ], "rejected": [] } }, { "id": "brainobot", "categories": [ "unknown" ], "pattern": { "accepted": [ "brainobot" ], "forbidden": [] }, "addition_date": "2014/06/24", "verification": [], "instances": { "accepted": [], "rejected": [] } }, { "id": "fr-crawler", "categories": [ "unknown" ], "pattern": { "accepted": [ "fr-crawler" ], "forbidden": [] }, "addition_date": "2014/07/31", "verification": [], "instances": { "accepted": [ "Mozilla/5.0 (compatible; fr-crawler/1.1)" ], "rejected": [] } }, { "id": "binlar", "categories": [ "unknown" ], "pattern": { "accepted": [ "binlar" ], "forbidden": [] }, "addition_date": "2014/09/12", "verification": [], "instances": { "accepted": [ "binlar_2.6.3 binlar2.6.3@unspecified.mail", "binlar_2.6.3 binlar_2.6.3@unspecified.mail", "binlar_2.6.3 larbin2.6.3@unspecified.mail", "binlar_2.6.3 phanendra_kalapala@McAfee.com", "binlar_2.6.3 test@mgmt.mic" ], "rejected": [] } }, { "id": "simple-crawler", "categories": [ "unknown" ], "pattern": { "accepted": [ "SimpleCrawler" ], "forbidden": [] }, "addition_date": "2014/09/12", "verification": [], "instances": { "accepted": [ "SimpleCrawler/0.1" ], "rejected": [] } }, { "id": "twitter-crawler", "categories": [ "preview", "social" ], "pattern": { "accepted": [ "Twitterbot" ], "forbidden": [ "facebookexternalhit", "Facebot" ] }, "addition_date": "2014/09/12", "url": "https://dev.twitter.com/cards/getting-started", "verification": [], "instances": { "accepted": [ "Twitterbot/0.1", "Twitterbot/1.0" ], "rejected": [ "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_1) AppleWebKit/601.2.4 (KHTML, like Gecko) Version/9.0.1 Safari/601.2.4 facebookexternalhit/1.1 Facebot Twitterbot/1.0" ] }, "aliases": [ "TwitterBot" ] }, { "id": "cxense-crawler", "categories": [ "unknown" ], "pattern": { "accepted": [ "cXensebot" ], "forbidden": [] }, "addition_date": "2014/10/05", "verification": [], "instances": { "accepted": [ "cXensebot/1.1a" ], "rejected": [] }, "url": "http://www.cxense.com/bot.html" }, { "id": "similartech-crawler", "categories": [ "unknown" ], "pattern": { "accepted": [ "smtbot" ], "forbidden": [] }, "addition_date": "2014/10/04", "verification": [], "instances": { "accepted": [ "Mozilla/5.0 (compatible; SMTBot/1.0; +http://www.similartech.com/smtbot)", "SMTBot (similartech.com/smtbot)", "Mozilla/5.0 (iPhone; CPU iPhone OS 6_0 like Mac OS X) AppleWebKit/536.26 (KHTML, like Gecko) Version/6.0 Mobile/10A5376e Safari/8536.25 (compatible; SMTBot/1.0; +http://www.similartech.com/smtbot)", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.75 Safari/537.36 (compatible; SMTBot/1.0; +http://www.similartech.com/smtbot)", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.75 Safari/537.36 (compatible; SMTBot/1.0; http://www.similartech.com/smtbot)" ], "rejected": [] }, "url": "http://www.similartech.com/smtbot" }, { "id": "bnf-crawler", "categories": [ "unknown" ], "pattern": { "accepted": [ "bnf\\.fr_bot" ], "forbidden": [] }, "addition_date": "2014/11/18", "url": "http://www.bnf.fr/fr/outils/a.dl_web_capture_robot.html", "verification": [], "instances": { "accepted": [ "Mozilla/5.0 (compatible; bnf.fr_bot; +http://bibnum.bnf.fr/robot/bnf.html)", "Mozilla/5.0 (compatible; bnf.fr_bot; +http://www.bnf.fr/fr/outils/a.dl_web_capture_robot.html)" ], "rejected": [] } }, { "id": "a6corp-crawler", "categories": [ "unknown" ], "pattern": { "accepted": [ "A6-Indexer" ], "forbidden": [] }, "addition_date": "2014/12/05", "url": "http://www.a6corp.com/a6-web-scraping-policy/", "verification": [], "instances": { "accepted": [ "A6-Indexer" ], "rejected": [] } }, { "id": "admantx-crawler", "categories": [ "unknown" ], "pattern": { "accepted": [ "ADmantX" ], "forbidden": [] }, "addition_date": "2014/12/05", "url": "http://www.admantx.com", "verification": [], "instances": { "accepted": [ "ADmantX Platform Semantic Analyzer - ADmantX Inc. - www.admantx.com - support@admantx.com" ], "rejected": [] } }, { "id": "facebook-crawler", "categories": [ "ai", "meta", "social" ], "pattern": { "accepted": [ "Face(book){0,1}[Bb]ot" ], "forbidden": [ "facebookexternalhit", "Twitterbot" ] }, "url": "https://developers.facebook.com/docs/sharing/bot", "addition_date": "2014/12/30", "verification": [ { "type": "cidr", "sources": [ { "type": "http-csv", "url": "http://www.facebook.com/peering/geofeed" } ] } ], "instances": { "accepted": [ "Facebot/1.0", "Mozilla/5.0 (compatible; FacebookBot/1.0; +https://developers.facebook.com/docs/sharing/webmasters/facebookbot/)" ], "rejected": [ "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_1) AppleWebKit/601.2.4 (KHTML, like Gecko) Version/9.0.1 Safari/601.2.4 facebookexternalhit/1.1 Facebot Twitterbot/1.0" ] }, "aliases": [ "FacebookBot" ] }, { "id": "orange-crawler", "categories": [ "unknown" ], "pattern": { "accepted": [ "OrangeBot\\/" ], "forbidden": [] }, "verification": [], "instances": { "accepted": [ "Mozilla/5.0 (compatible; OrangeBot/2.0; support.orangebot@orange.com" ], "rejected": [] }, "addition_date": "2015/01/12" }, { "id": "mignify-crawler", "categories": [ "unknown" ], "pattern": { "accepted": [ "memorybot" ], "forbidden": [] }, "url": "http://mignify.com/bot.htm", "verification": [], "instances": { "accepted": [ "Mozilla/5.0 (compatible; memorybot/1.21.14 +http://mignify.com/bot.html)" ], "rejected": [] }, "addition_date": "2015/02/01" }, { "id": "advbot-crawler", "categories": [ "unknown" ], "pattern": { "accepted": [ "AdvBot" ], "forbidden": [] }, "url": "http://advbot.net/bot.html", "verification": [], "instances": { "accepted": [ "Mozilla/5.0 (compatible; AdvBot/2.0; +http://advbot.net/bot.html)" ], "rejected": [] }, "addition_date": "2015/02/01" }, { "id": "megaindex-crawler", "categories": [ "unknown" ], "pattern": { "accepted": [ "MegaIndex" ], "forbidden": [] }, "url": "https://www.megaindex.ru/?tab=linkAnalyze", "verification": [], "instances": { "accepted": [ "Mozilla/5.0 (compatible; MegaIndex.ru/2.0; +https://www.megaindex.ru/?tab=linkAnalyze)", "Mozilla/5.0 (compatible; MegaIndex.ru/2.0; +http://megaindex.com/crawler)" ], "rejected": [] }, "addition_date": "2015/03/28" }, { "id": "semanticscholar-crawler", "categories": [ "ai", "academic" ], "pattern": { "accepted": [ "SemanticScholarBot" ], "forbidden": [] }, "url": "https://www.semanticscholar.org/crawler", "verification": [], "instances": { "accepted": [ "SemanticScholarBot/1.0 (+http://s2.allenai.org/bot.html)", "Mozilla/5.0 (compatible) SemanticScholarBot (+https://www.semanticscholar.org/crawler)" ], "rejected": [] }, "addition_date": "2015/03/28", "aliases": [ "SemanticScholarBot" ] }, { "id": "ltx71-crawler", "categories": [ "unknown" ], "pattern": { "accepted": [ "ltx71" ], "forbidden": [] }, "url": "http://ltx71.com/", "verification": [], "instances": { "accepted": [ "ltx71 - (http://ltx71.com/)" ], "rejected": [] }, "addition_date": "2015/04/04" }, { "id": "nerdybot-crawler", "categories": [ "unknown" ], "pattern": { "accepted": [ "nerdybot" ], "forbidden": [] }, "url": "http://nerdybot.com/", "verification": [], "instances": { "accepted": [ "nerdybot" ], "rejected": [] }, "addition_date": "2015/04/05" }, { "id": "xovibot-crawler", "categories": [ "unknown" ], "pattern": { "accepted": [ "xovibot" ], "forbidden": [] }, "url": "http://www.xovibot.net/", "verification": [], "instances": { "accepted": [ "Mozilla/5.0 (compatible; XoviBot/2.0; +http://www.xovibot.net/)" ], "rejected": [] }, "addition_date": "2015/04/05" }, { "id": "law-unimi-crawler", "categories": [ "academic", "tool" ], "pattern": { "accepted": [ "BUbiNG" ], "forbidden": [] }, "url": "http://law.di.unimi.it/BUbiNG.html", "verification": [], "instances": { "accepted": [ "BUbiNG (+http://law.di.unimi.it/BUbiNG.html)" ], "rejected": [] }, "addition_date": "2015/04/06" }, { "id": "qwant-crawler", "categories": [ "search-engine" ], "pattern": { "accepted": [ "Qwantify" ], "forbidden": [] }, "url": "https://www.qwant.com/", "verification": [], "instances": { "accepted": [ "Mozilla/5.0 (compatible; Qwantify/2.0n; +https://www.qwant.com/)/*", "Mozilla/5.0 (compatible; Qwantify/2.4w; +https://www.qwant.com/)/2.4w", "Mozilla/5.0 (compatible; Qwantify/Bleriot/1.1; +https://help.qwant.com/bot)", "Mozilla/5.0 (compatible; Qwantify/Bleriot/1.2.1; +https://help.qwant.com/bot)" ], "rejected": [] }, "addition_date": "2015/04/06" }, { "id": "archiveorg-archiver", "categories": [ "archive" ], "pattern": { "accepted": [ "archive\\.org_bot" ], "forbidden": [] }, "url": "http://www.archive.org/details/archive.org_bot", "depends_on": [ "heritrix" ], "verification": [], "instances": { "accepted": [ "Mozilla/5.0 (compatible; heritrix/3.1.1-SNAPSHOT-20120116.200628 +http://www.archive.org/details/archive.org_bot)", "Mozilla/5.0 (compatible; archive.org_bot/heritrix-1.15.4 +http://www.archive.org)", "Mozilla/5.0 (compatible; heritrix/3.3.0-SNAPSHOT-20140702-2247 +http://archive.org/details/archive.org_bot)", "Mozilla/5.0 (compatible; archive.org_bot +http://www.archive.org/details/archive.org_bot)", "Mozilla/5.0 (compatible; archive.org_bot +http://archive.org/details/archive.org_bot)", "Mozilla/5.0 (compatible; special_archiver/3.1.1 +http://www.archive.org/details/archive.org_bot)" ], "rejected": [] }, "addition_date": "2015/04/14", "aliases": [ "ArchiveOrgBot" ] }, { "id": "apple-crawler", "categories": [ "apple", "search-engine" ], "pattern": { "accepted": [ "Applebot" ], "forbidden": [] }, "url": "http://www.apple.com/go/applebot", "addition_date": "2015/04/15", "verification": [ { "type": "dns", "masks": [ "@.applebot.apple.com" ] } ], "instances": { "accepted": [ "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/600.2.5 (KHTML, like Gecko) Version/8.0.2 Safari/600.2.5 (Applebot/0.1)", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/600.2.5 (KHTML, like Gecko) Version/8.0.2 Safari/600.2.5 (Applebot/0.1; +http://www.apple.com/go/applebot)", "Mozilla/5.0 (compatible; Applebot/0.3; +http://www.apple.com/go/applebot)", "Mozilla/5.0 (iPhone; CPU iPhone OS 6_0 like Mac OS X) AppleWebKit/536.26 (KHTML, like Gecko) Version/6.0 Mobile/10A5376e Safari/8536.25 (compatible; Applebot/0.3; +http://www.apple.com/go/applebot)", "Mozilla/5.0 (iPhone; CPU iPhone OS 8_1 like Mac OS X) AppleWebKit/600.1.4 (KHTML, like Gecko) Version/8.0 Mobile/12B410 Safari/600.1.4 (Applebot/0.1; +http://www.apple.com/go/applebot)" ], "rejected": [] }, "aliases": [ "Applebot" ] }, { "id": "apple-feedfetcher", "categories": [ "feedfetcher" ], "pattern": { "accepted": [ "i[Tt][Mm][Ss]" ], "forbidden": [] }, "addition_date": "2024/09/19", "verification": [], "instances": { "accepted": [ "iTMS", "itms" ], "rejected": [] }, "url": "https://support.apple.com/en-us/119829" }, { "id": "tweetmemebot", "categories": [ "unknown" ], "pattern": { "accepted": [ "TweetmemeBot" ], "forbidden": [] }, "url": "http://datasift.com/bot.html", "verification": [], "instances": { "accepted": [ "Mozilla/5.0 (TweetmemeBot/4.0; +http://datasift.com/bot.html) Gecko/20100101 Firefox/31.0" ], "rejected": [] }, "addition_date": "2015/04/15" }, { "id": "java-crawler4j", "categories": [ "programmatic" ], "pattern": { "accepted": [ "crawler4j" ], "forbidden": [] }, "url": "https://github.com/yasserg/crawler4j", "verification": [], "instances": { "accepted": [ "crawler4j (http://code.google.com/p/crawler4j/)", "crawler4j (https://github.com/yasserg/crawler4j/)" ], "rejected": [] }, "addition_date": "2015/05/07" }, { "id": "privacore-crawler", "categories": [ "unknown" ], "pattern": { "accepted": [ "findxbot" ], "forbidden": [] }, "url": "http://www.findxbot.com", "verification": [], "instances": { "accepted": [ "Mozilla/5.0 (compatible; Findxbot/1.0; +http://www.findxbot.com)" ], "rejected": [] }, "addition_date": "2015/05/07" }, { "id": "semrush-crawler", "categories": [ "unknown" ], "pattern": { "accepted": [ "S[eE][mM]rushBot" ], "forbidden": [] }, "url": "http://www.semrush.com/bot.html", "verification": [], "instances": { "accepted": [ "Mozilla/5.0 (compatible; SemrushBot-SA/0.97; +http://www.semrush.com/bot.html)", "Mozilla/5.0 (compatible; SemrushBot-SI/0.97; +http://www.semrush.com/bot.html)", "Mozilla/5.0 (compatible; SemrushBot/3~bl; +http://www.semrush.com/bot.html)", "Mozilla/5.0 (compatible; SemrushBot/0.98~bl; +http://www.semrush.com/bot.html)", "Mozilla/5.0 (compatible; SemrushBot-BA; +http://www.semrush.com/bot.html)", "Mozilla/5.0 (compatible; SemrushBot/6~bl; +http://www.semrush.com/bot.html)", "Mozilla/5.0 (compatible; SemrushBot/7~bl; +http://www.semrush.com/bot.html)", "SEMrushBot" ], "rejected": [] }, "addition_date": "2015/05/26", "aliases": [ "SemRushBot" ] }, { "id": "yooz-crawler", "categories": [ "unknown" ], "pattern": { "accepted": [ "yoozBot" ], "forbidden": [] }, "url": "http://yooz.ir", "verification": [], "instances": { "accepted": [ "Mozilla/5.0 (compatible; yoozBot-2.2; http://yooz.ir; info@yooz.ir)" ], "rejected": [] }, "addition_date": "2015/05/26" }, { "id": "lipperhey-crawler", "categories": [ "unknown" ], "pattern": { "accepted": [ "lipperhey" ], "forbidden": [] }, "url": "http://www.lipperhey.com/", "verification": [], "instances": { "accepted": [ "Mozilla/5.0 (compatible; Lipperhey Link Explorer; http://www.lipperhey.com/)", "Mozilla/5.0 (compatible; Lipperhey SEO Service; http://www.lipperhey.com/)", "Mozilla/5.0 (compatible; Lipperhey Site Explorer; http://www.lipperhey.com/)", "Mozilla/5.0 (compatible; Lipperhey-Kaus-Australis/5.0; +https://www.lipperhey.com/en/about/)" ], "rejected": [] }, "addition_date": "2015/08/26" }, { "id": "yahoo-crawler-japan", "categories": [ "yahoo" ], "pattern": { "accepted": [ "Y!J" ], "forbidden": [] }, "url": "https://www.yahoo-help.jp/app/answers/detail/p/595/a_id/42716/~/%E3%82%A6%E3%82%A7%E3%83%96%E3%83%9A%E3%83%BC%E3%82%B8%E3%81%AB%E3%82%A2%E3%82%AF%E3%82%BB%E3%82%B9%E3%81%99%E3%82%8B%E3%82%B7%E3%82%B9%E3%83%86%E3%83%A0%E3%81%AE%E3%83%A6%E3%83%BC%E3%82%B6%E3%83%BC%E3%82%A8%E3%83%BC%E3%82%B8%E3%82%A7%E3%83%B3%E3%83%88%E3%81%AB%E3%81%A4%E3%81%84%E3%81%A6", "verification": [], "instances": { "accepted": [ "Y!J-ASR/0.1 crawler (http://www.yahoo-help.jp/app/answers/detail/p/595/a_id/42716/)", "Y!J-BRJ/YATS crawler (http://help.yahoo.co.jp/help/jp/search/indexing/indexing-15.html)", "Y!J-PSC/1.0 crawler (http://help.yahoo.co.jp/help/jp/search/indexing/indexing-15.html)", "Y!J-BRW/1.0 crawler (http://help.yahoo.co.jp/help/jp/search/indexing/indexing-15.html)", "Mozilla/5.0 (iPhone; Y!J-BRY/YATSH crawler; http://help.yahoo.co.jp/help/jp/search/indexing/indexing-15.html)", "Mozilla/5.0 (compatible; Y!J SearchMonkey/1.0 (Y!J-AGENT; http://help.yahoo.co.jp/help/jp/search/indexing/indexing-15.html))" ], "rejected": [] }, "addition_date": "2015/05/26" }, { "id": "domainreanimator-crawler", "categories": [ "unknown" ], "pattern": { "accepted": [ "Domain Re-Animator Bot" ], "forbidden": [] }, "url": "http://domainreanimator.com", "verification": [], "instances": { "accepted": [ "Domain Re-Animator Bot (http://domainreanimator.com) - support@domainreanimator.com" ], "rejected": [] }, "addition_date": "2015/04/14" }, { "id": "addthis-crawler", "categories": [ "unknown" ], "pattern": { "accepted": [ "AddThis" ], "forbidden": [] }, "url": "https://www.addthis.com", "verification": [], "instances": { "accepted": [ "AddThis.com robot tech.support@clearspring.com" ], "rejected": [] }, "addition_date": "2015/06/02" }, { "id": "screamingfrog-crawler", "categories": [ "optimizer" ], "pattern": { "accepted": [ "Screaming Frog SEO Spider" ], "forbidden": [] }, "url": "http://www.screamingfrog.co.uk/seo-spider", "verification": [], "instances": { "accepted": [ "Screaming Frog SEO Spider/5.1" ], "rejected": [] }, "addition_date": "2016/01/08" }, { "id": "metauri-crawler", "categories": [ "unknown" ], "pattern": { "accepted": [ "MetaURI" ], "forbidden": [] }, "url": "http://www.useragentstring.com/MetaURI_id_17683.php", "verification": [], "instances": { "accepted": [ "MetaURI API/2.0 +metauri.com" ], "rejected": [] }, "addition_date": "2016/01/02" }, { "id": "python-scrapy", "categories": [ "ai", "programmatic" ], "pattern": { "accepted": [ "Scrapy" ], "forbidden": [] }, "url": "http://scrapy.org/", "verification": [], "instances": { "accepted": [ "Scrapy/1.0.3 (+http://scrapy.org)" ], "rejected": [] }, "addition_date": "2016/01/02" }, { "id": "livelap-crawler", "categories": [ "unknown" ], "pattern": { "accepted": [ "Livelap[bB]ot" ], "forbidden": [] }, "url": "http://site.livelap.com/crawler", "verification": [], "instances": { "accepted": [ "LivelapBot/0.2 (http://site.livelap.com/crawler)", "Livelapbot/0.1" ], "rejected": [] }, "addition_date": "2016/01/02" }, { "id": "openhose-crawler", "categories": [ "unknown" ], "pattern": { "accepted": [ "OpenHoseBot" ], "forbidden": [] }, "url": "http://www.openhose.org/bot.html", "verification": [], "instances": { "accepted": [ "Mozilla/5.0 (compatible; OpenHoseBot/2.1; +http://www.openhose.org/bot.html)" ], "rejected": [] }, "addition_date": "2016/01/02" }, { "id": "capsulink-crawler", "categories": [ "unknown" ], "pattern": { "accepted": [ "CapsuleChecker" ], "forbidden": [] }, "url": "http://www.capsulink.com/about", "verification": [], "instances": { "accepted": [ "CapsuleChecker (http://www.capsulink.com/)" ], "rejected": [] }, "addition_date": "2016/01/02" }, { "id": "infegy-crawler", "categories": [ "ai" ], "pattern": { "accepted": [ "collection@infegy\\.com" ], "forbidden": [] }, "url": "http://infegy.com/", "verification": [], "instances": { "accepted": [ "Mozilla/5.0 (compatible) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/47.0.2526.73 Safari/537.36 collection@infegy.com" ], "rejected": [] }, "addition_date": "2016/01/03" }, { "id": "tiscali-crawler", "categories": [ "unknown" ], "pattern": { "accepted": [ "IstellaBot" ], "forbidden": [] }, "url": "http://www.tiscali.it/", "verification": [], "instances": { "accepted": [ "Mozilla/5.0 (compatible; IstellaBot/1.23.15 +http://www.tiscali.it/)" ], "rejected": [] }, "addition_date": "2016/01/09" }, { "id": "deusu-crawler", "categories": [ "unknown" ], "pattern": { "accepted": [ "DeuSu\\/" ], "forbidden": [] }, "addition_date": "2016/01/23", "url": "https://deusu.de/robot.html", "verification": [], "instances": { "accepted": [ "Mozilla/5.0 (compatible; DeuSu/0.1.0; +https://deusu.org)", "Mozilla/5.0 (compatible; DeuSu/5.0.2; +https://deusu.de/robot.html)" ], "rejected": [] } }, { "id": "betabot", "categories": [ "unknown" ], "pattern": { "accepted": [ "betaBot" ], "forbidden": [] }, "addition_date": "2016/01/23", "verification": [], "instances": { "accepted": [], "rejected": [] } }, { "id": "cliqz-crawler", "categories": [ "unknown" ], "pattern": { "accepted": [ "Cliqzbot\\/" ], "forbidden": [] }, "addition_date": "2016/01/23", "url": "http://cliqz.com/company/cliqzbot", "verification": [], "instances": { "accepted": [ "Mozilla/5.0 (compatible; Cliqzbot/2.0; +http://cliqz.com/company/cliqzbot)", "Cliqzbot/0.1 (+http://cliqz.com +cliqzbot@cliqz.com)", "Cliqzbot/0.1 (+http://cliqz.com/company/cliqzbot)", "Mozilla/5.0 (compatible; Cliqzbot/0.1 +http://cliqz.com/company/cliqzbot)", "Mozilla/5.0 (compatible; Cliqzbot/1.0 +http://cliqz.com/company/cliqzbot)" ], "rejected": [] } }, { "id": "mojeek-crawler", "categories": [ "search-engine" ], "pattern": { "accepted": [ "MojeekBot\\/" ], "forbidden": [] }, "addition_date": "2016/01/23", "url": "https://www.mojeek.com/bot.html", "verification": [], "instances": { "accepted": [ "MojeekBot/0.2 (archi; http://www.mojeek.com/bot.html)", "Mozilla/5.0 (compatible; MojeekBot/0.2; http://www.mojeek.com/bot.html#relaunch)", "Mozilla/5.0 (compatible; MojeekBot/0.2; http://www.mojeek.com/bot.html)", "Mozilla/5.0 (compatible; MojeekBot/0.5; http://www.mojeek.com/bot.html)", "Mozilla/5.0 (compatible; MojeekBot/0.6; +https://www.mojeek.com/bot.html)", "Mozilla/5.0 (compatible; MojeekBot/0.6; http://www.mojeek.com/bot.html)" ], "rejected": [] } }, { "id": "netestate-crawler", "categories": [ "search-engine" ], "pattern": { "accepted": [ "netEstate NE Crawler" ], "forbidden": [] }, "addition_date": "2016/01/23", "url": "http://www.website-datenbank.de/", "verification": [], "instances": { "accepted": [ "netEstate NE Crawler (+http://www.sengine.info/)", "netEstate NE Crawler (+http://www.website-datenbank.de/)" ], "rejected": [] } }, { "id": "avira-crawler", "categories": [ "search-engine" ], "pattern": { "accepted": [ "SafeSearch microdata crawler" ], "forbidden": [] }, "addition_date": "2016/01/23", "url": "https://safesearch.avira.com", "verification": [], "instances": { "accepted": [ "SafeSearch microdata crawler (https://safesearch.avira.com, safesearch-abuse@avira.com)" ], "rejected": [] } }, { "id": "glutenfreepleasure-crawler", "categories": [ "ai" ], "pattern": { "accepted": [ "Gluten Free Crawler\\/" ], "forbidden": [] }, "addition_date": "2016/01/23", "url": "http://glutenfreepleasure.com/", "verification": [], "instances": { "accepted": [ "Mozilla/5.0 (compatible; Gluten Free Crawler/1.0; +http://glutenfreepleasure.com/)" ], "rejected": [] } }, { "id": "yamanalab-crawler", "categories": [ "academic" ], "pattern": { "accepted": [ "Sonic" ], "forbidden": [] }, "addition_date": "2016/02/08", "url": "https://www.yama.info.waseda.ac.jp/~crawler/info_en.html", "verification": [], "instances": { "accepted": [ "Mozilla/5.0 (compatible; RankSonicSiteAuditor/1.0; +https://ranksonic.com/ranksonic_sab.html)", "Mozilla/5.0 (compatible; Sonic/1.0; http://www.yama.info.waseda.ac.jp/~crawler/info.html)", "Mozzila/5.0 (compatible; Sonic/1.0; http://www.yama.info.waseda.ac.jp/~crawler/info.html)" ], "rejected": [] } }, { "id": "sysomos-crawler", "categories": [ "unknown" ], "pattern": { "accepted": [ "Sysomos" ], "forbidden": [] }, "addition_date": "2016/02/08", "url": "http://www.sysomos.com", "verification": [], "instances": { "accepted": [ "Mozilla/5.0 (compatible; Sysomos/1.0; +http://www.sysomos.com/; Sysomos)" ], "rejected": [] } }, { "id": "trove-crawler", "categories": [ "unknown" ], "pattern": { "accepted": [ "Trove" ], "forbidden": [] }, "addition_date": "2016/02/08", "url": "http://www.trove.com", "verification": [], "instances": { "accepted": [], "rejected": [] } }, { "id": "deadlinkchecker", "categories": [ "monitor" ], "pattern": { "accepted": [ "deadlinkchecker" ], "forbidden": [] }, "addition_date": "2016/02/08", "url": "http://www.deadlinkchecker.com", "verification": [], "instances": { "accepted": [ "www.deadlinkchecker.com Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/46.0.2490.86 Safari/537.36", "www.deadlinkchecker.com XMLHTTP/1.0", "www.deadlinkchecker.com XMLHTTP/1.0 Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/46.0.2490.86 Safari/537.36" ], "rejected": [] } }, { "id": "slack-image-proxy", "categories": [ "slack", "preview" ], "pattern": { "accepted": [ "Slack-ImgProxy" ], "forbidden": [] }, "addition_date": "2016/04/25", "url": "https://api.slack.com/robots", "verification": [], "instances": { "accepted": [ "Slack-ImgProxy (+https://api.slack.com/robots)", "Slack-ImgProxy 0.59 (+https://api.slack.com/robots)", "Slack-ImgProxy 0.66 (+https://api.slack.com/robots)", "Slack-ImgProxy 1.106 (+https://api.slack.com/robots)", "Slack-ImgProxy 1.138 (+https://api.slack.com/robots)", "Slack-ImgProxy 149 (+https://api.slack.com/robots)" ], "rejected": [] } }, { "id": "embedly-crawler", "categories": [ "preview" ], "pattern": { "accepted": [ "Embedly" ], "forbidden": [] }, "addition_date": "2016/04/25", "url": "http://support.embed.ly", "verification": [], "instances": { "accepted": [ "Embedly +support@embed.ly", "Mozilla/5.0 (compatible; Embedly/0.2; +http://support.embed.ly/)", "Mozilla/5.0 (compatible; Embedly/0.2; snap; +http://support.embed.ly/)" ], "rejected": [] }, "aliases": [ "EmbedlyBot" ] }, { "id": "rankactive-crawler", "categories": [ "unknown" ], "pattern": { "accepted": [ "RankActiveLinkBot" ], "forbidden": [] }, "addition_date": "2016/06/20", "url": "https://rankactive.com/resources/rankactive-linkbot", "verification": [], "instances": { "accepted": [ "Mozilla/5.0 (compatible; RankActiveLinkBot; +https://rankactive.com/resources/rankactive-linkbot)" ], "rejected": [] } }, { "id": "iskanie-crawler", "categories": [ "unknown" ], "pattern": { "accepted": [ "iskanie" ], "forbidden": [] }, "addition_date": "2016/09/02", "url": "http://www.iskanie.com", "verification": [], "instances": { "accepted": [ "iskanie (+http://www.iskanie.com)" ], "rejected": [] } }, { "id": "safedns-crawler", "categories": [ "ai", "monitor" ], "pattern": { "accepted": [ "SafeDNSBot" ], "forbidden": [] }, "addition_date": "2016/09/10", "url": "https://www.safedns.com/searchbot", "verification": [], "instances": { "accepted": [ "SafeDNSBot (https://www.safedns.com/searchbot)" ], "rejected": [] } }, { "id": "skype-preview", "categories": [ "microsoft", "preview" ], "pattern": { "accepted": [ "SkypeUriPreview" ], "forbidden": [] }, "addition_date": "2016/10/10", "verification": [], "instances": { "accepted": [ "Mozilla/5.0 (Windows NT 6.1; WOW64) SkypeUriPreview Preview/0.5" ], "rejected": [] } }, { "id": "veooz-crawler", "categories": [ "unknown" ], "pattern": { "accepted": [ "Veoozbot" ], "forbidden": [] }, "addition_date": "2016/11/03", "url": "http://www.veooz.com/veoozbot.html", "verification": [], "instances": { "accepted": [ "Mozilla/5.0 (compatible; Veoozbot/1.0; +http://www.veooz.com/veoozbot.html)" ], "rejected": [] } }, { "id": "slack-crawler", "categories": [ "slack" ], "pattern": { "accepted": [ "Slackbot" ], "forbidden": [] }, "addition_date": "2016/11/03", "url": "https://api.slack.com/robots", "verification": [], "instances": { "accepted": [ "Slackbot-LinkExpanding (+https://api.slack.com/robots)", "Slackbot-LinkExpanding 1.0 (+https://api.slack.com/robots)", "Slackbot 1.0 (+https://api.slack.com/robots)" ], "rejected": [] }, "aliases": [ "SlackBot" ] }, { "id": "reddit-crawler", "categories": [ "social" ], "pattern": { "accepted": [ "redditbot" ], "forbidden": [] }, "addition_date": "2016/11/03", "url": "http://www.reddit.com/feedback", "verification": [], "instances": { "accepted": [ "Mozilla/5.0 (compatible; redditbot/1.0; +http://www.reddit.com/feedback)" ], "rejected": [] }, "aliases": [ "RedditBot" ] }, { "id": "datagnion-crawler", "categories": [ "unknown" ], "pattern": { "accepted": [ "datagnionbot" ], "forbidden": [] }, "addition_date": "2016/11/03", "url": "http://www.datagnion.com/bot.html", "verification": [], "instances": { "accepted": [ "datagnionbot (+http://www.datagnion.com/bot.html)" ], "rejected": [] } }, { "id": "google-adwords", "categories": [ "advertising", "google" ], "pattern": { "accepted": [ "Google-Adwords-Instant" ], "forbidden": [] }, "addition_date": "2016/11/03", "url": "http://www.google.com/adsbot.html", "verification": [], "instances": { "accepted": [ "Google-Adwords-Instant (+http://www.google.com/adsbot.html)" ], "rejected": [] } }, { "id": "adbeat-crawler", "categories": [ "unknown" ], "pattern": { "accepted": [ "adbeat_bot" ], "forbidden": [] }, "addition_date": "2016/11/04", "verification": [], "instances": { "accepted": [ "Mozilla/5.0 (compatible; adbeat_bot; +support@adbeat.com; support@adbeat.com)", "adbeat_bot" ], "rejected": [] } }, { "id": "whatsapp-crawler", "categories": [ "meta", "preview", "social" ], "pattern": { "accepted": [ "WhatsApp" ], "forbidden": [] }, "addition_date": "2016/11/15", "url": "https://www.whatsapp.com/", "verification": [], "instances": { "accepted": [ "WhatsApp", "WhatsApp/0.3.4479 N", "WhatsApp/0.3.4679 N", "WhatsApp/0.3.4941 N", "WhatsApp/2.12.15/i", "WhatsApp/2.12.16/i", "WhatsApp/2.12.17/i", "WhatsApp/2.12.449 A", "WhatsApp/2.12.453 A", "WhatsApp/2.12.510 A", "WhatsApp/2.12.540 A", "WhatsApp/2.12.548 A", "WhatsApp/2.12.555 A", "WhatsApp/2.12.556 A", "WhatsApp/2.16.1/i", "WhatsApp/2.16.13 A", "WhatsApp/2.16.2/i", "WhatsApp/2.16.42 A", "WhatsApp/2.16.57 A", "WhatsApp/2.19.92 i", "WhatsApp/2.19.175 A", "WhatsApp/2.19.244 A", "WhatsApp/2.19.258 A", "WhatsApp/2.19.308 A", "WhatsApp/2.19.330 A" ], "rejected": [] }, "aliases": [ "WhatsAppBot" ] }, { "id": "contxbot", "categories": [ "unknown" ], "pattern": { "accepted": [ "contxbot" ], "forbidden": [] }, "addition_date": "2017/02/25", "verification": [], "instances": { "accepted": [ "Mozilla/5.0 (compatible;contxbot/1.0)" ], "rejected": [] } }, { "id": "pintrest-crawler", "categories": [ "social" ], "pattern": { "accepted": [ "pinterest\\.com\\/bot" ], "forbidden": [] }, "addition_date": "2017/03/03", "verification": [], "instances": { "accepted": [ "Mozilla/5.0 (compatible; Pinterestbot/1.0; +http://www.pinterest.com/bot.html)", "Pinterest/0.2 (+http://www.pinterest.com/bot.html)" ], "rejected": [] }, "url": "http://www.pinterest.com/bot.html", "aliases": [ "PinterestBot" ] }, { "id": "duedil-crawler", "categories": [ "unknown" ], "pattern": { "accepted": [ "electricmonk" ], "forbidden": [] }, "addition_date": "2017/03/04", "verification": [], "instances": { "accepted": [ "Mozilla/5.0 (compatible; electricmonk/3.2.0 +https://www.duedil.com/our-crawler/)" ], "rejected": [] }, "url": "https://www.duedil.com/our-crawler/" }, { "id": "garlik-crawler", "categories": [ "unknown" ], "pattern": { "accepted": [ "GarlikCrawler" ], "forbidden": [] }, "addition_date": "2017/03/18", "verification": [], "instances": { "accepted": [ "GarlikCrawler/1.2 (http://garlik.com/, crawler@garlik.com)" ], "rejected": [] }, "url": "http://garlik.com/" }, { "id": "bing-preview", "categories": [ "microsoft", "preview" ], "pattern": { "accepted": [ "BingPreview\\/" ], "forbidden": [] }, "addition_date": "2017/04/23", "url": "https://www.bing.com/webmaster/help/which-crawlers-does-bing-use-8c184ec0", "verification": [ { "type": "cidr", "sources": [ { "type": "http-json", "url": "https://www.bing.com/toolbox/bingbot.json", "selector": "$.prefixes[*][\\\"ipv6Prefix\\\",\\\"ipv4Prefix\\\"]" } ] }, { "type": "dns", "masks": [ "@.search.msn.com" ] } ], "instances": { "accepted": [ "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/534+ (KHTML, like Gecko) BingPreview/1.0b", "Mozilla/5.0 (Windows NT 6.3; WOW64; Trident/7.0; rv:11.0; BingPreview/1.0b) like Gecko", "Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.2; Trident/6.0; WOW64; Trident/6.0; BingPreview/1.0b)", "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0; WOW64; Trident/5.0; BingPreview/1.0b)", "Mozilla/5.0 (iPhone; CPU iPhone OS 7_0 like Mac OS X) AppleWebKit/537.51.1 (KHTML, like Gecko) Version/7.0 Mobile/11A465 Safari/9537.53 BingPreview/1.0b" ], "rejected": [] }, "aliases": [ "BingPreviewBot" ] }, { "id": "vebidoo-crawler", "categories": [ "unknown" ], "pattern": { "accepted": [ "vebidoobot" ], "forbidden": [] }, "addition_date": "2017/05/08", "verification": [], "instances": { "accepted": [ "Mozilla/5.0 (compatible; vebidoobot/1.0; +https://blog.vebidoo.de/vebidoobot/" ], "rejected": [] }, "url": "https://blog.vebidoo.de/vebidoobot/" }, { "id": "femtosearch-crawler", "categories": [ "unknown" ], "pattern": { "accepted": [ "FemtosearchBot" ], "forbidden": [] }, "addition_date": "2017/05/16", "verification": [], "instances": { "accepted": [ "Mozilla/5.0 (compatible; FemtosearchBot/1.0; http://femtosearch.com)" ], "rejected": [] }, "url": "http://femtosearch.com" }, { "id": "yahoo-preview", "categories": [ "yahoo", "preview" ], "pattern": { "accepted": [ "Yahoo Link Preview" ], "forbidden": [] }, "addition_date": "2017/06/28", "verification": [], "instances": { "accepted": [ "Mozilla/5.0 (compatible; Yahoo Link Preview; https://help.yahoo.com/kb/mail/yahoo-link-preview-SLN23615.html)" ], "rejected": [] }, "url": "https://help.yahoo.com/kb/mail/yahoo-link-preview-SLN23615.html" }, { "id": "metajob-crawler", "categories": [ "unknown" ], "pattern": { "accepted": [ "MetaJobBot" ], "forbidden": [] }, "addition_date": "2017/08/16", "verification": [], "instances": { "accepted": [ "Mozilla/5.0 (compatible; MetaJobBot; http://www.metajob.de/crawler)" ], "rejected": [] }, "url": "http://www.metajob.de/the/crawler" }, { "id": "domainstats-crawler", "categories": [ "unknown" ], "pattern": { "accepted": [ "DomainStatsBot" ], "forbidden": [] }, "addition_date": "2017/08/16", "verification": [], "instances": { "accepted": [ "DomainStatsBot/1.0 (http://domainstats.io/our-bot)" ], "rejected": [] }, "url": "http://domainstats.io/our-bot" }, { "id": "datenbutler-crawler", "categories": [ "unknown" ], "pattern": { "accepted": [ "mindUpBot" ], "forbidden": [] }, "addition_date": "2017/08/16", "verification": [], "instances": { "accepted": [ "mindUpBot (datenbutler.de)" ], "rejected": [] }, "url": "http://www.datenbutler.de/" }, { "id": "daum-crawler", "categories": [ "unknown" ], "pattern": { "accepted": [ "Daum\\/" ], "forbidden": [] }, "addition_date": "2017/08/16", "verification": [], "instances": { "accepted": [ "Mozilla/5.0 (compatible; Daum/4.1; +http://cs.daum.net/faq/15/4118.html?faqId=28966)" ], "rejected": [] }, "url": "http://cs.daum.net/faq/15/4118.html?faqId=28966" }, { "id": "jusprog-crawler", "categories": [ "unknown" ], "pattern": { "accepted": [ "Jugendschutzprogramm-Crawler" ], "forbidden": [] }, "addition_date": "2017/08/16", "verification": [], "instances": { "accepted": [ "Jugendschutzprogramm-Crawler; Info: http://www.jugendschutzprogramm.de" ], "rejected": [] }, "url": "http://www.jugendschutzprogramm.de" }, { "id": "xenu-crawler", "categories": [ "monitor", "tool" ], "pattern": { "accepted": [ "Xenu Link Sleuth" ], "forbidden": [] }, "addition_date": "2017/08/19", "verification": [], "instances": { "accepted": [ "Xenu Link Sleuth/1.3.8" ], "rejected": [] }, "url": "http://home.snafu.de/tilman/xenulink.html" }, { "id": "perl-pcore", "categories": [ "programmatic" ], "pattern": { "accepted": [ "Pcore-HTTP" ], "forbidden": [] }, "addition_date": "2017/08/19", "verification": [], "instances": { "accepted": [ "Pcore-HTTP/v0.40.3", "Pcore-HTTP/v0.44.0" ], "rejected": [] }, "url": "https://bitbucket.org/softvisio/pcore/overview" }, { "id": "moat-crawler", "categories": [ "advertising" ], "pattern": { "accepted": [ "moatbot" ], "forbidden": [] }, "addition_date": "2017/09/16", "verification": [], "instances": { "accepted": [ "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/40.0.2214.111 Safari/537.36 moatbot", "Mozilla/5.0 (iPhone; CPU iPhone OS 8_0 like Mac OS X) AppleWebKit/600.1.3 (KHTML, like Gecko) Version/8.0 Mobile/12A4345d Safari/600.1.4 moatbot" ], "rejected": [] }, "url": "https://moat.com" }, { "id": "kosmio-crawler", "categories": [ "unknown" ], "pattern": { "accepted": [ "KosmioBot" ], "forbidden": [] }, "addition_date": "2017/09/16", "verification": [], "instances": { "accepted": [ "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/44.0.2403.125 Safari/537.36 (compatible; KosmioBot/1.0; +http://kosm.io/bot.html)" ], "rejected": [] }, "url": "http://kosm.io/bot.html" }, { "id": "pingdom-crawler", "categories": [ "monitor" ], "pattern": { "accepted": [ "[pP]ingdom" ], "forbidden": [] }, "addition_date": "2017/09/16", "verification": [ { "type": "ip", "sources": [ { "type": "http-text", "url": "https://my.pingdom.com/probes/ipv4" }, { "type": "http-text", "url": "https://my.pingdom.com/probes/ipv6" } ] } ], "instances": { "accepted": [ "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Ubuntu Chromium/59.0.3071.109 Chrome/59.0.3071.109 Safari/537.36 PingdomPageSpeed/1.0 (pingbot/2.0; +http://www.pingdom.com/)", "Mozilla/5.0 (compatible; pingbot/2.0; +http://www.pingdom.com/)", "Pingdom.com_bot_version_1.4_(http://www.pingdom.com/)", "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Ubuntu Chromium/61.0.3163.100 Chrome/61.0.3163.100 Safari/537.36 PingdomPageSpeed/1.0 (pingbot/2.0; +http://www.pingdom.com/)", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) browser/2020.2.1 Chrome/78.0.3904.130 Electron/7.3.2 Safari/537.36 PingdomTMS/2020.2", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) browser/2020.2.5 Chrome/78.0.3904.130 Electron/7.3.15 Safari/537.36 PingdomTMS/2020.2", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) browser/2020.2.0 Chrome/78.0.3904.130 Electron/7.1.7 Safari/537.36 PingdomTMS/2020.2", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) renderer/2020.2.0 Chrome/78.0.3904.130 Electron/7.1.7 Safari/537.36 PingdomTMS/2020.2", "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Ubuntu Chromium/61.0.3163.100 Chrome/61.0.3163.100 Safari/537.36 PingdomPageSpeed/1.0 (pingbot/2.0; http://www.pingdom.com/)" ], "rejected": [] }, "url": "http://www.pingdom.com", "aliases": [ "PingdomBot" ] }, { "id": "azure-app-insights", "categories": [ "microsoft", "monitor" ], "pattern": { "accepted": [ "AppInsights" ], "forbidden": [] }, "addition_date": "2019/03/09", "verification": [], "instances": { "accepted": [ "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0; AppInsights)" ], "rejected": [] }, "url": "https://docs.microsoft.com/en-us/azure/azure-monitor/app/app-insights-overview" }, { "id": "javascript-phantom", "categories": [ "programmatic" ], "pattern": { "accepted": [ "PhantomJS" ], "forbidden": [] }, "addition_date": "2017/09/18", "verification": [], "instances": { "accepted": [ "Mozilla/5.0 (Unknown; Linux x86_64) AppleWebKit/538.1 (KHTML, like Gecko) PhantomJS/2.1.1 Safari/538.1 bl.uk_lddc_renderbot/2.0.0 (+ http://www.bl.uk/aboutus/legaldeposit/websites/websites/faqswebmaster/index.html)" ], "rejected": [] }, "url": "http://phantomjs.org/" }, { "id": "gowiki-crawler", "categories": [ "programmatic" ], "pattern": { "accepted": [ "Gowikibot" ], "forbidden": [] }, "addition_date": "2017/10/26", "verification": [], "instances": { "accepted": [ "Mozilla/5.0 (compatible; Gowikibot/1.0; +http://www.gowikibot.com)" ], "rejected": [] }, "url": "http://www.gowikibot.com" }, { "id": "pipl-crawler", "categories": [ "search-engine" ], "pattern": { "accepted": [ "PiplBot" ], "forbidden": [] }, "addition_date": "2017/10/30", "verification": [], "instances": { "accepted": [ "PiplBot (+http://www.pipl.com/bot/)", "Mozilla/5.0+(compatible;+PiplBot;+http://www.pipl.com/bot/)" ], "rejected": [] }, "url": "http://www.pipl.com/bot/" }, { "id": "discord-crawler", "categories": [ "preview", "social" ], "pattern": { "accepted": [ "Discordbot" ], "forbidden": [] }, "addition_date": "2017/09/22", "url": "https://discordapp.com", "verification": [], "instances": { "accepted": [ "Mozilla/5.0 (compatible; Discordbot/2.0; +https://discordapp.com)" ], "rejected": [] }, "aliases": [ "DiscordBot" ] }, { "id": "telegram-crawler", "categories": [ "preview", "social" ], "pattern": { "accepted": [ "TelegramBot" ], "forbidden": [] }, "addition_date": "2017/10/01", "verification": [], "instances": { "accepted": [ "TelegramBot (like TwitterBot)" ], "rejected": [] }, "aliases": [ "TelegramBot" ] }, { "id": "jetslide-crawler", "categories": [ "unknown" ], "pattern": { "accepted": [ "Jetslide" ], "forbidden": [] }, "addition_date": "2017/09/27", "url": "http://jetsli.de/crawler", "verification": [], "instances": { "accepted": [ "Mozilla/5.0 (compatible; Jetslide; +http://jetsli.de/crawler)" ], "rejected": [] } }, { "id": "newsharecounts-crawler", "categories": [ "unknown" ], "pattern": { "accepted": [ "newsharecounts" ], "forbidden": [] }, "addition_date": "2017/09/30", "url": "http://newsharecounts.com/crawler", "verification": [], "instances": { "accepted": [ "Mozilla/5.0 (compatible; NewShareCounts.com/1.0; +http://newsharecounts.com/crawler)" ], "rejected": [] } }, { "id": "cognitiveseo-crawler", "categories": [ "unknown" ], "pattern": { "accepted": [ "James BOT" ], "forbidden": [] }, "addition_date": "2017/10/12", "url": "http://cognitiveseo.com/bot.html", "verification": [], "instances": { "accepted": [ "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.8.1.6) Gecko/20070725 Firefox/2.0.0.6 - James BOT - WebCrawler http://cognitiveseo.com/bot.html" ], "rejected": [] } }, { "id": "exensa-crawler", "categories": [ "unknown" ], "pattern": { "accepted": [ "Bark[rR]owler" ], "forbidden": [] }, "addition_date": "2017/10/09", "url": "http://www.exensa.com/crawl", "verification": [], "instances": { "accepted": [ "Barkrowler/0.5.1 (experimenting / debugging - sorry for your logs ) http://www.exensa.com/crawl - admin@exensa.com -- based on BuBiNG", "Barkrowler/0.7 (+http://www.exensa.com/crawl)", "BarkRowler/0.7 (+http://www.exensa.com/crawling)", "Barkrowler/0.9 (+http://www.exensa.com/crawl)" ], "rejected": [] } }, { "id": "tineye-crawler", "categories": [ "unknown" ], "pattern": { "accepted": [ "TinEye" ], "forbidden": [] }, "addition_date": "2017/10/14", "url": "http://www.tineye.com/crawler.html", "verification": [], "instances": { "accepted": [ "Mozilla/5.0 (compatible; TinEye-bot/1.31; +http://www.tineye.com/crawler.html)", "TinEye/1.1 (http://tineye.com/crawler.html)" ], "rejected": [] } }, { "id": "socialrank-crawler", "categories": [ "unknown" ], "pattern": { "accepted": [ "SocialRankIOBot" ], "forbidden": [] }, "addition_date": "2017/10/19", "url": "http://socialrank.io/about", "verification": [], "instances": { "accepted": [ "SocialRankIOBot; http://socialrank.io/about" ], "rejected": [] } }, { "id": "trendiction-crawler", "categories": [ "unknown" ], "pattern": { "accepted": [ "trendictionbot" ], "forbidden": [] }, "addition_date": "2017/10/30", "url": "http://www.trendiction.de/bot", "verification": [], "instances": { "accepted": [ "Mozilla/5.0 (Windows; U; Windows NT 6.0; en-GB; rv:1.0; trendictionbot0.5.0; trendiction search; http://www.trendiction.de/bot; please let us know of any problems; web at trendiction.com) Gecko/20071127 Firefox/3.0.0.11", "Mozilla/5.0 (Windows NT 10.0; Win64; x64; trendictionbot0.5.0; trendiction search; http://www.trendiction.de/bot; please let us know of any problems; web at trendiction.com) Gecko/20170101 Firefox/67.0" ], "rejected": [] } }, { "id": "ocarinabot", "categories": [ "unknown" ], "pattern": { "accepted": [ "Ocarinabot" ], "forbidden": [] }, "addition_date": "2017/09/27", "verification": [], "instances": { "accepted": [ "Ocarinabot" ], "rejected": [] } }, { "id": "epictions-crawler", "categories": [ "unknown" ], "pattern": { "accepted": [ "epicbot" ], "forbidden": [] }, "addition_date": "2017/10/31", "url": "http://www.epictions.com/epicbot", "verification": [], "instances": { "accepted": [ "Mozilla/5.0 (compatible; epicbot; +http://www.epictions.com/epicbot)" ], "rejected": [] } }, { "id": "primal-crawler", "categories": [ "ai" ], "pattern": { "accepted": [ "Primalbot" ], "forbidden": [] }, "addition_date": "2017/09/27", "url": "https://www.primal.com", "verification": [], "instances": { "accepted": [ "Mozilla/5.0 (compatible; Primalbot; +https://www.primal.com;)" ], "rejected": [] } }, { "id": "duckduckgo-crawler-favicons", "categories": [ "preview", "search-engine" ], "pattern": { "accepted": [ "DuckDuckGo-Favicons-Bot" ], "forbidden": [] }, "addition_date": "2017/10/06", "url": "http://duckduckgo.com", "verification": [], "instances": { "accepted": [ "Mozilla/5.0 (compatible; DuckDuckGo-Favicons-Bot/1.0; +http://duckduckgo.com)" ], "rejected": [] } }, { "id": "gnowit-crawler", "categories": [ "unknown" ], "pattern": { "accepted": [ "GnowitNewsbot" ], "forbidden": [] }, "addition_date": "2017/10/30", "url": "http://www.gnowit.com", "verification": [], "instances": { "accepted": [ "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:49.0) Gecko/20100101 Firefox/49.0 / GnowitNewsbot / Contact information at http://www.gnowit.com" ], "rejected": [] } }, { "id": "leiki-crawler", "categories": [ "unknown" ], "pattern": { "accepted": [ "Leikibot" ], "forbidden": [] }, "addition_date": "2017/09/24", "url": "http://www.leiki.com", "verification": [], "instances": { "accepted": [ "Mozilla/5.0 (Windows NT 6.3;compatible; Leikibot/1.0; +http://www.leiki.com)" ], "rejected": [] } }, { "id": "linkarchiver", "categories": [ "archive", "social" ], "pattern": { "accepted": [ "LinkArchiver" ], "forbidden": [] }, "addition_date": "2017/09/24", "url": "https://github.com/thisisparker/linkarchiver", "verification": [], "instances": { "accepted": [ "@LinkArchiver twitter bot" ], "rejected": [] } }, { "id": "linkfluence-crawler", "categories": [ "unknown" ], "pattern": { "accepted": [ "YaK\\/" ], "forbidden": [] }, "addition_date": "2017/09/25", "url": "http://linkfluence.com", "verification": [], "instances": { "accepted": [ "Mozilla/5.0 (compatible; YaK/1.0; http://linkfluence.com/; bot@linkfluence.com)" ], "rejected": [] } }, { "id": "paperli-crawler", "categories": [ "unknown" ], "pattern": { "accepted": [ "PaperLiBot" ], "forbidden": [] }, "addition_date": "2017/09/25", "url": "http://support.paper.li/entries/20023257-what-is-paper-li", "verification": [], "instances": { "accepted": [ "Mozilla/5.0 (compatible; PaperLiBot/2.1; http://support.paper.li/entries/20023257-what-is-paper-li)", "Mozilla/5.0 (compatible; PaperLiBot/2.1; https://support.paper.li/entries/20023257-what-is-paper-li)" ], "rejected": [] } }, { "id": "digg-crawler", "categories": [ "social" ], "pattern": { "accepted": [ "Digg Deeper" ], "forbidden": [] }, "addition_date": "2017/09/26", "url": "http://digg.com/about", "verification": [], "instances": { "accepted": [ "Digg Deeper/v1 (http://digg.com/about)" ], "rejected": [] } }, { "id": "dcrawl", "categories": [ "tool" ], "pattern": { "accepted": [ "dcrawl" ], "forbidden": [] }, "addition_date": "2017/09/22", "url": "https://github.com/kgretzky/dcrawl", "verification": [], "instances": { "accepted": [ "dcrawl/1.0" ], "rejected": [] } }, { "id": "java-snacktory", "categories": [ "programmatic" ], "pattern": { "accepted": [ "Snacktory" ], "forbidden": [] }, "addition_date": "2017/09/23", "url": "https://github.com/karussell/snacktory", "verification": [], "instances": { "accepted": [ "Mozilla/5.0 (compatible; Snacktory; +https://github.com/karussell/snacktory)" ], "rejected": [] } }, { "id": "anderspink-crawler", "categories": [ "unknown" ], "pattern": { "accepted": [ "AndersPinkBot" ], "forbidden": [] }, "addition_date": "2017/09/24", "url": "http://anderspink.com/bot.html", "verification": [], "instances": { "accepted": [ "Mozilla/5.0 (compatible; AndersPinkBot/1.0; +http://anderspink.com/bot.html)" ], "rejected": [] } }, { "id": "fyrebot", "categories": [ "unknown" ], "pattern": { "accepted": [ "Fyrebot" ], "forbidden": [] }, "addition_date": "2017/09/22", "verification": [], "instances": { "accepted": [ "Fyrebot/1.0" ], "rejected": [] } }, { "id": "everyonesocial-crawler", "categories": [ "social" ], "pattern": { "accepted": [ "EveryoneSocialBot" ], "forbidden": [] }, "addition_date": "2017/09/22", "url": "http://everyonesocial.com", "verification": [], "instances": { "accepted": [ "Mozilla/5.0 (compatible; EveryoneSocialBot/1.0; support@everyonesocial.com http://everyonesocial.com/)" ], "rejected": [] } }, { "id": "mediatoolkit-crawler", "categories": [ "ai" ], "pattern": { "accepted": [ "Mediatoolkitbot" ], "forbidden": [] }, "addition_date": "2017/10/06", "url": "http://mediatoolkit.com", "verification": [], "instances": { "accepted": [ "Mediatoolkitbot (complaints@mediatoolkit.com)" ], "rejected": [] } }, { "id": "luminator-crawler", "categories": [ "unknown" ], "pattern": { "accepted": [ "Luminator-robots" ], "forbidden": [] }, "addition_date": "2017/09/22", "verification": [], "instances": { "accepted": [ "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_2) AppleWebKit/537.13 (KHTML, like Gecko) Chrome/30.0.1599.66 Safari/537.13 Luminator-robots/2.0" ], "rejected": [] } }, { "id": "extlinks-crawler", "categories": [ "unknown" ], "pattern": { "accepted": [ "ExtLinksBot" ], "forbidden": [] }, "addition_date": "2017/11/02", "url": "https://extlinks.com/Bot.html", "verification": [], "instances": { "accepted": [ "Mozilla/5.0 (compatible; ExtLinksBot/1.5 +https://extlinks.com/Bot.html)" ], "rejected": [] } }, { "id": "domaintools-crawler", "categories": [ "unknown" ], "pattern": { "accepted": [ "SurveyBot" ], "forbidden": [] }, "addition_date": "2017/11/02", "verification": [], "instances": { "accepted": [ "Mozilla/5.0 (Windows; U; Windows NT 5.1; en; rv:1.9.0.13) Gecko/2009073022 Firefox/3.5.2 (.NET CLR 3.5.30729) SurveyBot/2.3 (DomainTools)" ], "rejected": [] } }, { "id": "ning-crawler", "categories": [ "unknown" ], "pattern": { "accepted": [ "NING\\/" ], "forbidden": [] }, "addition_date": "2017/11/02", "verification": [], "instances": { "accepted": [ "NING/1.0" ], "rejected": [] } }, { "id": "java-okhttp", "categories": [ "programmatic" ], "pattern": { "accepted": [ "okhttp" ], "forbidden": [] }, "addition_date": "2017/11/02", "verification": [], "instances": { "accepted": [ "okhttp/2.5.0", "okhttp/2.7.5", "okhttp/3.2.0", "okhttp/3.5.0", "okhttp/4.1.0" ], "rejected": [] } }, { "id": "nuzzel-crawler", "categories": [ "unknown" ], "pattern": { "accepted": [ "Nuzzel" ], "forbidden": [] }, "addition_date": "2017/11/02", "verification": [], "instances": { "accepted": [ "Nuzzel" ], "rejected": [] } }, { "id": "omgili-crawler", "categories": [ "ai" ], "pattern": { "accepted": [ "omgili" ], "forbidden": [] }, "addition_date": "2017/11/02", "url": "https://webz.io/blog/company/from-omgilibot-to-the-webzbot-duo-a-powerful-leap-for-ethical-and-comprehensive-data-collection/", "verification": [], "instances": { "accepted": [ "omgili/0.5 +http://omgili.com" ], "rejected": [] } }, { "id": "pocket-crawler", "categories": [ "feedfetcher" ], "pattern": { "accepted": [ "PocketParser" ], "forbidden": [] }, "addition_date": "2017/11/02", "url": "https://getpocket.com/pocketparser_ua", "verification": [], "instances": { "accepted": [ "PocketParser/2.0 (+https://getpocket.com/pocketparser_ua)" ], "rejected": [] } }, { "id": "yisou-crawler", "categories": [ "unknown" ], "pattern": { "accepted": [ "YisouSpider" ], "forbidden": [] }, "addition_date": "2017/11/02", "verification": [], "instances": { "accepted": [ "YisouSpider", "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.81 YisouSpider/5.0 Safari/537.36" ], "rejected": [] } }, { "id": "ubermetrics-crawler", "categories": [ "unknown" ], "pattern": { "accepted": [ "um-LN" ], "forbidden": [] }, "addition_date": "2017/11/02", "verification": [], "instances": { "accepted": [ "Mozilla/5.0 (compatible; um-LN/1.0; mailto: techinfo@ubermetrics-technologies.com)" ], "rejected": [] } }, { "id": "toutiao-crawler", "categories": [ "unknown" ], "pattern": { "accepted": [ "ToutiaoSpider" ], "forbidden": [] }, "addition_date": "2017/11/02", "url": "http://web.toutiao.com/media_cooperation/", "verification": [], "instances": { "accepted": [ "Mozilla/5.0 (compatible; ToutiaoSpider/1.0; http://web.toutiao.com/media_cooperation/;)" ], "rejected": [] } }, { "id": "muckrack-crawler", "categories": [ "unknown" ], "pattern": { "accepted": [ "MuckRack" ], "forbidden": [] }, "addition_date": "2017/11/02", "url": "http://muckrack.com", "verification": [], "instances": { "accepted": [ "Mozilla/5.0 (compatible; MuckRack/1.0; +http://muckrack.com)" ], "rejected": [] } }, { "id": "jamiembrown-crawler", "categories": [ "unknown" ], "pattern": { "accepted": [ "Jamie's Spider" ], "forbidden": [] }, "addition_date": "2017/11/02", "url": "http://jamiembrown.com/", "verification": [], "instances": { "accepted": [ "Jamie's Spider (http://jamiembrown.com/)" ], "rejected": [] } }, { "id": "java-asynchttpclient", "categories": [ "programmatic" ], "pattern": { "accepted": [ "AHC\\/" ], "forbidden": [] }, "addition_date": "2017/11/02", "url": "https://github.com/AsyncHttpClient/async-http-client", "verification": [], "instances": { "accepted": [ "AHC/2.0" ], "rejected": [] } }, { "id": "netcraft-crawler", "categories": [ "unknown" ], "pattern": { "accepted": [ "NetcraftSurveyAgent" ], "forbidden": [] }, "addition_date": "2017/11/02", "verification": [], "instances": { "accepted": [ "Mozilla/5.0 (compatible; NetcraftSurveyAgent/1.0; +info@netcraft.com)" ], "rejected": [] } }, { "id": "laserlike-crawler", "categories": [ "unknown" ], "pattern": { "accepted": [ "Laserlikebot" ], "forbidden": [] }, "addition_date": "2017/11/02", "verification": [], "instances": { "accepted": [ "Mozilla/5.0 (iPhone; CPU iPhone OS 8_3 like Mac OS X) AppleWebKit/600.1.4 (KHTML, like Gecko) Version/8.0 Mobile/12F70 Safari/600.1.4 (compatible; Laserlikebot/0.1)" ], "rejected": [] } }, { "id": "java-apache-httpclient", "categories": [ "programmatic" ], "pattern": { "accepted": [ "^Apache-HttpClient" ], "forbidden": [] }, "addition_date": "2017/11/02", "verification": [], "instances": { "accepted": [ "Apache-HttpClient/4.2.3 (java 1.5)", "Apache-HttpClient/4.2.5 (java 1.5)", "Apache-HttpClient/4.3.1 (java 1.5)", "Apache-HttpClient/4.3.3 (java 1.5)", "Apache-HttpClient/4.3.5 (java 1.5)", "Apache-HttpClient/4.4.1 (Java/1.8.0_65)", "Apache-HttpClient/4.5.2 (Java/1.8.0_65)", "Apache-HttpClient/4.5.2 (Java/1.8.0_151)", "Apache-HttpClient/4.5.2 (Java/1.8.0_161)", "Apache-HttpClient/4.5.2 (Java/1.8.0_181)", "Apache-HttpClient/4.5.3 (Java/1.8.0_121)", "Apache-HttpClient/4.5.3-SNAPSHOT (Java/1.8.0_152)", "Apache-HttpClient/4.5.7 (Java/11.0.3)", "Apache-HttpClient/4.5.10 (Java/1.8.0_201)" ], "rejected": [] } }, { "id": "google-appengine", "categories": [ "google", "programmatic" ], "pattern": { "accepted": [ "AppEngine-Google" ], "forbidden": [] }, "addition_date": "2017/11/02", "verification": [], "instances": { "accepted": [ "AppEngine-Google; (+http://code.google.com/appengine; appid: example)", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.181 Safari/537.36 AppEngine-Google; (+http://code.google.com/appengine; appid: s~feedly-nikon3)" ], "rejected": [] } }, { "id": "java-jetty", "categories": [ "programmatic" ], "pattern": { "accepted": [ "Jetty" ], "forbidden": [] }, "addition_date": "2017/11/02", "verification": [], "instances": { "accepted": [ "Jetty/9.3.z-SNAPSHOT" ], "rejected": [] } }, { "id": "upflow-crawler", "categories": [ "unknown" ], "pattern": { "accepted": [ "Upflow" ], "forbidden": [] }, "addition_date": "2017/11/02", "verification": [], "instances": { "accepted": [ "Upflow/1.0" ], "rejected": [] } }, { "id": "thinklab-crawler", "categories": [ "unknown" ], "pattern": { "accepted": [ "Thinklab" ], "forbidden": [] }, "addition_date": "2017/11/02", "url": "https://thinklab.com", "verification": [], "instances": { "accepted": [ "Thinklab (thinklab.com)" ], "rejected": [] } }, { "id": "traackr-crawler", "categories": [ "unknown" ], "pattern": { "accepted": [ "Traackr\\.com" ], "forbidden": [] }, "addition_date": "2017/11/02", "url": "https://www.traackr.com/", "verification": [], "instances": { "accepted": [ "Traackr.com" ], "rejected": [] } }, { "id": "twurly-crawler", "categories": [ "unknown" ], "pattern": { "accepted": [ "Twurly" ], "forbidden": [] }, "addition_date": "2017/11/02", "url": "http://twurly.org", "verification": [], "instances": { "accepted": [ "Ruby, Twurly v1.1 (http://twurly.org)" ], "rejected": [] } }, { "id": "mastodon-crawler", "categories": [ "preview", "social" ], "pattern": { "accepted": [ "Mastodon" ], "forbidden": [] }, "addition_date": "2017/11/02", "verification": [], "instances": { "accepted": [ "http.rb/2.2.2 (Mastodon/1.5.1; +https://example-masto-instance.org/)" ], "rejected": [] }, "aliases": [ "MastodonBot" ] }, { "id": "http-get", "categories": [ "programmatic" ], "pattern": { "accepted": [ "http_get" ], "forbidden": [] }, "addition_date": "2017/11/02", "verification": [], "instances": { "accepted": [ "http_get" ], "rejected": [] } }, { "id": "dnyz-crawler", "categories": [ "unknown" ], "pattern": { "accepted": [ "DnyzBot" ], "forbidden": [] }, "addition_date": "2017/11/20", "verification": [], "instances": { "accepted": [ "Mozilla/5.0 (compatible; DnyzBot/1.0)" ], "rejected": [] } }, { "id": "botify-crawler", "categories": [ "ai" ], "pattern": { "accepted": [ "botify" ], "forbidden": [] }, "addition_date": "2018/02/01", "verification": [], "instances": { "accepted": [ "Mozilla/5.0 (compatible; botify; http://botify.com)" ], "rejected": [] } }, { "id": "sistrix-007ac9-crawler", "categories": [ "unknown" ], "pattern": { "accepted": [ "007ac9 Crawler" ], "forbidden": [] }, "addition_date": "2018/02/09", "verification": [], "instances": { "accepted": [ "Mozilla/5.0 (compatible; 007ac9 Crawler; http://crawler.007ac9.net/)" ], "rejected": [] } }, { "id": "webeaver-crawler", "categories": [ "unknown" ], "pattern": { "accepted": [ "BehloolBot" ], "forbidden": [] }, "addition_date": "2018/02/09", "verification": [], "instances": { "accepted": [ "Mozilla/5.0 (compatible; BehloolBot/beta; +http://www.webeaver.com/bot)" ], "rejected": [] } }, { "id": "brandverity-crawler", "categories": [ "monitor" ], "pattern": { "accepted": [ "BrandVerity" ], "forbidden": [] }, "addition_date": "2018/02/27", "verification": [], "instances": { "accepted": [ "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.10; rv:41.0) Gecko/20100101 Firefox/55.0 BrandVerity/1.0 (http://www.brandverity.com/why-is-brandverity-visiting-me)", "Mozilla/5.0 (iPhone; CPU iPhone OS 7_0 like Mac OS X) AppleWebKit/537.51.1 (KHTML, like Gecko) Mobile/11A465 Twitter for iPhone BrandVerity/1.0 (http://www.brandverity.com/why-is-brandverity-visiting-me)" ], "rejected": [] }, "url": "http://www.brandverity.com/why-is-brandverity-visiting-me" }, { "id": "nagios-check-http", "categories": [ "tool" ], "pattern": { "accepted": [ "check_http" ], "forbidden": [] }, "addition_date": "2018/02/09", "verification": [], "instances": { "accepted": [ "check_http/v2.2.1 (nagios-plugins 2.2.1)" ], "rejected": [] } }, { "id": "bigdatacorp-crawler", "categories": [ "unknown" ], "pattern": { "accepted": [ "BDCbot" ], "forbidden": [] }, "addition_date": "2018/02/09", "verification": [], "instances": { "accepted": [ "Mozilla/5.0 (Windows NT 6.1; compatible; BDCbot/1.0; +http://bigweb.bigdatacorp.com.br/faq.aspx) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2272.118 Safari/537.36", "Mozilla/5.0 (Windows NT 10.0; Win64; x64; BDCbot/1.0; +http://bigweb.bigdatacorp.com.br/faq.aspx) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.100 Safari/537.36" ], "rejected": [] } }, { "id": "zum-crawler", "categories": [ "unknown" ], "pattern": { "accepted": [ "ZumBot" ], "forbidden": [] }, "addition_date": "2018/02/09", "verification": [], "instances": { "accepted": [ "Mozilla/5.0 (compatible; ZumBot/1.0; http://help.zum.com/inquiry)" ], "rejected": [] }, "aliases": [ "ZumBot" ] }, { "id": "ezid-crawler", "categories": [ "academic", "archive" ], "pattern": { "accepted": [ "EZID" ], "forbidden": [] }, "addition_date": "2018/02/09", "verification": [], "instances": { "accepted": [ "EZID (EZID link checker; https://ezid.cdlib.org/)" ], "rejected": [] } }, { "id": "nict-crawler", "categories": [ "academic", "ai" ], "pattern": { "accepted": [ "ICC-Crawler" ], "forbidden": [] }, "addition_date": "2018/02/28", "verification": [], "instances": { "accepted": [ "ICC-Crawler/2.0 (Mozilla-compatible; ; http://ucri.nict.go.jp/en/icccrawler.html)" ], "rejected": [] }, "url": "http://ucri.nict.go.jp/en/icccrawler.html" }, { "id": "irc-archivebot", "categories": [ "archive", "social" ], "pattern": { "accepted": [ "ArchiveBot" ], "forbidden": [] }, "addition_date": "2018/02/28", "verification": [], "instances": { "accepted": [ "ArchiveTeam ArchiveBot/20170106.02 (wpull 2.0.2)" ], "rejected": [] }, "url": "https://github.com/ArchiveTeam/ArchiveBot", "aliases": [ "ArchiveBot" ] }, { "id": "leipzig-lcc", "categories": [ "academic" ], "pattern": { "accepted": [ "^LCC " ], "forbidden": [] }, "addition_date": "2018/02/28", "verification": [], "instances": { "accepted": [ "LCC (+http://corpora.informatik.uni-leipzig.de/crawler_faq.html)" ], "rejected": [] }, "url": "http://corpora.informatik.uni-leipzig.de/crawler_faq.html" }, { "id": "iss-crawler", "categories": [ "unknown" ], "pattern": { "accepted": [ "filterdb\\.iss\\.net\\/crawler" ], "forbidden": [] }, "addition_date": "2018/03/16", "verification": [], "instances": { "accepted": [ "Mozilla/5.0 (compatible; oBot/2.3.1; +http://filterdb.iss.net/crawler/)" ], "rejected": [] }, "url": "http://filterdb.iss.net/crawler/" }, { "id": "blp-bbot", "categories": [ "unknown" ], "pattern": { "accepted": [ "BLP_bbot" ], "forbidden": [] }, "addition_date": "2018/03/27", "verification": [], "instances": { "accepted": [ "BLP_bbot/0.1" ], "rejected": [] } }, { "id": "bombora-crawler", "categories": [ "unknown" ], "pattern": { "accepted": [ "BomboraBot" ], "forbidden": [] }, "addition_date": "2018/03/27", "verification": [], "instances": { "accepted": [ "Mozilla/5.0 (compatible; BomboraBot/1.0; +http://www.bombora.com/bot)" ], "rejected": [] }, "url": "http://www.bombora.com/bot" }, { "id": "hypefactors-crawler", "categories": [ "unknown" ], "pattern": { "accepted": [ "Buck\\/" ], "forbidden": [] }, "addition_date": "2018/03/27", "verification": [], "instances": { "accepted": [ "Buck/2.2; (+https://app.hypefactors.com/media-monitoring/about.html)" ], "rejected": [] }, "url": "https://app.hypefactors.com/media-monitoring/about.html" }, { "id": "companybook-crawler", "categories": [ "unknown" ], "pattern": { "accepted": [ "Companybook-Crawler" ], "forbidden": [] }, "addition_date": "2018/03/27", "verification": [], "instances": { "accepted": [ "Companybook-Crawler (+https://www.companybooknetworking.com/)" ], "rejected": [] }, "url": "https://www.companybooknetworking.com/" }, { "id": "genieo-crawler", "categories": [ "unknown" ], "pattern": { "accepted": [ "Genieo" ], "forbidden": [] }, "addition_date": "2018/03/27", "verification": [], "instances": { "accepted": [ "Mozilla/5.0 (compatible; Genieo/1.0 http://www.genieo.com/webfilter.html)" ], "rejected": [] }, "url": "http://www.genieo.com/webfilter.html" }, { "id": "brandwatch-crawler", "categories": [ "unknown" ], "pattern": { "accepted": [ "magpie-crawler" ], "forbidden": [] }, "addition_date": "2018/03/27", "verification": [], "instances": { "accepted": [ "magpie-crawler/1.1 (U; Linux amd64; en-GB; +http://www.brandwatch.net)" ], "rejected": [] }, "url": "http://www.brandwatch.net" }, { "id": "meltwater-crawler", "categories": [ "unknown" ], "pattern": { "accepted": [ "MeltwaterNews" ], "forbidden": [] }, "addition_date": "2018/03/27", "verification": [], "instances": { "accepted": [ "MeltwaterNews www.meltwater.com" ], "rejected": [] }, "url": "http://www.meltwater.com" }, { "id": "moreover-crawler", "categories": [ "unknown" ], "pattern": { "accepted": [ "Moreover" ], "forbidden": [] }, "addition_date": "2018/03/27", "verification": [], "instances": { "accepted": [ "Mozilla/5.0 Moreover/5.1 (+http://www.moreover.com)" ], "rejected": [] }, "url": "http://www.moreover.com" }, { "id": "newspaper", "categories": [ "unknown" ], "pattern": { "accepted": [ "newspaper\\/" ], "forbidden": [] }, "addition_date": "2018/03/27", "verification": [], "instances": { "accepted": [ "newspaper/0.1.0.7", "newspaper/0.2.5", "newspaper/0.2.6", "newspaper/0.2.8" ], "rejected": [] } }, { "id": "scoutjet-crawler", "categories": [ "unknown" ], "pattern": { "accepted": [ "ScoutJet" ], "forbidden": [] }, "addition_date": "2018/03/27", "verification": [], "instances": { "accepted": [ "Mozilla/5.0 (compatible; ScoutJet; +http://www.scoutjet.com/)" ], "rejected": [] }, "url": "http://www.scoutjet.com/" }, { "id": "sentry-crawler", "categories": [ "monitor" ], "pattern": { "accepted": [ "(^| )sentry\\/" ], "forbidden": [] }, "addition_date": "2018/03/27", "verification": [], "instances": { "accepted": [ "sentry/8.22.0 (https://sentry.io)" ], "rejected": [] }, "url": "https://sentry.io" }, { "id": "storygize-crawler", "categories": [ "ai" ], "pattern": { "accepted": [ "StorygizeBot" ], "forbidden": [] }, "addition_date": "2018/03/27", "verification": [], "instances": { "accepted": [ "Mozilla/5.0 (compatible; StorygizeBot; http://www.storygize.com)" ], "rejected": [] }, "url": "http://www.storygize.com" }, { "id": "uptimerobot-monitor", "categories": [ "monitor" ], "pattern": { "accepted": [ "UptimeRobot" ], "forbidden": [] }, "addition_date": "2018/03/27", "verification": [ { "type": "dns", "masks": [ "ip.uptimerobot.com" ] } ], "instances": { "accepted": [ "Mozilla/5.0+(compatible; UptimeRobot/2.0; http://www.uptimerobot.com/)" ], "rejected": [] }, "url": "https://uptimerobot.com/help/locations/", "aliases": [ "UptimeRobot" ] }, { "id": "outclicks-crawler", "categories": [ "unknown" ], "pattern": { "accepted": [ "OutclicksBot" ], "forbidden": [] }, "addition_date": "2018/04/21", "verification": [], "instances": { "accepted": [ "OutclicksBot/2 +https://www.outclicks.net/agent/VjzDygCuk4ubNmg40ZMbFqT0sIh7UfOKk8s8ZMiupUR", "OutclicksBot/2 +https://www.outclicks.net/agent/gIYbZ38dfAuhZkrFVl7sJBFOUhOVct6J1SvxgmBZgCe", "OutclicksBot/2 +https://www.outclicks.net/agent/PryJzTl8POCRHfvEUlRN5FKtZoWDQOBEvFJ2wh6KH5J", "OutclicksBot/2 +https://www.outclicks.net/agent/p2i4sNUh7eylJF1S6SGgRs5mP40ExlYvsr9GBxVQG6h" ], "rejected": [] }, "url": "https://www.outclicks.net" }, { "id": "seoscanners-crawler", "categories": [ "unknown" ], "pattern": { "accepted": [ "seoscanners" ], "forbidden": [] }, "addition_date": "2018/05/27", "verification": [], "instances": { "accepted": [ "Mozilla/5.0 (compatible; seoscanners.net/1; +spider@seoscanners.net)" ], "rejected": [] }, "url": "http://www.seoscanners.net/" }, { "id": "hatena-crawler", "categories": [ "unknown" ], "pattern": { "accepted": [ "Hatena" ], "forbidden": [] }, "addition_date": "2018/05/29", "verification": [], "instances": { "accepted": [ "Hatena Antenna/0.3", "Hatena::Russia::Crawler/0.01", "Hatena-Favicon/2 (http://www.hatena.ne.jp/faq/)", "Hatena::Scissors/0.01", "HatenaBookmark/4.0 (Hatena::Bookmark; Analyzer)", "Hatena::Fetcher/0.01 (master) Furl/3.13" ], "rejected": [] } }, { "id": "google-preview", "categories": [ "google", "preview", "social" ], "pattern": { "accepted": [ "Google Web Preview" ], "forbidden": [] }, "addition_date": "2018/05/31", "verification": [], "instances": { "accepted": [ "Mozilla/5.0 (Linux; U; Android 2.3.4; generic) AppleWebKit/537.36 (KHTML, like Gecko; Google Web Preview) Version/4.0 Mobile Safari/537.36", "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko; Google Web Preview) Chrome/27.0.1453 Safari/537.36" ], "rejected": [] } }, { "id": "mauibot", "categories": [ "unknown" ], "pattern": { "accepted": [ "MauiBot" ], "forbidden": [] }, "addition_date": "2018/06/06", "verification": [], "instances": { "accepted": [ "MauiBot (crawler.feedback+wc@gmail.com)" ], "rejected": [] } }, { "id": "alphaseobot-crawler", "categories": [ "unknown" ], "pattern": { "accepted": [ "AlphaBot" ], "forbidden": [] }, "addition_date": "2018/05/27", "verification": [], "instances": { "accepted": [ "Mozilla/5.0 (compatible; AlphaBot/3.2; +http://alphaseobot.com/bot.html)" ], "rejected": [] }, "url": "http://alphaseobot.com/bot.html" }, { "id": "softbytelabs-crawler", "categories": [ "unknown" ], "pattern": { "accepted": [ "SBL-BOT" ], "forbidden": [] }, "addition_date": "2018/06/06", "verification": [], "instances": { "accepted": [ "SBL-BOT (http://sbl.net)" ], "rejected": [] }, "url": "http://sbl.net", "description": "Bot of SoftByte BlackWidow" }, { "id": "integralads-crawler", "categories": [ "ai" ], "pattern": { "accepted": [ "IAS crawler" ], "forbidden": [] }, "addition_date": "2018/06/06", "verification": [], "instances": { "accepted": [ "IAS crawler (ias_crawler; http://integralads.com/site-indexing-policy/)" ], "rejected": [] }, "url": "http://integralads.com/site-indexing-policy/", "description": "Bot of Integral Ad Science, Inc." }, { "id": "adscanner-crawler", "categories": [ "unknown" ], "pattern": { "accepted": [ "adscanner" ], "forbidden": [] }, "addition_date": "2018/06/24", "verification": [], "instances": { "accepted": [ "Mozilla/5.0 (compatible; adscanner/)" ], "rejected": [] } }, { "id": "netvibes-crawler", "categories": [ "unknown" ], "pattern": { "accepted": [ "Netvibes" ], "forbidden": [] }, "addition_date": "2018/06/24", "verification": [], "instances": { "accepted": [ "Netvibes (crawler/bot; http://www.netvibes.com", "Netvibes (crawler; http://www.netvibes.com)" ], "rejected": [] }, "url": "http://www.netvibes.com" }, { "id": "acapbot", "categories": [ "unknown" ], "pattern": { "accepted": [ "acapbot" ], "forbidden": [] }, "addition_date": "2018/06/27", "verification": [], "instances": { "accepted": [ "Mozilla/5.0 (compatible;acapbot/0.1;treat like Googlebot)", "Mozilla/5.0 (compatible;acapbot/0.1.;treat like Googlebot)" ], "rejected": [] } }, { "id": "baidu-cloud-watch", "categories": [ "unknown" ], "pattern": { "accepted": [ "Baidu-YunGuanCe" ], "forbidden": [] }, "addition_date": "2018/06/27", "verification": [], "instances": { "accepted": [ "Baidu-YunGuanCe-Bot(ce.baidu.com)", "Baidu-YunGuanCe-SLABot(ce.baidu.com)", "Baidu-YunGuanCe-ScanBot(ce.baidu.com)", "Baidu-YunGuanCe-PerfBot(ce.baidu.com)", "Baidu-YunGuanCe-VSBot(ce.baidu.com)" ], "rejected": [] }, "url": "https://ce.baidu.com/topic/topic20150908", "description": "Baidu Cloud Watch" }, { "id": "bitly-crawler", "categories": [ "preview" ], "pattern": { "accepted": [ "bitlybot" ], "forbidden": [] }, "addition_date": "2018/06/27", "verification": [], "instances": { "accepted": [ "bitlybot/3.0 (+http://bit.ly/)", "bitlybot/2.0", "bitlybot" ], "rejected": [] }, "url": "http://bit.ly/" }, { "id": "blogmura-crawler", "categories": [ "unknown" ], "pattern": { "accepted": [ "blogmuraBot" ], "forbidden": [] }, "addition_date": "2018/06/27", "verification": [], "instances": { "accepted": [ "blogmuraBot (+http://www.blogmura.com)" ], "rejected": [] }, "url": "http://www.blogmura.com", "description": "A blog ranking site which links to blogs on just about every theme possible." }, { "id": "araturka-crawler", "categories": [ "unknown" ], "pattern": { "accepted": [ "Bot\\.AraTurka\\.com" ], "forbidden": [] }, "addition_date": "2018/06/27", "verification": [], "instances": { "accepted": [ "Bot.AraTurka.com/0.0.1" ], "rejected": [] }, "url": "http://www.araturka.com" }, { "id": "chlooe-crawler", "categories": [ "unknown" ], "pattern": { "accepted": [ "bot-pge\\.chlooe\\.com" ], "forbidden": [] }, "addition_date": "2018/06/27", "verification": [], "instances": { "accepted": [ "bot-pge.chlooe.com/1.0.0 (+http://www.chlooe.com/)" ], "rejected": [] } }, { "id": "boxcar-crawler", "categories": [ "unknown" ], "pattern": { "accepted": [ "BoxcarBot" ], "forbidden": [] }, "addition_date": "2018/06/27", "verification": [], "instances": { "accepted": [ "Mozilla/5.0 (compatible; BoxcarBot/1.1; +awesome@boxcar.io)" ], "rejected": [] }, "url": "https://boxcar.io/" }, { "id": "utorrent-crawler", "categories": [ "unknown" ], "pattern": { "accepted": [ "BTWebClient" ], "forbidden": [] }, "addition_date": "2018/06/27", "verification": [], "instances": { "accepted": [ "BTWebClient/180B(9704)" ], "rejected": [] }, "url": "http://www.utorrent.com/", "description": "µTorrent BitTorrent Client" }, { "id": "contextad-crawler", "categories": [ "unknown" ], "pattern": { "accepted": [ "ContextAd Bot" ], "forbidden": [] }, "addition_date": "2018/06/27", "verification": [], "instances": { "accepted": [ "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.0;.NET CLR 1.0.3705; ContextAd Bot 1.0)", "ContextAd Bot 1.0" ], "rejected": [] } }, { "id": "digincore-crawler", "categories": [ "unknown" ], "pattern": { "accepted": [ "Digincore bot" ], "forbidden": [] }, "addition_date": "2018/06/27", "verification": [], "instances": { "accepted": [ "Mozilla/5.0 (compatible; Digincore bot; https://www.digincore.com/crawler.html for rules and instructions.)" ], "rejected": [] }, "url": "http://www.digincore.com/crawler.html" }, { "id": "disqus-crawler", "categories": [ "monitor" ], "pattern": { "accepted": [ "Disqus" ], "forbidden": [] }, "addition_date": "2018/06/27", "verification": [], "instances": { "accepted": [ "Disqus/1.0" ], "rejected": [] }, "url": "https://disqus.com/", "description": "validate and quality check pages." }, { "id": "feedly-feedfetcher", "categories": [ "feedfetcher" ], "pattern": { "accepted": [ "Feedly" ], "forbidden": [] }, "addition_date": "2018/06/27", "verification": [], "instances": { "accepted": [ "Feedly/1.0 (+http://www.feedly.com/fetcher.html; like FeedFetcher-Google)", "FeedlyBot/1.0 (http://feedly.com)" ], "rejected": [] }, "url": "https://www.feedly.com/fetcher.html", "description": "Feedly Fetcher is how Feedly grabs RSS or Atom feeds when users choose to add them to their Feedly or any of the other applications built on top of the feedly cloud." }, { "id": "scritch-crawler", "categories": [ "unknown" ], "pattern": { "accepted": [ "Fetch\\/" ], "forbidden": [] }, "addition_date": "2018/06/27", "verification": [], "instances": { "accepted": [ "Fetch/2.0a (CMS Detection/Web/SEO analysis tool, see http://guess.scritch.org)" ], "rejected": [] } }, { "id": "feedafever-crawler", "categories": [ "unknown" ], "pattern": { "accepted": [ "Fever" ], "forbidden": [] }, "addition_date": "2018/06/27", "verification": [], "instances": { "accepted": [ "Fever/1.38 (Feed Parser; http://feedafever.com; Allow like Gecko)" ], "rejected": [] }, "url": "http://feedafever.com" }, { "id": "flamingosearch-crawler", "categories": [ "unknown" ], "pattern": { "accepted": [ "Flamingo_SearchEngine" ], "forbidden": [] }, "addition_date": "2018/06/27", "verification": [], "instances": { "accepted": [ "Flamingo_SearchEngine (+http://www.flamingosearch.com/bot)" ], "rejected": [] } }, { "id": "flipboard-proxy", "categories": [ "monitor", "preview" ], "pattern": { "accepted": [ "FlipboardProxy" ], "forbidden": [] }, "addition_date": "2018/06/27", "verification": [], "instances": { "accepted": [ "Mozilla/5.0 (compatible; FlipboardProxy/1.1; +http://flipboard.com/browserproxy)", "Mozilla/5.0 (compatible; FlipboardProxy/1.2; +http://flipboard.com/browserproxy)", "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10.6; en-US; rv:1.9.2) Gecko/20100115 Firefox/3.6 (FlipboardProxy/1.1; +http://flipboard.com/browserproxy)", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.9; rv:28.0) Gecko/20100101 Firefox/28.0 (FlipboardProxy/1.1; +http://flipboard.com/browserproxy)", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.11; rv:49.0) Gecko/20100101 Firefox/49.0 (FlipboardProxy/1.2; +http://flipboard.com/browserproxy)" ], "rejected": [] }, "url": "https://about.flipboard.com/browserproxy/", "description": "a proxy service to fetch, validate, and prepare certain elements of websites for presentation through the Flipboard Application" }, { "id": "g2reader-crawler", "categories": [ "feedfetcher" ], "pattern": { "accepted": [ "g2reader-bot" ], "forbidden": [] }, "addition_date": "2018/06/27", "verification": [], "instances": { "accepted": [ "g2reader-bot/1.0 (+http://www.g2reader.com/)" ], "rejected": [] }, "url": "http://www.g2reader.com/" }, { "id": "g2webservices-crawler", "categories": [ "unknown" ], "pattern": { "accepted": [ "G2 Web Services" ], "forbidden": [] }, "addition_date": "2019/03/01", "verification": [], "instances": { "accepted": [ "G2 Web Services/1.0 (built with StormCrawler Archetype 1.8; https://www.g2webservices.com/; developers@g2llc.com)" ], "rejected": [] }, "url": "https://www.g2webservices.com/" }, { "id": "mignify-imrbot", "categories": [ "unknown" ], "pattern": { "accepted": [ "imrbot" ], "forbidden": [] }, "addition_date": "2018/06/27", "verification": [], "instances": { "accepted": [ "Mozilla/5.0 (compatible; imrbot/1.10.8 +http://www.mignify.com)" ], "rejected": [] }, "url": "http://www.mignify.com" }, { "id": "k7computing-crawler", "categories": [ "unknown" ], "pattern": { "accepted": [ "K7MLWCBot" ], "forbidden": [] }, "addition_date": "2018/06/27", "verification": [], "instances": { "accepted": [ "K7MLWCBot/1.0 (+http://www.k7computing.com)" ], "rejected": [] }, "url": "http://www.k7computing.com", "description": "Virus scanner" }, { "id": "kemvi-crawler", "categories": [ "unknown" ], "pattern": { "accepted": [ "Kemvibot" ], "forbidden": [] }, "addition_date": "2018/06/27", "verification": [], "instances": { "accepted": [ "Kemvibot/1.0 (http://kemvi.com, marco@kemvi.com)" ], "rejected": [] }, "url": "http://kemvi.com" }, { "id": "landaumedia-crawler", "categories": [ "unknown" ], "pattern": { "accepted": [ "Landau-Media-Spider" ], "forbidden": [] }, "addition_date": "2018/06/27", "verification": [], "instances": { "accepted": [ "Landau-Media-Spider/1.0(http://bots.landaumedia.de/bot.html)" ], "rejected": [] }, "url": "http://bots.landaumedia.de/bot.html" }, { "id": "linkapedia-crawler", "categories": [ "unknown" ], "pattern": { "accepted": [ "linkapediabot" ], "forbidden": [] }, "addition_date": "2018/06/27", "verification": [], "instances": { "accepted": [ "linkapediabot (+http://www.linkapedia.com)" ], "rejected": [] }, "url": "http://www.linkapedia.com" }, { "id": "vkshare-crawler", "categories": [ "unknown" ], "pattern": { "accepted": [ "vkShare" ], "forbidden": [] }, "addition_date": "2018/07/02", "verification": [], "instances": { "accepted": [ "Mozilla/5.0 (compatible; vkShare; +http://vk.com/dev/Share)" ], "rejected": [] }, "url": "http://vk.com/dev/Share" }, { "id": "siteimprove-crawler", "categories": [ "unknown" ], "pattern": { "accepted": [ "Siteimprove\\.com" ], "forbidden": [] }, "addition_date": "2018/06/22", "verification": [], "instances": { "accepted": [ "Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; Trident/6.0) LinkCheck by Siteimprove.com", "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.0) Match by Siteimprove.com", "Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; Trident/6.0) SiteCheck-sitecrawl by Siteimprove.com", "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.0) LinkCheck by Siteimprove.com" ], "rejected": [] } }, { "id": "webmeup-crawler", "categories": [ "unknown" ], "pattern": { "accepted": [ "BLEXBot\\/" ], "forbidden": [] }, "addition_date": "2018/07/07", "verification": [], "instances": { "accepted": [ "Mozilla/5.0 (compatible; BLEXBot/1.0; +http://webmeup-crawler.com/)" ], "rejected": [] }, "url": "http://webmeup-crawler.com" }, { "id": "dareboost-crawler", "categories": [ "optimizer" ], "pattern": { "accepted": [ "DareBoost" ], "forbidden": [] }, "addition_date": "2018/07/07", "verification": [], "instances": { "accepted": [ "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/62.0.3202.75 Safari/537.36 DareBoost" ], "rejected": [] }, "url": "https://www.dareboost.com/", "description": "Bot to test, Analyze and Optimize website" }, { "id": "zuperlist-crawler", "categories": [ "unknown" ], "pattern": { "accepted": [ "ZuperlistBot\\/" ], "forbidden": [] }, "addition_date": "2018/07/07", "verification": [], "instances": { "accepted": [ "Mozilla/5.0 (compatible; ZuperlistBot/1.0)" ], "rejected": [] } }, { "id": "miniflux-feedfetcher", "categories": [ "feedfetcher" ], "pattern": { "accepted": [ "Miniflux\\/" ], "forbidden": [] }, "addition_date": "2018/07/07", "verification": [], "instances": { "accepted": [ "Mozilla/5.0 (compatible; Miniflux/2.0.x-dev; +https://miniflux.net)", "Mozilla/5.0 (compatible; Miniflux/2.0.3; +https://miniflux.net)", "Mozilla/5.0 (compatible; Miniflux/2.0.7; +https://miniflux.net)", "Mozilla/5.0 (compatible; Miniflux/2.0.10; +https://miniflux.net)", "Mozilla/5.0 (compatibl$; Miniflux/2.0.x-dev; +https://miniflux.app)", "Mozilla/5.0 (compatible; Miniflux/2.0.11; +https://miniflux.app)", "Mozilla/5.0 (compatible; Miniflux/2.0.12; +https://miniflux.app)", "Mozilla/5.0 (compatible; Miniflux/ae1dc1a; +https://miniflux.app)", "Mozilla/5.0 (compatible; Miniflux/3b6e44c; +https://miniflux.app)" ], "rejected": [] }, "url": "https://miniflux.net", "description": "Miniflux is a minimalist and opinionated feed reader." }, { "id": "feedspot-feedfetcher", "categories": [ "feedfetcher" ], "pattern": { "accepted": [ "Feedspot" ], "forbidden": [] }, "addition_date": "2018/07/07", "verification": [], "instances": { "accepted": [ "Mozilla/5.0 (compatible; Feedspotbot/1.0; +http://www.feedspot.com/fs/bot)", "Mozilla/5.0 (compatible; Feedspot/1.0 (+https://www.feedspot.com/fs/fetcher; like FeedFetcher-Google)" ], "rejected": [] }, "url": "http://www.feedspot.com/fs/bot" }, { "id": "diffbot-crawler", "categories": [ "ai" ], "pattern": { "accepted": [ "Diffbot\\/" ], "forbidden": [] }, "addition_date": "2018/07/07", "verification": [], "instances": { "accepted": [ "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.1.2) Gecko/20090729 Firefox/3.5.2 (.NET CLR 3.5.30729; Diffbot/0.1; +http://www.diffbot.com)" ], "rejected": [] }, "url": "http://www.diffbot.com" }, { "id": "seokicks-crawler", "categories": [ "unknown" ], "pattern": { "accepted": [ "SEOkicks" ], "forbidden": [] }, "addition_date": "2018/08/22", "verification": [], "instances": { "accepted": [ "Mozilla/5.0 (compatible; SEOkicks; +https://www.seokicks.de/robot.html)" ], "rejected": [] }, "url": "https://www.seokicks.de/robot.html" }, { "id": "tracemyfile-crawler", "categories": [ "unknown" ], "pattern": { "accepted": [ "tracemyfile" ], "forbidden": [] }, "addition_date": "2018/08/23", "verification": [], "instances": { "accepted": [ "Mozilla/5.0 (compatible; tracemyfile/1.0; +bot@tracemyfile.com)" ], "rejected": [] } }, { "id": "cloudsystemnetworks-crawler", "categories": [ "unknown" ], "pattern": { "accepted": [ "Nimbostratus-Bot" ], "forbidden": [] }, "addition_date": "2018/08/29", "verification": [], "instances": { "accepted": [ "Mozilla/5.0 (compatible; Nimbostratus-Bot/v1.3.2; http://cloudsystemnetworks.com)" ], "rejected": [] } }, { "id": "zgrab", "categories": [ "tool" ], "pattern": { "accepted": [ "zgrab" ], "forbidden": [] }, "addition_date": "2018/08/30", "verification": [], "instances": { "accepted": [ "Mozilla/5.0 zgrab/0.x" ], "rejected": [] }, "url": "https://github.com/zmap/zgrab2" }, { "id": "prcy-crawler", "categories": [ "unknown" ], "pattern": { "accepted": [ "PR-CY\\.RU" ], "forbidden": [] }, "addition_date": "2018/08/30", "verification": [], "instances": { "accepted": [ "Mozilla/5.0 (compatible; PR-CY.RU; + https://a.pr-cy.ru)" ], "rejected": [] }, "url": "https://a.pr-cy.ru/" }, { "id": "adstxtcrawler", "categories": [ "unknown" ], "pattern": { "accepted": [ "AdsTxtCrawler" ], "forbidden": [] }, "addition_date": "2018/08/30", "verification": [], "instances": { "accepted": [ "AdsTxtCrawler/1.0" ], "rejected": [] } }, { "id": "datafeedwatch-crawler", "categories": [ "unknown" ], "pattern": { "accepted": [ "Datafeedwatch" ], "forbidden": [] }, "addition_date": "2018/09/05", "verification": [], "instances": { "accepted": [ "Datafeedwatch/2.1.x" ], "rejected": [] }, "url": "https://www.datafeedwatch.com/" }, { "id": "zabbix-monitor", "categories": [ "monitor" ], "pattern": { "accepted": [ "Zabbix" ], "forbidden": [] }, "addition_date": "2018/09/05", "verification": [], "instances": { "accepted": [ "Zabbix" ], "rejected": [] }, "url": "https://www.zabbix.com/documentation/3.4/manual/web_monitoring" }, { "id": "tangiblee-crawler", "categories": [ "unknown" ], "pattern": { "accepted": [ "TangibleeBot" ], "forbidden": [] }, "addition_date": "2018/09/05", "verification": [], "instances": { "accepted": [ "TangibleeBot/1.0.0.0 (http://tangiblee.com/bot)" ], "rejected": [] }, "url": "http://tangiblee.com/bot" }, { "id": "google-xrawler", "categories": [ "google" ], "pattern": { "accepted": [ "google-xrawler" ], "forbidden": [] }, "addition_date": "2018/09/05", "verification": [], "instances": { "accepted": [ "google-xrawler" ], "rejected": [] }, "url": "https://webmasters.stackexchange.com/questions/105560/what-is-the-google-xrawler-user-agent-used-for" }, { "id": "javascript-axios", "categories": [ "programmatic" ], "pattern": { "accepted": [ "axios" ], "forbidden": [] }, "addition_date": "2018/09/06", "verification": [], "instances": { "accepted": [ "axios/0.18.0", "axios/0.19.0" ], "rejected": [] }, "url": "https://github.com/axios/axios" }, { "id": "amazon-cloudfront", "categories": [ "amazon" ], "pattern": { "accepted": [ "Amazon CloudFront" ], "forbidden": [] }, "addition_date": "2018/09/07", "verification": [], "instances": { "accepted": [ "Amazon CloudFront" ], "rejected": [] }, "url": "https://aws.amazon.com/cloudfront/" }, { "id": "pulsepoint-crawler", "categories": [ "unknown" ], "pattern": { "accepted": [ "Pulsepoint" ], "forbidden": [] }, "addition_date": "2018/09/24", "verification": [], "instances": { "accepted": [ "Pulsepoint XT3 web scraper" ], "rejected": [] } }, { "id": "cloudflare-archiver", "categories": [ "archive" ], "pattern": { "accepted": [ "CloudFlare-AlwaysOnline" ], "forbidden": [] }, "addition_date": "2018/09/27", "verification": [], "instances": { "accepted": [ "Mozilla/5.0 (compatible; CloudFlare-AlwaysOnline/1.0; +http://www.cloudflare.com/always-online) AppleWebKit/534.34", "Mozilla/5.0 (compatible; CloudFlare-AlwaysOnline/1.0; +https://www.cloudflare.com/always-online) AppleWebKit/534.34" ], "rejected": [] }, "url": "https://www.cloudflare.com/always-online/" }, { "id": "google-structured-data-testing-tool", "categories": [ "google", "monitor", "optimizer" ], "pattern": { "accepted": [ "Google-Structured-Data-Testing-Tool" ], "forbidden": [] }, "addition_date": "2018/10/02", "verification": [], "instances": { "accepted": [ "Mozilla/5.0 (compatible; Google-Structured-Data-Testing-Tool +https://search.google.com/structured-data/testing-tool)", "Mozilla/5.0 (compatible; Google-Structured-Data-Testing-Tool +http://developers.google.com/structured-data/testing-tool/)" ], "rejected": [] }, "url": "https://search.google.com/structured-data/testing-tool" }, { "id": "wordup-crawler", "categories": [ "unknown" ], "pattern": { "accepted": [ "WordupInfoSearch" ], "forbidden": [] }, "addition_date": "2018/10/07", "verification": [], "instances": { "accepted": [ "WordupInfoSearch/1.0" ], "rejected": [] } }, { "id": "webdatastats-crawler", "categories": [ "unknown" ], "pattern": { "accepted": [ "WebDataStats" ], "forbidden": [] }, "addition_date": "2018/10/08", "verification": [], "instances": { "accepted": [ "Mozilla/5.0 (compatible; WebDataStats/1.0 ; +https://webdatastats.com/policy.html)" ], "rejected": [] }, "url": "https://webdatastats.com/" }, { "id": "java-jersey", "categories": [ "programmatic" ], "pattern": { "accepted": [ "HttpUrlConnection" ], "forbidden": [] }, "addition_date": "2018/10/08", "verification": [], "instances": { "accepted": [ "Jersey/2.25.1 (HttpUrlConnection 1.8.0_141)" ], "rejected": [] } }, { "id": "seozoom-crawler", "categories": [ "unknown" ], "pattern": { "accepted": [ "ZoomBot" ], "forbidden": [] }, "addition_date": "2018/10/10", "verification": [], "instances": { "accepted": [ "ZoomBot (Linkbot 1.0 http://suite.seozoom.it/bot.html)" ], "rejected": [] }, "url": "http://suite.seozoom.it/bot.html", "aliases": [ "ZoomBot" ] }, { "id": "velen-crawler", "categories": [ "ai" ], "pattern": { "accepted": [ "VelenPublicWebCrawler" ], "forbidden": [] }, "addition_date": "2018/10/09", "url": "https://velen.io/", "verification": [], "instances": { "accepted": [ "VelenPublicWebCrawler (velen.io)" ], "rejected": [] } }, { "id": "moodle-crawler", "categories": [ "unknown" ], "pattern": { "accepted": [ "MoodleBot" ], "forbidden": [] }, "addition_date": "2018/10/10", "verification": [], "instances": { "accepted": [ "MoodleBot/1.0" ], "rejected": [] } }, { "id": "vipnytt-crawler", "categories": [ "unknown" ], "pattern": { "accepted": [ "jpg-newsbot" ], "forbidden": [] }, "addition_date": "2018/10/10", "verification": [], "instances": { "accepted": [ "jpg-newsbot/2.0; (+https://vipnytt.no/bots/)" ], "rejected": [] }, "url": "https://vipnytt.no/bots/" }, { "id": "outbrain-link-checker", "categories": [ "monitor" ], "pattern": { "accepted": [ "outbrain" ], "forbidden": [] }, "addition_date": "2018/10/14", "verification": [], "instances": { "accepted": [ "Mozilla/5.0 (Java) outbrain" ], "rejected": [] }, "url": "https://www.outbrain.com/help/advertisers/invalid-url/" }, { "id": "w3c-validator-html", "categories": [ "monitor" ], "pattern": { "accepted": [ "W3C_Validator" ], "forbidden": [] }, "addition_date": "2018/10/14", "verification": [], "instances": { "accepted": [ "W3C_Validator/1.3" ], "rejected": [] }, "url": "https://validator.w3.org/services" }, { "id": "w3c-validator-html-nu", "categories": [ "monitor" ], "pattern": { "accepted": [ "Validator\\.nu" ], "forbidden": [] }, "addition_date": "2018/10/14", "verification": [], "instances": { "accepted": [ "Validator.nu/LV" ], "rejected": [] }, "url": "https://validator.w3.org/services" }, { "id": "w3c-validator-links", "categories": [ "monitor" ], "pattern": { "accepted": [ "W3C-checklink" ], "forbidden": [] }, "addition_date": "2018/10/14", "depends_on": [ "libwww-perl" ], "verification": [], "instances": { "accepted": [ "W3C-checklink/2.90 libwww-perl/5.64", "W3C-checklink/3.6.2.3 libwww-perl/5.64", "W3C-checklink/4.2 [4.20] libwww-perl/5.803", "W3C-checklink/4.2.1 [4.21] libwww-perl/5.803", "W3C-checklink/4.3 [4.42] libwww-perl/5.805", "W3C-checklink/4.3 [4.42] libwww-perl/5.808", "W3C-checklink/4.3 [4.42] libwww-perl/5.820", "W3C-checklink/4.5 [4.154] libwww-perl/5.823", "W3C-checklink/4.5 [4.160] libwww-perl/5.823" ], "rejected": [] }, "url": "https://validator.w3.org/services" }, { "id": "w3c-validator-mobile", "categories": [ "monitor" ], "pattern": { "accepted": [ "W3C-mobileOK" ], "forbidden": [] }, "addition_date": "2018/10/14", "verification": [], "instances": { "accepted": [ "W3C-mobileOK/DDC-1.0" ], "rejected": [] }, "url": "https://validator.w3.org/services" }, { "id": "w3c-validator-i18n", "categories": [ "monitor" ], "pattern": { "accepted": [ "W3C_I18n-Checker" ], "forbidden": [] }, "addition_date": "2018/10/14", "verification": [], "instances": { "accepted": [ "W3C_I18n-Checker/1.0" ], "rejected": [] }, "url": "https://validator.w3.org/services" }, { "id": "w3c-validator-feed", "categories": [ "monitor" ], "pattern": { "accepted": [ "FeedValidator" ], "forbidden": [] }, "addition_date": "2018/10/14", "verification": [], "instances": { "accepted": [ "FeedValidator/1.3" ], "rejected": [] }, "url": "https://validator.w3.org/services" }, { "id": "w3c-validator-css", "categories": [ "monitor" ], "pattern": { "accepted": [ "W3C_CSS_Validator" ], "forbidden": [] }, "addition_date": "2018/10/14", "verification": [], "instances": { "accepted": [ "Jigsaw/2.3.0 W3C_CSS_Validator_JFouffa/2.0" ], "rejected": [] }, "url": "https://validator.w3.org/services" }, { "id": "w3c-validator-unified", "categories": [ "monitor" ], "pattern": { "accepted": [ "W3C_Unicorn" ], "forbidden": [] }, "addition_date": "2018/10/14", "verification": [], "instances": { "accepted": [ "W3C_Unicorn/1.0" ], "rejected": [] }, "url": "https://validator.w3.org/services" }, { "id": "google-physical-web", "categories": [ "google" ], "pattern": { "accepted": [ "Google-PhysicalWeb" ], "forbidden": [] }, "addition_date": "2018/10/21", "verification": [], "instances": { "accepted": [ "Mozilla/5.0 (Google-PhysicalWeb)" ], "rejected": [] } }, { "id": "blackboard-crawler", "categories": [ "academic" ], "pattern": { "accepted": [ "Blackboard" ], "forbidden": [] }, "addition_date": "2018/10/28", "verification": [], "instances": { "accepted": [ "Blackboard Safeassign" ], "rejected": [] }, "url": "https://help.blackboard.com/Learn/Administrator/Hosting/Tools_Management/SafeAssign" }, { "id": "ideasandcode-crawler", "categories": [ "unknown" ], "pattern": { "accepted": [ "ICBot\\/" ], "forbidden": [] }, "addition_date": "2018/10/23", "verification": [], "instances": { "accepted": [ "Mozilla/5.0 (compatible; ICBot/0.1; +https://ideasandcode.xyz" ], "rejected": [] }, "url": "https://ideasandcode.xyz" }, { "id": "bazqux-feedfetcher", "categories": [ "feedfetcher" ], "pattern": { "accepted": [ "BazQux" ], "forbidden": [] }, "addition_date": "2018/10/23", "verification": [], "instances": { "accepted": [ "Mozilla/5.0 (compatible; BazQux/2.4; +https://bazqux.com/fetcher; 1 subscribers)" ], "rejected": [] }, "url": "https://bazqux.com/fetcher" }, { "id": "twingly-crawler", "categories": [ "unknown" ], "pattern": { "accepted": [ "Twingly" ], "forbidden": [] }, "addition_date": "2018/10/23", "verification": [], "instances": { "accepted": [ "Mozilla/5.0 (compatible; Twingly Recon; twingly.com)" ], "rejected": [] }, "url": "https://twingly.com" }, { "id": "rivva-crawler", "categories": [ "unknown" ], "pattern": { "accepted": [ "Rivva" ], "forbidden": [] }, "addition_date": "2018/10/23", "verification": [], "instances": { "accepted": [ "Mozilla/5.0 (compatible; Rivva; http://rivva.de)" ], "rejected": [] }, "url": "http://rivva.de" }, { "id": "experibot-crawler", "categories": [ "unknown" ], "pattern": { "accepted": [ "Experibot" ], "forbidden": [] }, "addition_date": "2018/11/03", "verification": [], "instances": { "accepted": [ "Experibot-v2 http://goo.gl/ZAr8wX", "Experibot-v3 http://goo.gl/ZAr8wX" ], "rejected": [] }, "url": "https://amirkr.wixsite.com/experibot" }, { "id": "awesomecrawler", "categories": [ "unknown" ], "pattern": { "accepted": [ "awesomecrawler" ], "forbidden": [] }, "addition_date": "2018/11/24", "verification": [], "instances": { "accepted": [ "Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.22 (KHTML, like Gecko) Chrome/25.0.1364.5 Safari/537.22 +awesomecrawler" ], "rejected": [] } }, { "id": "dataprovider-crawler", "categories": [ "unknown" ], "pattern": { "accepted": [ "Dataprovider\\.com" ], "forbidden": [] }, "addition_date": "2018/11/24", "verification": [], "instances": { "accepted": [ "Mozilla/5.0 (compatible; Dataprovider.com)" ], "rejected": [] }, "url": "https://www.dataprovider.com/" }, { "id": "grouphigh-crawler", "categories": [ "unknown" ], "pattern": { "accepted": [ "GroupHigh\\/" ], "forbidden": [] }, "addition_date": "2018/11/24", "verification": [], "instances": { "accepted": [ "Mozilla/5.0 (compatible; GroupHigh/1.0; +http://www.grouphigh.com/" ], "rejected": [] }, "url": "http://www.grouphigh.com/" }, { "id": "theoldreader-crawler", "categories": [ "feedfetcher" ], "pattern": { "accepted": [ "theoldreader\\.com" ], "forbidden": [] }, "addition_date": "2018/12/02", "verification": [], "instances": { "accepted": [ "Mozilla/5.0 (compatible; theoldreader.com)" ], "rejected": [] }, "url": "https://www.theoldreader.com/" }, { "id": "schmorp-crawler", "categories": [ "unknown" ], "pattern": { "accepted": [ "AnyEvent" ], "forbidden": [] }, "addition_date": "2018/12/07", "verification": [], "instances": { "accepted": [ "Mozilla/5.0 (compatible; U; AnyEvent-HTTP/2.24; +http://software.schmorp.de/pkg/AnyEvent)" ], "rejected": [] }, "url": "http://software.schmorp.de/pkg/AnyEvent.html" }, { "id": "uptimebot-monitor", "categories": [ "monitor" ], "pattern": { "accepted": [ "Uptimebot\\.org" ], "forbidden": [] }, "addition_date": "2019/01/17", "verification": [], "instances": { "accepted": [ "Uptimebot.org - Free website monitoring" ], "rejected": [] }, "url": "http://uptimebot.org/" }, { "id": "nmap", "categories": [ "tool" ], "pattern": { "accepted": [ "Nmap Scripting Engine" ], "forbidden": [] }, "addition_date": "2019/02/04", "verification": [], "instances": { "accepted": [ "Mozilla/5.0 (compatible; Nmap Scripting Engine; https://nmap.org/book/nse.html)" ], "rejected": [] }, "url": "https://nmap.org/book/nse.html" }, { "id": "twoip-crawler-cms", "categories": [ "unknown" ], "pattern": { "accepted": [ "2ip\\.ru" ], "forbidden": [] }, "addition_date": "2019/02/12", "verification": [], "instances": { "accepted": [ "2ip.ru CMS Detector (https://2ip.ru/cms/)" ], "rejected": [] }, "url": "https://2ip.ru/cms/" }, { "id": "clickagy-crawler", "categories": [ "unknown" ], "pattern": { "accepted": [ "Clickagy" ], "forbidden": [] }, "addition_date": "2019/02/19", "verification": [], "instances": { "accepted": [ "Clickagy Intelligence Bot v2" ], "rejected": [] }, "url": "https://www.clickagy.com" }, { "id": "conductor-crawler", "categories": [ "optimizer" ], "pattern": { "accepted": [ "Caliperbot" ], "forbidden": [] }, "addition_date": "2019/03/02", "verification": [], "instances": { "accepted": [ "Caliperbot/1.0 (+http://www.conductor.com/caliperbot)" ], "rejected": [] }, "url": "http://www.conductor.com/caliperbot" }, { "id": "monitorbacklinks-crawler", "categories": [ "monitor" ], "pattern": { "accepted": [ "MBCrawler" ], "forbidden": [] }, "addition_date": "2019/03/02", "verification": [], "instances": { "accepted": [ "MBCrawler/1.0 (https://monitorbacklinks.com)" ], "rejected": [] }, "url": "https://monitorbacklinks.com" }, { "id": "webceo-crawler", "categories": [ "unknown" ], "pattern": { "accepted": [ "online-webceo-bot" ], "forbidden": [] }, "addition_date": "2019/03/02", "verification": [], "instances": { "accepted": [ "Mozilla/5.0 (compatible; online-webceo-bot/1.0; +http://online.webceo.com)" ], "rejected": [] }, "url": "http://online.webceo.com" }, { "id": "b2bbot", "categories": [ "unknown" ], "pattern": { "accepted": [ "B2B Bot" ], "forbidden": [] }, "addition_date": "2019/03/02", "verification": [], "instances": { "accepted": [ "B2B Bot" ], "rejected": [] } }, { "id": "addsearch-crawler", "categories": [ "search-engine" ], "pattern": { "accepted": [ "AddSearchBot" ], "forbidden": [] }, "addition_date": "2019/03/02", "verification": [], "instances": { "accepted": [ "Mozilla/5.0 (compatible; AddSearchBot/0.9; +http://www.addsearch.com/bot; info@addsearch.com)" ], "rejected": [] }, "url": "http://www.addsearch.com/bot" }, { "id": "google-favicon", "categories": [ "google", "preview" ], "pattern": { "accepted": [ "Google Favicon" ], "forbidden": [] }, "addition_date": "2019/03/14", "verification": [], "instances": { "accepted": [ "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/49.0.2623.75 Safari/537.36 Google Favicon" ], "rejected": [] } }, { "id": "hubspot-crawler", "categories": [ "monitor" ], "pattern": { "accepted": [ "HubSpot" ], "forbidden": [] }, "addition_date": "2019/04/15", "verification": [], "instances": { "accepted": [ "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/27.0.1453.116 Safari/537.36 HubSpot Webcrawler - web-crawlers@hubspot.com", "Mozilla/5.0 (X11; Linux x86_64; HubSpot Single Page link check; web-crawlers+links@hubspot.com) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.103 Safari/537.36", "Mozilla/5.0 (compatible; HubSpot Crawler; web-crawlers@hubspot.com)", "HubSpot Connect 2.0 (http://dev.hubspot.com/) - BizOpsCompanies-Tq2-BizCoDomainValidationAudit" ], "rejected": [] }, "aliases": [ "HubSpotBot" ] }, { "id": "google-lighthouse", "categories": [ "google", "optimizer" ], "pattern": { "accepted": [ "Chrome-Lighthouse" ], "forbidden": [] }, "addition_date": "2019/03/15", "verification": [], "instances": { "accepted": [ "Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5 Build/MRA58N) AppleWebKit/537.36(KHTML, like Gecko) Chrome/69.0.3464.0 Mobile Safari/537.36 Chrome-Lighthouse", "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36(KHTML, like Gecko) Chrome/69.0.3464.0 Safari/537.36 Chrome-Lighthouse", "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3694.0 Safari/537.36 Chrome-Lighthouse", "Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5 Build/MRA58N) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3694.0 Mobile Safari/537.36 Chrome-Lighthouse" ], "rejected": [] }, "url": "https://developers.google.com/speed/pagespeed/insights" }, { "id": "headless-chrome", "categories": [ "tool" ], "pattern": { "accepted": [ "HeadlessChrome" ], "forbidden": [] }, "url": "https://developers.google.com/web/updates/2017/04/headless-chrome", "addition_date": "2019/06/17", "verification": [], "instances": { "accepted": [ "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) HeadlessChrome/74.0.3729.169 Safari/537.36", "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) HeadlessChrome/69.0.3494.0 Safari/537.36", "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) HeadlessChrome/76.0.3803.0 Safari/537.36" ], "rejected": [] } }, { "id": "checkmarknetwork-crawler", "categories": [ "unknown" ], "pattern": { "accepted": [ "CheckMarkNetwork\\/" ], "forbidden": [] }, "addition_date": "2019/06/30", "verification": [], "instances": { "accepted": [ "CheckMarkNetwork/1.0 (+http://www.checkmarknetwork.com/spider.html)" ], "rejected": [] }, "url": "https://www.checkmarknetwork.com/" }, { "id": "uptime-monitor", "categories": [ "monitor" ], "pattern": { "accepted": [ "www\\.uptime\\.com" ], "forbidden": [] }, "addition_date": "2019/07/21", "verification": [], "instances": { "accepted": [ "Mozilla/5.0 (compatible; Uptimebot/1.0; +http://www.uptime.com/uptimebot)" ], "rejected": [] }, "url": "http://www.uptime.com/uptimebot" }, { "id": "ubt-crawler", "categories": [ "unknown" ], "pattern": { "accepted": [ "Streamline3Bot\\/" ], "forbidden": [] }, "addition_date": "2019/07/21", "verification": [], "instances": { "accepted": [ "Mozilla/5.0 (compatible; MSIE 8.0; Windows NT 5.1) Streamline3Bot/1.0", "Mozilla/5.0 (Windows NT 6.1; Win64; x64; +https://www.ubtsupport.com/legal/Streamline3Bot.php) Streamline3Bot/1.0" ], "rejected": [] }, "url": "https://www.ubtsupport.com/legal/Streamline3Bot.php" }, { "id": "serpstatbot-crawler", "categories": [ "unknown" ], "pattern": { "accepted": [ "serpstatbot\\/" ], "forbidden": [] }, "addition_date": "2019/07/25", "verification": [], "instances": { "accepted": [ "serpstatbot/1.0 (advanced backlink tracking bot; http://serpstatbot.com/; abuse@serpstatbot.com)", "serpstatbot/1.0 (advanced backlink tracking bot; curl/7.58.0; http://serpstatbot.com/; abuse@serpstatbot.com)" ], "rejected": [] }, "url": "http://serpstatbot.com" }, { "id": "mixnode-cache", "categories": [ "unknown" ], "pattern": { "accepted": [ "MixnodeCache\\/" ], "forbidden": [] }, "addition_date": "2019/08/04", "verification": [], "instances": { "accepted": [ "MixnodeCache/1.8(+https://cache.mixnode.com/)" ], "rejected": [] }, "url": "https://cache.mixnode.com/" }, { "id": "curl", "categories": [ "tool" ], "pattern": { "accepted": [ "^curl" ], "forbidden": [] }, "addition_date": "2019/08/15", "verification": [], "instances": { "accepted": [ "curl", "curl/7.29.0", "curl/7.47.0", "curl/7.54.0", "curl/7.55.1", "curl/7.64.0", "curl/7.64.1", "curl/7.65.3" ], "rejected": [] }, "url": "https://curl.haxx.se/" }, { "id": "php-simple-scraper", "categories": [ "programmatic" ], "pattern": { "accepted": [ "SimpleScraper" ], "forbidden": [] }, "addition_date": "2019/08/16", "verification": [], "instances": { "accepted": [ "Mozilla/5.0 (compatible; SimpleScraper)" ], "rejected": [] }, "url": "https://github.com/ramonkcom/simple-scraper/" }, { "id": "rssing-crawler", "categories": [ "feedfetcher" ], "pattern": { "accepted": [ "RSSingBot" ], "forbidden": [] }, "addition_date": "2019/09/15", "verification": [], "instances": { "accepted": [ "RSSingBot (http://www.rssing.com)" ], "rejected": [] }, "url": "http://www.rssing.com" }, { "id": "jooble-crawler", "categories": [ "unknown" ], "pattern": { "accepted": [ "Jooblebot" ], "forbidden": [] }, "addition_date": "2019/09/25", "verification": [], "instances": { "accepted": [ "Mozilla/5.0 (compatible; Jooblebot/2.0; Windows NT 6.1; WOW64; +http://jooble.org/jooble-bot) AppleWebKit/537.36 (KHTML, like Gecko) Safari/537.36" ], "rejected": [] }, "url": "http://jooble.org/jooble-bot" }, { "id": "fedoraplanet-crawler", "categories": [ "unknown" ], "pattern": { "accepted": [ "fedoraplanet" ], "forbidden": [] }, "addition_date": "2019/09/28", "verification": [], "instances": { "accepted": [ "venus/fedoraplanet" ], "rejected": [] }, "url": "http://fedoraplanet.org/" }, { "id": "hoyer-crawler", "categories": [ "unknown" ], "pattern": { "accepted": [ "Friendica" ], "forbidden": [] }, "addition_date": "2019/09/28", "verification": [], "instances": { "accepted": [ "Friendica 'The Tazmans Flax-lily' 2019.01-1293; https://hoyer.xyz" ], "rejected": [] }, "url": "https://hoyer.xyz" }, { "id": "nextcloud-crawler", "categories": [ "unknown" ], "pattern": { "accepted": [ "NextCloud" ], "forbidden": [] }, "addition_date": "2019/09/30", "verification": [], "instances": { "accepted": [ "NextCloud-News/1.0" ], "rejected": [] }, "url": "https://nextcloud.com/", "aliases": [ "NextCloudBot" ] }, { "id": "ttrss-feedfetcher", "categories": [ "feedfetcher" ], "pattern": { "accepted": [ "Tiny Tiny RSS" ], "forbidden": [] }, "addition_date": "2019/10/04", "verification": [], "instances": { "accepted": [ "Tiny Tiny RSS/1.15.3 (http://tt-rss.org/)", "Tiny Tiny RSS/17.12 (a2d1fa5) (http://tt-rss.org/)", "Tiny Tiny RSS/19.2 (b68db2d) (http://tt-rss.org/)", "Tiny Tiny RSS/19.8 (http://tt-rss.org/)" ], "rejected": [] }, "url": "http://tt-rss.org/" }, { "id": "stuttgart-crawler", "categories": [ "unknown" ], "pattern": { "accepted": [ "RegionStuttgartBot" ], "forbidden": [] }, "addition_date": "2019/10/17", "verification": [], "instances": { "accepted": [ "Mozilla/5.0 (compatible; RegionStuttgartBot/1.0; +http://it.region-stuttgart.de/competenzatlas/unternehmen-suchen/)" ], "rejected": [] }, "url": "http://it.region-stuttgart.de/competenzatlas/unternehmen-suchen/" }, { "id": "bytedance-crawler", "categories": [ "ai" ], "pattern": { "accepted": [ "Bytespider" ], "forbidden": [] }, "addition_date": "2019/11/11", "verification": [], "instances": { "accepted": [ "Mozilla/5.0 (iPhone; CPU iPhone OS 11_0 like Mac OS X) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/40.0.3754.1902 Mobile Safari/537.36; Bytespider", "Mozilla/5.0 (iPhone; CPU iPhone OS 11_0 like Mac OS X) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.4454.1745 Mobile Safari/537.36; Bytespider", "Mozilla/5.0 (iPhone; CPU iPhone OS 11_0 like Mac OS X) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/43.0.7597.1164 Mobile Safari/537.36; Bytespider;bytespider@bytedance.com", "Mozilla/5.0 (iPhone; CPU iPhone OS 11_0 like Mac OS X) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/46.0.2988.1545 Mobile Safari/537.36; Bytespider", "Mozilla/5.0 (iPhone; CPU iPhone OS 11_0 like Mac OS X) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.4141.1682 Mobile Safari/537.36; Bytespider", "Mozilla/5.0 (iPhone; CPU iPhone OS 11_0 like Mac OS X) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.3478.1649 Mobile Safari/537.36; Bytespider", "Mozilla/5.0 (Linux; Android 5.0; SM-G900P Build/LRX21T) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/47.0.5267.1259 Mobile Safari/537.36; Bytespider", "Mozilla/5.0 (Linux; Android 5.0; SM-G900P Build/LRX21T) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.7990.1979 Mobile Safari/537.36; Bytespider", "Mozilla/5.0 (Linux; Android 5.0; SM-G900P Build/LRX21T) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.2268.1523 Mobile Safari/537.36; Bytespider", "Mozilla/5.0 (Linux; Android 6.0; Nexus 5 Build/MRA58N) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/40.0.2576.1836 Mobile Safari/537.36; Bytespider", "Mozilla/5.0 (Linux; Android 6.0; Nexus 5 Build/MRA58N) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/43.0.9681.1227 Mobile Safari/537.36; Bytespider", "Mozilla/5.0 (Linux; Android 6.0; Nexus 5 Build/MRA58N) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/44.0.6023.1635 Mobile Safari/537.36; Bytespider", "Mozilla/5.0 (Linux; Android 6.0; Nexus 5 Build/MRA58N) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/50.0.4944.1981 Mobile Safari/537.36; Bytespider", "Mozilla/5.0 (Linux; Android 6.0; Nexus 5 Build/MRA58N) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.3613.1739 Mobile Safari/537.36; Bytespider", "Mozilla/5.0 (Linux; Android 6.0; Nexus 5 Build/MRA58N) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.4022.1033 Mobile Safari/537.36; Bytespider", "Mozilla/5.0 (Linux; Android 6.0; Nexus 5 Build/MRA58N) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/52.0.3248.1547 Mobile Safari/537.36; Bytespider", "Mozilla/5.0 (Linux; Android 6.0; Nexus 5 Build/MRA58N) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/57.0.5527.1507 Mobile Safari/537.36; Bytespider", "Mozilla/5.0 (Linux; Android 8.0; Pixel 2 Build/OPD3.170816.012) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.5216.1326 Mobile Safari/537.36; Bytespider", "Mozilla/5.0 (Linux; Android 8.0; Pixel 2 Build/OPD3.170816.012) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.9038.1080 Mobile Safari/537.36; Bytespider" ], "rejected": [] }, "url": "https://stackoverflow.com/questions/57908900/what-is-the-bytespider-user-agent", "aliases": [ "ByteSpider" ] }, { "id": "datanyze-crawler", "categories": [ "unknown" ], "pattern": { "accepted": [ "Datanyze" ], "forbidden": [] }, "addition_date": "2019/11/17", "verification": [], "instances": { "accepted": [ "Mozilla/5.0 (X11; Datanyze; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.181 Safari/537.36" ], "rejected": [] }, "url": "https://www.datanyze.com/dnyzbot/" }, { "id": "google-site-verification", "categories": [ "google", "monitor" ], "pattern": { "accepted": [ "Google-Site-Verification" ], "forbidden": [] }, "addition_date": "2019/12/11", "verification": [ { "type": "cidr", "sources": [ { "type": "http-json", "url": "https://developers.google.com/static/search/apis/ipranges/user-triggered-fetchers.json", "selector": "$.prefixes[*][\\\"ipv6Prefix\\\",\\\"ipv4Prefix\\\"]" }, { "type": "http-json", "url": "https://developers.google.com/static/search/apis/ipranges/user-triggered-fetchers-google.json", "selector": "$.prefixes[*][\\\"ipv6Prefix\\\",\\\"ipv4Prefix\\\"]" } ] }, { "type": "dns", "masks": [ "***-***-***-***.gae.googleusercontent.com", "google-proxy-***-***-***-***.google.com" ] } ], "instances": { "accepted": [ "Mozilla/5.0 (compatible; Google-Site-Verification/1.0)" ], "rejected": [] }, "url": "https://support.google.com/webmasters/answer/9008080" }, { "id": "trendsmap-crawler", "categories": [ "archive", "social" ], "pattern": { "accepted": [ "TrendsmapResolver" ], "forbidden": [] }, "addition_date": "2020/02/24", "verification": [], "instances": { "accepted": [ "Mozilla/5.0 (compatible; TrendsmapResolver/0.1)" ], "rejected": [] }, "url": "https://www.trendsmap.com/" }, { "id": "tweetedtimes-crawler", "categories": [ "social" ], "pattern": { "accepted": [ "tweetedtimes" ], "forbidden": [] }, "addition_date": "2020/02/24", "verification": [], "instances": { "accepted": [ "Mozilla/5.0 (compatible; +http://tweetedtimes.com)" ], "rejected": [] }, "url": "https://tweetedtimes.com/" }, { "id": "ntent-crawler", "categories": [ "ai" ], "pattern": { "accepted": [ "NTENTbot" ], "forbidden": [] }, "addition_date": "2020/02/24", "verification": [], "instances": { "accepted": [ "Mozilla/5.0 (compatible; NTENTbot; +http://www.ntent.com/ntentbot)" ], "rejected": [] }, "url": "https://www.seekr.com/" }, { "id": "gwene-crawler", "categories": [ "unknown" ], "pattern": { "accepted": [ "Gwene" ], "forbidden": [] }, "addition_date": "2020/02/24", "verification": [], "instances": { "accepted": [ "Gwene/1.0 (The gwene.org rss-to-news gateway) Googlebot" ], "rejected": [] }, "url": "https://gwene.org" }, { "id": "php-simplepie", "categories": [ "programmatic" ], "pattern": { "accepted": [ "SimplePie" ], "forbidden": [] }, "addition_date": "2020/02/24", "verification": [], "instances": { "accepted": [ "SimplePie/1.3-dev (Feed Parser; http://simplepie.org; Allow like Gecko)" ], "rejected": [] }, "url": "http://simplepie.org" }, { "id": "searchatlas-crawler", "categories": [ "ai" ], "pattern": { "accepted": [ "SearchAtlas" ], "forbidden": [] }, "addition_date": "2020/03/02", "verification": [], "instances": { "accepted": [ "SearchAtlas.com SEO Crawler" ], "rejected": [] }, "url": "http://SearchAtlas.com" }, { "id": "superfeedr-crawler", "categories": [ "feedfetcher" ], "pattern": { "accepted": [ "Superfeedr" ], "forbidden": [] }, "addition_date": "2020/03/02", "verification": [], "instances": { "accepted": [ "Superfeedr bot/2.0 http://superfeedr.com - Make your feeds realtime: get in touch - feed-id:1162088860" ], "rejected": [] }, "url": "http://superfeedr.com" }, { "id": "wordpress-crawler-rss", "categories": [ "feedfetcher" ], "pattern": { "accepted": [ "feedbot" ], "forbidden": [] }, "addition_date": "2020/03/02", "verification": [], "instances": { "accepted": [ "wp.com feedbot/1.0 (+https://wp.com)" ], "rejected": [] }, "url": "http://wp.com" }, { "id": "utexas-crawler", "categories": [ "academic" ], "pattern": { "accepted": [ "UT-Dorkbot" ], "forbidden": [] }, "addition_date": "2020/03/02", "verification": [], "instances": { "accepted": [ "UT-Dorkbot/1.0" ], "rejected": [] }, "url": "https://security.utexas.edu/dorkbot" }, { "id": "amazon-crawler", "categories": [ "amazon" ], "pattern": { "accepted": [ "Amazonbot" ], "forbidden": [] }, "addition_date": "2020/03/02", "verification": [], "instances": { "accepted": [ "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/600.2.5 (KHTML, like Gecko) Version/8.0.2 Safari/600.2.5 (Amazonbot/0.1; +https://developer.amazon.com/support/amazonbot)" ], "rejected": [] }, "url": "https://developer.amazon.com/amazonbot", "aliases": [ "AmazonBot" ] }, { "id": "serendeputy-crawler", "categories": [ "feedfetcher" ], "pattern": { "accepted": [ "SerendeputyBot" ], "forbidden": [] }, "addition_date": "2020/03/02", "verification": [], "instances": { "accepted": [ "SerendeputyBot/0.8.6 (http://serendeputy.com/about/serendeputy-bot)" ], "rejected": [] }, "url": "http://serendeputy.com/about/serendeputy-bot" }, { "id": "eyeota-crawler", "categories": [ "unknown" ], "pattern": { "accepted": [ "Eyeotabot" ], "forbidden": [] }, "addition_date": "2020/03/02", "verification": [], "instances": { "accepted": [ "Mozilla/5.0 (compatible; Eyeotabot/1.0; +http://www.eyeota.com)" ], "rejected": [] }, "url": "http://www.eyeota.com" }, { "id": "bing-office-store", "categories": [ "microsoft", "search-engine" ], "pattern": { "accepted": [ "officestorebot" ], "forbidden": [] }, "addition_date": "2020/03/02", "verification": [], "instances": { "accepted": [ "Mozilla/5.0 (compatible; officestorebot/1.0; +https://aka.ms/officestorebot)" ], "rejected": [] }, "url": "https://aka.ms/officestorebot" }, { "id": "neticle-crawler", "categories": [ "social" ], "pattern": { "accepted": [ "Neticle Crawler" ], "forbidden": [] }, "addition_date": "2020/03/02", "verification": [], "instances": { "accepted": [ "Neticle Crawler v1.0 ( https://neticle.com/bot/en/ )" ], "rejected": [] }, "url": "https://neticle.com/bot/en/" }, { "id": "surly-crawler", "categories": [ "monitor" ], "pattern": { "accepted": [ "SurdotlyBot" ], "forbidden": [] }, "addition_date": "2020/03/02", "verification": [], "instances": { "accepted": [ "Mozilla/5.0 (compatible; SurdotlyBot/1.0; +http://sur.ly/bot.html; Linux; Android 4; iPhone; CPU iPhone OS 6_0_1 like Mac OS X)" ], "rejected": [] }, "url": "http://sur.ly/bot.html" }, { "id": "linkis-crawler", "categories": [ "unknown" ], "pattern": { "accepted": [ "LinkisBot" ], "forbidden": [] }, "addition_date": "2020/03/02", "verification": [], "instances": { "accepted": [ "Mozilla/5.0 (compatible; LinkisBot/1.0; bot@linkis.com) (iPhone; CPU iPhone OS 8_4_1 like Mac OS X) Mobile/12H321" ], "rejected": [] } }, { "id": "awario-crawler-smart", "categories": [ "unknown" ], "pattern": { "accepted": [ "AwarioSmartBot" ], "forbidden": [] }, "addition_date": "2020/03/02", "verification": [], "instances": { "accepted": [ "AwarioSmartBot/1.0 (+https://awario.com/bots.html; bots@awario.com)" ], "rejected": [] }, "url": "https://awario.com/bots.html" }, { "id": "awario-crawler-rss", "categories": [ "feedfetcher" ], "pattern": { "accepted": [ "AwarioRssBot" ], "forbidden": [] }, "addition_date": "2020/03/02", "verification": [], "instances": { "accepted": [ "AwarioRssBot/1.0 (+https://awario.com/bots.html; bots@awario.com)" ], "rejected": [] }, "url": "https://awario.com/bots.html" }, { "id": "ryte-crawler", "categories": [ "unknown" ], "pattern": { "accepted": [ "RyteBot" ], "forbidden": [] }, "addition_date": "2020/03/02", "verification": [], "instances": { "accepted": [ "RyteBot/1.0.0 (+https://bot.ryte.com/)" ], "rejected": [] }, "url": "https://bot.ryte.com/" }, { "id": "freewebmonitoring-monitor", "categories": [ "monitor" ], "pattern": { "accepted": [ "FreeWebMonitoring SiteChecker" ], "forbidden": [] }, "addition_date": "2020/03/02", "verification": [], "instances": { "accepted": [ "FreeWebMonitoring SiteChecker/0.2 (+https://www.freewebmonitoring.com/bot.html)" ], "rejected": [] }, "url": "https://www.freewebmonitoring.com/bot.html" }, { "id": "aspiegel-crawler", "categories": [ "unknown" ], "pattern": { "accepted": [ "AspiegelBot" ], "forbidden": [] }, "addition_date": "2020/03/16", "verification": [], "instances": { "accepted": [ "Mozilla/5.0 (Linux; Android 7.0;) AppleWebKit/537.36 (KHTML, like Gecko) Mobile Safari/537.36 (compatible; AspiegelBot)" ], "rejected": [] }, "url": "https://aspiegel.com" }, { "id": "naver-crawler-rss", "categories": [ "feedfetcher" ], "pattern": { "accepted": [ "NAVER Blog Rssbot" ], "forbidden": [] }, "addition_date": "2020/03/16", "verification": [], "instances": { "accepted": [ "NAVER Blog Rssbot" ], "rejected": [] }, "url": "http://www.naver.com" }, { "id": "logly-crawler", "categories": [ "unknown" ], "pattern": { "accepted": [ "zenback bot" ], "forbidden": [] }, "addition_date": "2020/03/16", "verification": [], "instances": { "accepted": [ "Mozilla/5.0 (compatible; zenback bot; powered by logly +http://corp.logly.co.jp/)" ], "rejected": [] }, "url": "http://corp.logly.co.jp/" }, { "id": "sentione-crawler", "categories": [ "ai" ], "pattern": { "accepted": [ "SentiBot" ], "forbidden": [] }, "addition_date": "2020/03/16", "verification": [], "instances": { "accepted": [ "SentiBot www.sentibot.eu (compatible with Googlebot)" ], "rejected": [] }, "url": "https://sites.google.com/senti1.com/sentibot-eu/home" }, { "id": "domainsproject-crawler", "categories": [ "academic", "tool" ], "pattern": { "accepted": [ "Domains Project\\/" ], "forbidden": [] }, "addition_date": "2020/03/16", "verification": [], "instances": { "accepted": [ "Mozilla/5.0 (compatible; Domains Project/1.0.3; +https://github.com/tb0hdan/domains)" ], "rejected": [] }, "url": "https://github.com/tb0hdan/domains" }, { "id": "domainsbot-crawler", "categories": [ "unknown" ], "pattern": { "accepted": [ "Pandalytics" ], "forbidden": [] }, "addition_date": "2020/03/16", "verification": [], "instances": { "accepted": [ "Pandalytics/1.0 (https://domainsbot.com/pandalytics/)" ], "rejected": [] }, "url": "https://domainsbot.com/pandalytics/" }, { "id": "vkrobot-crawler", "categories": [ "unknown" ], "pattern": { "accepted": [ "VKRobot" ], "forbidden": [] }, "addition_date": "2020/03/16", "verification": [], "instances": { "accepted": [ "Mozilla/5.0 (compatible; VKRobot/1.0)" ], "rejected": [] } }, { "id": "bidswitch-crawler", "categories": [ "unknown" ], "pattern": { "accepted": [ "bidswitchbot" ], "forbidden": [] }, "addition_date": "2020/03/16", "verification": [], "instances": { "accepted": [ "bidswitchbot/1.0" ], "rejected": [] }, "url": "https://www.bidswitch.com/about-us/" }, { "id": "tiger-crawler", "categories": [ "unknown" ], "pattern": { "accepted": [ "tigerbot" ], "forbidden": [] }, "addition_date": "2020/03/16", "verification": [], "instances": { "accepted": [ "tigerbot" ], "rejected": [] } }, { "id": "nixstats-crawler", "categories": [ "monitor" ], "pattern": { "accepted": [ "NIXStatsbot" ], "forbidden": [] }, "addition_date": "2020/03/16", "verification": [], "instances": { "accepted": [ "Mozilla/5.0 (compatible; NIXStatsbot/1.1; +http://www.nixstats.com/bot.html)" ], "rejected": [] }, "url": "http://www.nixstats.com/" }, { "id": "rssmicro-feedfetcher", "categories": [ "feedfetcher" ], "pattern": { "accepted": [ "Atom Feed Robot" ], "forbidden": [] }, "addition_date": "2020/03/16", "verification": [], "instances": { "accepted": [ "RSSMicro.com RSS/Atom Feed Robot" ], "rejected": [] }, "url": "https://rssmicro.com" }, { "id": "curebot-crawler", "categories": [ "unknown" ], "pattern": { "accepted": [ "[Cc]urebot" ], "forbidden": [] }, "addition_date": "2020/03/16", "verification": [], "instances": { "accepted": [ "Curebot/1.0", "curebot-feed-fetcher" ], "rejected": [] } }, { "id": "pagepeeker-crawler", "categories": [ "monitor", "preview" ], "pattern": { "accepted": [ "PagePeeker\\/" ], "forbidden": [] }, "addition_date": "2020/03/16", "verification": [], "instances": { "accepted": [ "Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.120 Safari/537.36 (compatible; PagePeeker/3.0; +https://pagepeeker.com/robots/)" ], "rejected": [] }, "url": "https://pagepeeker.com/robots/" }, { "id": "vigil-crawler", "categories": [ "unknown" ], "pattern": { "accepted": [ "Vigil\\/" ], "forbidden": [] }, "addition_date": "2020/03/16", "verification": [], "instances": { "accepted": [ "Mozilla/5.0 (compatible; Vigil/1.0; +http://vigil-app.com/bot.html)" ], "rejected": [] }, "url": "http://vigil-app.com/bot.html" }, { "id": "rssbot-feedfetcher", "categories": [ "feedfetcher" ], "pattern": { "accepted": [ "rssbot\\/" ], "forbidden": [] }, "addition_date": "2020/03/16", "verification": [], "instances": { "accepted": [ "rssbot/1.4.3 (+https://t.me/RustRssBot)" ], "rejected": [] }, "url": "https://github.com/iovxw/rssbot" }, { "id": "startme-crawler", "categories": [ "feedfetcher" ], "pattern": { "accepted": [ "startmebot\\/" ], "forbidden": [] }, "addition_date": "2020/03/16", "verification": [], "instances": { "accepted": [ "Mozilla/5.0 (compatible; startmebot/1.0; +https://start.me/bot)" ], "rejected": [] }, "url": "https://start.me/bot" }, { "id": "jobboerse-crawler", "categories": [ "unknown" ], "pattern": { "accepted": [ "JobboerseBot" ], "forbidden": [] }, "addition_date": "2020/03/16", "verification": [], "instances": { "accepted": [ "Mozilla/5.0 (X11; U; Linux Core i7-4980HQ; de; rv:32.0; compatible; JobboerseBot; http://www.jobboerse.com/bot.htm) Gecko/20100101 Firefox/38.0" ], "rejected": [] }, "url": "http://www.jobboerse.com/bot.htm" }, { "id": "seewithkids-crawler", "categories": [ "unknown" ], "pattern": { "accepted": [ "seewithkids" ], "forbidden": [] }, "addition_date": "2020/03/16", "verification": [], "instances": { "accepted": [ "http://seewithkids.com/bot" ], "rejected": [] }, "url": "http://seewithkids.com/bot" }, { "id": "ninjabot", "categories": [ "unknown" ], "pattern": { "accepted": [ "NINJA bot" ], "forbidden": [] }, "addition_date": "2020/03/16", "verification": [], "instances": { "accepted": [ "NINJA bot" ], "rejected": [] } }, { "id": "cutbot-crawler", "categories": [ "unknown" ], "pattern": { "accepted": [ "Cutbot" ], "forbidden": [] }, "addition_date": "2020/03/16", "verification": [], "instances": { "accepted": [ "Cutbot; 1.5; http://cutbot.net/" ], "rejected": [] }, "url": "http://cutbot.net/" }, { "id": "bublup-crawler", "categories": [ "unknown" ], "pattern": { "accepted": [ "BublupBot" ], "forbidden": [] }, "addition_date": "2020/03/16", "verification": [], "instances": { "accepted": [ "BublupBot (+https://www.bublup.com/bublup-bot.html)" ], "rejected": [] }, "url": "https://www.bublup.com/bublup-bot.html" }, { "id": "brandonmedia-crawler", "categories": [ "unknown" ], "pattern": { "accepted": [ "BrandONbot" ], "forbidden": [] }, "addition_date": "2020/03/16", "verification": [], "instances": { "accepted": [ "BrandONbot (http://brandonmedia.net)" ], "rejected": [] }, "url": "http://brandonmedia.net" }, { "id": "ridder-crawler", "categories": [ "unknown" ], "pattern": { "accepted": [ "RidderBot" ], "forbidden": [] }, "addition_date": "2020/03/16", "verification": [], "instances": { "accepted": [ "Mozilla/5.0 (compatible; RidderBot/1.0; bot@ridder.co)", "Mozilla/5.0 (compatible; RidderBot/1.0; bot@ridder.co) (iPhone; CPU iPhone OS 8_4_1 like Mac OS X) Mobile/12H321" ], "rejected": [] }, "url": "https://ridder.co/" }, { "id": "taboola-crawler", "categories": [ "unknown" ], "pattern": { "accepted": [ "Taboolabot" ], "forbidden": [] }, "addition_date": "2020/03/16", "verification": [], "instances": { "accepted": [ "Mozilla/5.0 (compatible; Taboolabot/3.7; +http://www.taboola.com)" ], "rejected": [] }, "url": "http://www.taboola.com" }, { "id": "dubbot-crawler", "categories": [ "monitor", "optimizer" ], "pattern": { "accepted": [ "Dubbotbot" ], "forbidden": [] }, "addition_date": "2020/03/16", "verification": [], "instances": { "accepted": [ "Mozilla/5.0 (compatible; Dubbotbot/0.2; +http://dubbot.com)" ], "rejected": [] }, "url": "http://dubbot.com" }, { "id": "itinfluentials-crawler", "categories": [ "unknown" ], "pattern": { "accepted": [ "FindITAnswersbot" ], "forbidden": [] }, "addition_date": "2020/03/16", "verification": [], "instances": { "accepted": [ "Mozilla/5.0 (compatible;FindITAnswersbot/1.0;+http://search.it-influentials.com/bot.htm)" ], "rejected": [] }, "url": "http://search.it-influentials.com/bot.htm" }, { "id": "infoo-crawler", "categories": [ "unknown" ], "pattern": { "accepted": [ "infoobot" ], "forbidden": [] }, "addition_date": "2020/03/16", "verification": [], "instances": { "accepted": [ "infoobot/0.1 (https://www.infoo.nl/bot.html)" ], "rejected": [] }, "url": "https://www.infoo.nl/bot.html" }, { "id": "refind-crawler", "categories": [ "feedfetcher" ], "pattern": { "accepted": [ "Refindbot" ], "forbidden": [] }, "addition_date": "2020/03/16", "verification": [], "instances": { "accepted": [ "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.103 Safari/537.36 (Refindbot/1.0)" ], "rejected": [] }, "url": "https://refind.com/about" }, { "id": "blogtraffic-feedfetcher", "categories": [ "feedfetcher" ], "pattern": { "accepted": [ "BlogTraffic\\/\\d\\.\\d+ Feed-Fetcher" ], "forbidden": [] }, "addition_date": "2020/03/16", "verification": [], "instances": { "accepted": [ "Mozilla/5.0 (compatible; BlogTraffic/1.4 Feed-Fetcher; +http://www.blogtraffic.de/rss-bot.html)" ], "rejected": [] }, "url": "http://www.blogtraffic.de/rss-bot.html" }, { "id": "seobility-crawler", "categories": [ "unknown" ], "pattern": { "accepted": [ "SeobilityBot" ], "forbidden": [] }, "addition_date": "2020/03/16", "verification": [], "instances": { "accepted": [ "SeobilityBot (SEO Tool; https://www.seobility.net/sites/bot.html)" ], "rejected": [] }, "url": "https://www.seobility.net/sites/bot.html" }, { "id": "cincrawdata-crawler", "categories": [ "unknown" ], "pattern": { "accepted": [ "Cincraw" ], "forbidden": [] }, "addition_date": "2020/03/16", "verification": [], "instances": { "accepted": [ "Mozilla/5.0 (compatible; Cincraw/1.0; +http://cincrawdata.net/bot/)" ], "rejected": [] }, "url": "http://cincrawdata.net/bot/" }, { "id": "dragonmetrics-crawler", "categories": [ "unknown" ], "pattern": { "accepted": [ "Dragonbot" ], "forbidden": [] }, "addition_date": "2020/03/16", "verification": [], "instances": { "accepted": [ "Mozilla/5.0 (Windows NT 6.1; rv:34.0) Gecko/20100101 Firefox/34.0; Dragonbot; http://www.dragonmetrics.com" ], "rejected": [] }, "url": "http://www.dragonmetrics.com" }, { "id": "codewise-crawler", "categories": [ "unknown" ], "pattern": { "accepted": [ "VoluumDSP-content-bot" ], "forbidden": [] }, "addition_date": "2020/03/16", "verification": [], "instances": { "accepted": [ "Mozilla/5.0 (compatible; VoluumDSP-content-bot/2.0; +dsp-dev@codewise.com)" ], "rejected": [] }, "url": "https://codewise.com" }, { "id": "freshrss-feedfetcher", "categories": [ "feedfetcher" ], "pattern": { "accepted": [ "FreshRSS" ], "forbidden": [] }, "addition_date": "2020/03/16", "verification": [], "instances": { "accepted": [ "FreshRSS/1.11.2 (Linux; https://freshrss.org) like Googlebot" ], "rejected": [] }, "url": "https://freshrss.org" }, { "id": "python-bitbot", "categories": [ "programmatic" ], "pattern": { "accepted": [ "BitBot" ], "forbidden": [] }, "addition_date": "2020/03/16", "verification": [], "instances": { "accepted": [ "Mozilla/5.0 (compatible; BitBot/v1.19.0; +https://bitbot.dev)" ], "rejected": [] }, "url": "https://bitbot.dev" }, { "id": "php-curlclass", "categories": [ "programmatic" ], "pattern": { "accepted": [ "^PHP-Curl-Class" ], "forbidden": [] }, "addition_date": "2020/12/10", "verification": [], "instances": { "accepted": [ "PHP-Curl-Class/4.13.0 (+https://github.com/php-curl-class/php-curl-class) PHP/7.2.24 curl/7.61.1", "PHP-Curl-Class/4.13.0 (+https://github.com/php-curl-class/php-curl-class) PHP/7.3.19 curl/7.66.0", "PHP-Curl-Class/4.13.0 (+https://github.com/php-curl-class/php-curl-class) PHP/7.3.23 curl/7.66.0", "PHP-Curl-Class/4.13.0 (+https://github.com/php-curl-class/php-curl-class) PHP/7.4.7 curl/7.69.1", "PHP-Curl-Class/4.13.0 (+https://github.com/php-curl-class/php-curl-class) PHP/7.4.9 curl/7.69.1", "PHP-Curl-Class/4.13.0 (+https://github.com/php-curl-class/php-curl-class) PHP/7.4.10 curl/7.69.1", "PHP-Curl-Class/4.13.0 (+https://github.com/php-curl-class/php-curl-class) PHP/7.4.11 curl/7.69.1" ], "rejected": [] }, "url": "https://github.com/php-curl-class/php-curl-class" }, { "id": "google-certificates-bridge", "categories": [ "google", "monitor" ], "pattern": { "accepted": [ "Google-Certificates-Bridge" ], "forbidden": [] }, "addition_date": "2020/12/23", "verification": [], "instances": { "accepted": [ "Google-Certificates-Bridge" ], "rejected": [] } }, { "id": "centurybot", "categories": [ "unknown" ], "pattern": { "accepted": [ "centurybot" ], "forbidden": [] }, "addition_date": "2022/04/26", "verification": [], "instances": { "accepted": [ "Mozilla/5.0 (compatible; Go-http-client/1.1; +centurybot9@gmail.com)" ], "rejected": [] } }, { "id": "viber-crawler", "categories": [ "preview", "social" ], "pattern": { "accepted": [ "Viber" ], "forbidden": [] }, "addition_date": "2021/04/27", "verification": [], "instances": { "accepted": [ "Viber" ], "rejected": [] }, "url": "https://www.viber.com/", "aliases": [ "ViberBot" ] }, { "id": "eventures-crawler", "categories": [ "unknown" ], "pattern": { "accepted": [ "e\\.ventures Investment Crawler" ], "forbidden": [] }, "addition_date": "2021/06/05", "url": "https://www.eventures.vc/", "verification": [], "instances": { "accepted": [ "e.ventures Investment Crawler (eventures.vc)" ], "rejected": [] } }, { "id": "eventures-crawler-batch", "categories": [ "unknown" ], "pattern": { "accepted": [ "evc-batch" ], "forbidden": [] }, "addition_date": "2021/06/07", "url": "https://www.eventures.vc/", "verification": [], "instances": { "accepted": [ "Mozilla/5.0 (compatible; evc-batch/2.0)" ], "rejected": [] } }, { "id": "petalsearch-crawler", "categories": [ "ai", "search-engine" ], "pattern": { "accepted": [ "PetalBot" ], "forbidden": [] }, "addition_date": "2021/06/07", "verification": [], "instances": { "accepted": [ "Mozilla/5.0 (compatible;PetalBot;+https://webmaster.petalsearch.com/site/petalbot)", "Mozilla/5.0 (Linux; Android 7.0;) AppleWebKit/537.36 (KHTML, like Gecko) Mobile Safari/537.36 (compatible; PetalBot;+https://webmaster.petalsearch.com/site/petalbot)" ], "rejected": [] }, "url": "https://webmaster.petalsearch.com/site/petalbot" }, { "id": "virustotal-crawler", "categories": [ "unknown" ], "pattern": { "accepted": [ "virustotal" ], "forbidden": [] }, "addition_date": "2021/09/22", "verification": [], "instances": { "accepted": [ "Mozilla/5.0 (Windows; U; MSIE 9.0; Windows NT 9.0; en-US) AppEngine-Google; (+http://code.google.com/appengine; appid: s~virustotalcloud)", "AppEngine-Google; (+http://code.google.com/appengine; appid: s~virustotalcloud)" ], "rejected": [] }, "url": "https://www.virustotal.com/gui/home/url" }, { "id": "webpagetest-crawler", "categories": [ "monitor", "optimizer" ], "pattern": { "accepted": [ "(^| )PTST\\/" ], "forbidden": [] }, "addition_date": "2021/12/05", "verification": [], "instances": { "accepted": [ "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/96.0.4664.45 Safari/537.36 PTST/211202.211915", "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:94.0) Gecko/20100101 Firefox/94.0 PTST/211202.211915" ], "rejected": [] }, "url": "https://www.webpagetest.org" }, { "id": "testomato-crawler", "categories": [ "monitor", "optimizer" ], "pattern": { "accepted": [ "minicrawler" ], "forbidden": [] }, "addition_date": "2022/01/12", "verification": [], "instances": { "accepted": [ "Testomatobot/1.0 (Linux x86_64; +https://www.testomato.com/testomatobot) minicrawler/5.2.2" ], "rejected": [] }, "url": "https://www.testomato.com/bot" }, { "id": "cookiebot-crawler", "categories": [ "unknown" ], "pattern": { "accepted": [ "Cookiebot" ], "forbidden": [] }, "addition_date": "2022/01/23", "url": "https://www.cookiebot.com/", "verification": [], "instances": { "accepted": [ "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko; compatible; Cookiebot/1.0; +http://cookiebot.com/) Chrome/97.0.4692.71 Safari/537.36" ], "rejected": [] } }, { "id": "trovit-crawler", "categories": [ "unknown" ], "pattern": { "accepted": [ "trovitBot" ], "forbidden": [] }, "addition_date": "2022/06/08", "url": "http://www.trovit.com/bot.html", "verification": [], "instances": { "accepted": [ "Mozilla/5.0 (compatible; trovitBot 1.0; +http://www.trovit.com/bot.html)" ], "rejected": [] } }, { "id": "seostar-crawler", "categories": [ "unknown" ], "pattern": { "accepted": [ "seostar\\.co" ], "forbidden": [] }, "addition_date": "2022/08/04", "url": "https://seostar.co/robot/", "verification": [], "instances": { "accepted": [ "Mozilla/5.0 (compatible; Adsbot/3.1; +https://seostar.co/robot/)" ], "rejected": [] } }, { "id": "ionos-crawler", "categories": [ "unknown" ], "pattern": { "accepted": [ "IonCrawl" ], "forbidden": [] }, "addition_date": "2022/08/04", "url": "https://www.ionos.de/terms-gtc/faq-crawler-en", "verification": [], "instances": { "accepted": [ "IonCrawl (https://www.ionos.de/terms-gtc/faq-crawler-en/)" ], "rejected": [] } }, { "id": "kuma-monitor", "categories": [ "monitor" ], "pattern": { "accepted": [ "Uptime-Kuma" ], "forbidden": [] }, "addition_date": "2022/10/17", "url": "https://uptime.kuma.pet/", "verification": [], "instances": { "accepted": [ "Uptime-Kuma/1.18.0" ], "rejected": [] } }, { "id": "seekport-crawler", "categories": [ "search-engine" ], "pattern": { "accepted": [ "Seekport" ], "forbidden": [] }, "addition_date": "2022/10/17", "url": "https://bot.seekport.com", "verification": [], "instances": { "accepted": [ "Mozilla/5.0 (compatible; SeekportBot; +https://bot.seekport.com)", "Mozilla/5.0 (compatible; Seekport Crawler; http://seekport.com/)" ], "rejected": [] } }, { "id": "freshworks-monitor", "categories": [ "monitor" ], "pattern": { "accepted": [ "FreshpingBot" ], "forbidden": [] }, "addition_date": "2022/10/17", "url": "https://www.freshworks.com/website-monitoring/", "verification": [], "instances": { "accepted": [ "FreshpingBot/1.0 (+https://freshping.io/)" ], "rejected": [] } }, { "id": "feedbin-crawler", "categories": [ "feedfetcher" ], "pattern": { "accepted": [ "Feedbin" ], "forbidden": [] }, "addition_date": "2022/11/05", "url": "https://feedbin.com/", "verification": [], "instances": { "accepted": [ "Feedbin feed-id:2005098 - 2 subscribers" ], "rejected": [] } }, { "id": "criteo-crawler", "categories": [ "unknown" ], "pattern": { "accepted": [ "CriteoBot" ], "forbidden": [] }, "addition_date": "2022/11/13", "url": "https://www.criteo.com/", "verification": [], "instances": { "accepted": [ "CriteoBot/0.1 (+https://www.criteo.com/criteo-crawler/)" ], "rejected": [] } }, { "id": "snap-preview", "categories": [ "preview", "social" ], "pattern": { "accepted": [ "Snap URL Preview Service" ], "forbidden": [] }, "addition_date": "2022/11/13", "url": "https://developers.snap.com/robots", "verification": [], "instances": { "accepted": [ "Snap URL Preview Service; bot; snapchat; https://developers.snap.com/robots" ], "rejected": [] } }, { "id": "betteruptime-monitor", "categories": [ "monitor" ], "pattern": { "accepted": [ "Better Uptime Bot" ], "forbidden": [] }, "addition_date": "2022/11/13", "url": "https://betteruptime.com/", "verification": [], "instances": { "accepted": [ "Better Uptime Bot Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.169 Safari/537.36" ], "rejected": [] } }, { "id": "betterstack-monitor", "categories": [ "monitor" ], "pattern": { "accepted": [ "Better Stack" ], "forbidden": [] }, "addition_date": "2026/02/08", "url": "https://betterstack.com/", "verification": [ { "type": "ip", "sources": [ { "type": "http-json", "url": "https://uptime.betterstack.com/ips-by-cluster.json", "selector": "$.*[*]" } ] } ], "instances": { "accepted": [ "Better Stack Better Uptime Bot Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/130.0.0.0 Safari/537.36" ], "rejected": [] } }, { "id": "dynatrace-monitor", "categories": [ "monitor" ], "pattern": { "accepted": [ "RuxitSynthetic" ], "forbidden": [] }, "addition_date": "2023/02/16", "url": "https://www.dynatrace.com/support/help/platform-modules/digital-experience/synthetic-monitoring/browser-monitors/configure-browser-monitors#expand--default-user-agent", "verification": [], "instances": { "accepted": [ "RuxitSynthetic/1.0" ], "rejected": [] } }, { "id": "google-read-aloud", "categories": [ "google" ], "pattern": { "accepted": [ "Google-Read-Aloud" ], "forbidden": [] }, "addition_date": "2023/02/16", "url": "https://developers.google.com/search/docs/crawling-indexing/overview-google-crawlers", "verification": [], "instances": { "accepted": [ "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2272.118 Safari/537.36 (compatible; Google-Read-Aloud; +https://developers.google.com/search/docs/crawling-indexing/overview-google-crawlers)", "Mozilla/5.0 (Linux; Android 7.0; SM-G930V Build/NRD90M) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/59.0.3071.125 Mobile Safari/537.36 (compatible; Google-Read-Aloud; +https://developers.google.com/search/docs/crawling-indexing/overview-google-crawlers)" ], "rejected": [] } }, { "id": "steam-preview", "categories": [ "preview", "social" ], "pattern": { "accepted": [ "Valve\\/Steam" ], "forbidden": [] }, "addition_date": "2023/05/24", "verification": [], "instances": { "accepted": [ "Valve/Steam HTTP Client 1.0 (SteamChatURLLookup)" ], "rejected": [] }, "aliases": [ "ValveSteamBot" ] }, { "id": "okru-crawler", "categories": [ "unknown" ], "pattern": { "accepted": [ "OdklBot\\/" ], "forbidden": [] }, "addition_date": "2023/05/24", "verification": [], "instances": { "accepted": [ "OdklBot/1.0 (share@odnoklassniki.ru)", "Mozilla/5.0 (compatible; OdklBot/1.0 like Linux; klass@odnoklassniki.ru)" ], "rejected": [] }, "url": "https://odnoklassniki.ru/" }, { "id": "openai-crawler", "categories": [ "ai" ], "pattern": { "accepted": [ "GPTBot" ], "forbidden": [] }, "addition_date": "2023/08/09", "verification": [ { "type": "cidr", "sources": [ { "type": "http-json", "url": "https://openai.com/gptbot.json", "selector": "$.prefixes[*][\\\"ipv6Prefix\\\",\\\"ipv4Prefix\\\"]" } ] } ], "instances": { "accepted": [ "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; GPTBot/1.0; +https://openai.com/gptbot)" ], "rejected": [] }, "url": "https://platform.openai.com/docs/gptbot" }, { "id": "openai-crawler-user", "categories": [ "ai" ], "pattern": { "accepted": [ "ChatGPT-User" ], "forbidden": [] }, "addition_date": "2024/04/19", "verification": [ { "type": "cidr", "sources": [ { "type": "http-json", "url": "https://openai.com/chatgpt-user.json", "selector": "$.prefixes[*][\\\"ipv6Prefix\\\",\\\"ipv4Prefix\\\"]" } ] } ], "instances": { "accepted": [ "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko); compatible; ChatGPT-User/1.0; +https://openai.com/bot" ], "rejected": [] }, "url": "https://openai.com/bot" }, { "id": "yandex-crawler-javascript", "categories": [ "search-engine" ], "pattern": { "accepted": [ "YandexRenderResourcesBot\\/" ], "forbidden": [] }, "addition_date": "2023/08/16", "verification": [ { "type": "dns", "masks": [ "@.yandex.ru", "@.yandex.com", "@.yandex.net" ] } ], "instances": { "accepted": [ "Mozilla/5.0 (compatible; YandexRenderResourcesBot/1.0; +http://yandex.com/bots) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/108.0.0.0" ], "rejected": [] }, "url": "http://yandex.com/bots" }, { "id": "lightspeedsystems-crawler", "categories": [ "unknown" ], "pattern": { "accepted": [ "LightspeedSystemsCrawler" ], "forbidden": [] }, "addition_date": "2023/08/16", "verification": [], "instances": { "accepted": [ "LightspeedSystemsCrawler Mozilla/5.0 (Windows; U; MSIE 9.0; Windows NT 9.0; en-US" ], "rejected": [] } }, { "id": "headline-crawler", "categories": [ "unknown" ], "pattern": { "accepted": [ "ev-crawler\\/" ], "forbidden": [] }, "addition_date": "2023/08/16", "verification": [], "instances": { "accepted": [ "Mozilla/5.0 (compatible; ev-crawler/1.0; +https://headline.com/legal/crawler)" ], "rejected": [] }, "url": "https://headline.com/legal/crawler" }, { "id": "bitsight-crawler", "categories": [ "unknown" ], "pattern": { "accepted": [ "BitSightBot\\/" ], "forbidden": [] }, "addition_date": "2023/08/16", "verification": [], "instances": { "accepted": [ "Mozilla/5.0 (compatible; BitSightBot/1.0)" ], "rejected": [] }, "url": "https://www.bitsight.com" }, { "id": "woorank-crawler-review", "categories": [ "unknown" ], "pattern": { "accepted": [ "woorankreview\\/" ], "forbidden": [] }, "addition_date": "2023/08/16", "verification": [], "instances": { "accepted": [ "Mozilla/5.0 (iPhone; CPU iPhone OS 11_0 like Mac OS X) AppleWebKit/604.1.38 (KHTML, like Gecko) Version/11.0 Mobile/15A372 Safari/604.1 (compatible; woorankreview/2.0; +https://www.woorank.com/)", "Mozilla/5.0 (compatible; woorankreview/2.0; +https://www.woorank.com/)" ], "rejected": [] }, "url": "https://www.woorank.com/" }, { "id": "google-crawler-safety", "categories": [ "google" ], "pattern": { "accepted": [ "Google-Safety" ], "forbidden": [] }, "addition_date": "2023/08/17", "verification": [ { "type": "cidr", "sources": [ { "type": "http-json", "url": "https://developers.google.com/static/search/apis/ipranges/special-crawlers.json", "selector": "$.prefixes[*][\\\"ipv6Prefix\\\",\\\"ipv4Prefix\\\"]" } ] }, { "type": "dns", "masks": [ "rate-limited-proxy-***-***-***-***.google.com" ] } ], "instances": { "accepted": [ "Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.5735.179 Mobile Safari/537.36 (compatible; Google-Safety; +http://www.google.com/bot.html)", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.5735.179 Safari/537.36 (compatible; Google-Safety; +http://www.google.com/bot.html)", "Google-Safety" ], "rejected": [] }, "url": "https://developers.google.com/search/docs/crawling-indexing/overview-google-crawlers" }, { "id": "awario-crawler", "categories": [ "unknown" ], "pattern": { "accepted": [ "AwarioBot" ], "forbidden": [] }, "addition_date": "2023/08/23", "verification": [], "instances": { "accepted": [ "Mozilla/5.0 (compatible; AwarioBot/1.0; +https://awario.com/bots.html)" ], "rejected": [] }, "url": "https://awario.com/bots.html" }, { "id": "dataforseo-crawler", "categories": [ "unknown" ], "pattern": { "accepted": [ "DataForSeoBot" ], "forbidden": [] }, "addition_date": "2023/08/23", "verification": [], "instances": { "accepted": [ "Mozilla/5.0 (compatible; DataForSeoBot/1.0; +https://dataforseo.com/dataforseo-bot)" ], "rejected": [] }, "url": "https://dataforseo.com/dataforseo-bot" }, { "id": "line-crawler", "categories": [ "unknown" ], "pattern": { "accepted": [ "Linespider" ], "forbidden": [] }, "addition_date": "2023/08/24", "verification": [], "instances": { "accepted": [ "Mozilla/5.0 (compatible; Linespider/1.1; +https://lin.ee/4dwXkTH)", "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Linespider/1.1; +https://lin.ee/4dwXkTH) Chrome/W.X.Y.Z Safari/537.36" ], "rejected": [] }, "url": "https://help2.line.me/linesearchbot/web/?contentId=50006055&lang=en" }, { "id": "wellknown-crawler", "categories": [ "search-engine" ], "pattern": { "accepted": [ "WellKnownBot" ], "forbidden": [] }, "addition_date": "2023/08/29", "verification": [], "instances": { "accepted": [ "Mozilla/5.0 (compatible; WellKnownBot/0.1; +https://well-known.dev/about/#bot)" ], "rejected": [] }, "url": "https://well-known.dev/about/#bot)" }, { "id": "epfl-crawler", "categories": [ "academic" ], "pattern": { "accepted": [ "A Patent Crawler" ], "forbidden": [] }, "addition_date": "2023/08/29", "verification": [], "instances": { "accepted": [ "E. Orliac, G. Fourestey/2.3 (A Patent Crawler; http://scitas.epfl.ch/; etienne.orliac@epfl.ch, gilles.fourestey@epfl.ch)" ], "rejected": [] }, "url": "http://scitas.epfl.ch/" }, { "id": "stract-crawler", "categories": [ "search-engine" ], "pattern": { "accepted": [ "StractBot" ], "forbidden": [] }, "addition_date": "2023/09/06", "verification": [], "instances": { "accepted": [ "Mozilla/5.0 (compatible; StractBot/0.1; open source search engine; +https://trystract.com/webmasters)" ], "rejected": [] }, "url": "https://trystract.com/webmasters" }, { "id": "marginalia-crawler", "categories": [ "search-engine" ], "pattern": { "accepted": [ "search\\.marginalia\\.nu" ], "forbidden": [] }, "addition_date": "2023/09/08", "verification": [], "instances": { "accepted": [ "search.marginalia.nu" ], "rejected": [] }, "url": "https://search.marginalia.nu" }, { "id": "you-crawler", "categories": [ "ai" ], "pattern": { "accepted": [ "YouBot" ], "forbidden": [] }, "addition_date": "2023/09/08", "verification": [], "instances": { "accepted": [ "YouBot (+http://www.you.com)" ], "rejected": [] }, "url": "https://you.com/" }, { "id": "nicecrawler-archive", "categories": [ "archive" ], "pattern": { "accepted": [ "Nicecrawler" ], "forbidden": [] }, "addition_date": "2023/09/08", "verification": [], "instances": { "accepted": [ "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Nicecrawler/1.1; +http://www.nicecrawler.com/) Chrome/90.0.4430.97 Safari/537.36" ], "rejected": [] }, "url": "http://www.nicecrawler.com/" }, { "id": "neeva-crawler", "categories": [ "unknown" ], "pattern": { "accepted": [ "Neevabot" ], "forbidden": [] }, "addition_date": "2023/09/08", "verification": [], "instances": { "accepted": [ "Mozilla/5.0 (compatible; Neevabot/1.0; +https://neeva.com/neevabot)" ], "rejected": [] }, "url": "https://neeva.com/neevabot" }, { "id": "brightedge-crawler", "categories": [ "unknown" ], "pattern": { "accepted": [ "BrightEdge Crawler" ], "forbidden": [] }, "addition_date": "2023/09/08", "verification": [], "instances": { "accepted": [ "BrightEdge Crawler/1.0 (crawler@brightedge.com)" ], "rejected": [] }, "url": "https://www.brightedge.com/" }, { "id": "sitechecker-crawler", "categories": [ "unknown" ], "pattern": { "accepted": [ "SiteCheckerBotCrawler" ], "forbidden": [] }, "addition_date": "2023/09/08", "verification": [], "instances": { "accepted": [ "SiteCheckerBotCrawler/1.0 (+http://sitechecker.pro)" ], "rejected": [] }, "url": "http://sitechecker.pro" }, { "id": "tombascraper-crawler", "categories": [ "unknown" ], "pattern": { "accepted": [ "TombaPublicWebCrawler" ], "forbidden": [] }, "addition_date": "2023/09/08", "verification": [], "instances": { "accepted": [ "Mozilla/5.0 (compatible; TombaPublicWebCrawler/1.0; +https://tombascraper.com)" ], "rejected": [] }, "url": "https://tombascraper.com" }, { "id": "digitaldragon-crawler", "categories": [ "unknown" ], "pattern": { "accepted": [ "CrawlyProjectCrawler" ], "forbidden": [] }, "addition_date": "2023/09/08", "verification": [], "instances": { "accepted": [ "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36 (compatible; CrawlyProjectCrawler/0.1.3; crawlyproject@digitaldragon.dev +https://crawlyproject.digitaldragon.dev/)" ], "rejected": [] }, "url": "https://crawlyproject.digitaldragon.dev/" }, { "id": "komodia-crawler", "categories": [ "unknown" ], "pattern": { "accepted": [ "KomodiaBot" ], "forbidden": [] }, "addition_date": "2023/09/08", "verification": [], "instances": { "accepted": [ "Mozilla/5.0 (Windows NT 6.1; Win64; x64; +http://www.komodia.com/newwiki/index.php/URL_server_crawler) KomodiaBot/1.0" ], "rejected": [] }, "url": "http://www.komodia.com/newwiki/index.php/URL_server_crawler" }, { "id": "urlclassification-crawler", "categories": [ "unknown" ], "pattern": { "accepted": [ "KStandBot" ], "forbidden": [] }, "addition_date": "2023/09/08", "verification": [], "instances": { "accepted": [ "Mozilla/5.0 (Windows NT 6.1; Win64; x64; +http://url-classification.io/wiki/index.php?title=URL_server_crawler) KStandBot/1.0" ], "rejected": [] }, "url": "http://url-classification.io" }, { "id": "cispa-crawler", "categories": [ "academic" ], "pattern": { "accepted": [ "CISPA Webcrawler" ], "forbidden": [] }, "addition_date": "2023/09/08", "verification": [], "instances": { "accepted": [ "CISPA Webcrawler (https://vuln-notify-checker.cispa.saarland)" ], "rejected": [] }, "url": "https://vuln-notify-checker.cispa.saarland" }, { "id": "metricstools-crawler", "categories": [ "unknown" ], "pattern": { "accepted": [ "MTRobot" ], "forbidden": [] }, "addition_date": "2023/09/08", "verification": [], "instances": { "accepted": [ "MTRobot/0.2 (Metrics Tools Analytics Crawler; https://metrics-tools.de/robot.html; crawler@metrics-tools.de)" ], "rejected": [] }, "url": "https://metrics-tools.de/robot.html" }, { "id": "hyscore-crawler", "categories": [ "unknown" ], "pattern": { "accepted": [ "hyscore\\.io" ], "forbidden": [] }, "addition_date": "2023/09/08", "verification": [], "instances": { "accepted": [ "Mozilla/5.0 (iPhone; CPU iPhone OS 8_3 like Mac OS X) AppleWebKit/600.1.4 (KHTML, like Gecko) Version/8.0 Mobile/12F70 Safari/600.1. 4 (compatible; HyScore/1.0; +https://hyscore.io/crawler/)" ], "rejected": [] }, "url": "https://hyscore.io/crawler/" }, { "id": "alexandria-crawler", "categories": [ "search-engine" ], "pattern": { "accepted": [ "AlexandriaOrgBot" ], "forbidden": [] }, "addition_date": "2023/09/08", "verification": [], "instances": { "accepted": [ "Mozilla/5.0 (Linux) (compatible; AlexandriaOrgBot/1.0; +https://www.alexandria.org/bot.html)" ], "rejected": [] }, "url": "https://www.alexandria.org/bot.html" }, { "id": "twoip-crawler", "categories": [ "unknown" ], "pattern": { "accepted": [ "2ip bot" ], "forbidden": [] }, "addition_date": "2023/09/08", "verification": [], "instances": { "accepted": [ "2ip bot/1.1 (+http://2ip.io)" ], "rejected": [] }, "url": "http://2ip.io" }, { "id": "yellowbp-crawler", "categories": [ "unknown" ], "pattern": { "accepted": [ "Yellowbrandprotectionbot" ], "forbidden": [] }, "addition_date": "2023/09/08", "verification": [], "instances": { "accepted": [ "Mozilla/5.0 (compatible; Yellowbrandprotectionbot/1.0; +https://www.yellowbp.com/bot.html)" ], "rejected": [] }, "url": "https://www.yellowbp.com/bot.html" }, { "id": "seolizer-crawler", "categories": [ "unknown" ], "pattern": { "accepted": [ "SEOlizer" ], "forbidden": [] }, "addition_date": "2023/09/08", "verification": [], "instances": { "accepted": [ "SEOlizer/1.1 (Windows; U; Windows NT 5.1; en-US; rv:1.8.1.13) Gecko/20080311 Firefox/2.0.0.13 (+https://www.seolizer.de/bot.html)" ], "rejected": [] }, "url": "https://www.seolizer.de/bot.html" }, { "id": "vuhuv-crawler", "categories": [ "unknown" ], "pattern": { "accepted": [ "vuhuvBot" ], "forbidden": [] }, "addition_date": "2023/09/08", "verification": [], "instances": { "accepted": [ "Mozilla/5.0 (compatible; vuhuvBot/1.0; +http://vuhuv.com/bot.html)" ], "rejected": [] }, "url": "http://vuhuv.com/bot.html" }, { "id": "inetdex-crawler", "categories": [ "unknown" ], "pattern": { "accepted": [ "INETDEX-BOT" ], "forbidden": [] }, "addition_date": "2023/09/08", "verification": [], "instances": { "accepted": [ "INETDEX-BOT/1.5 (Mozilla/5.0; https://inetdex.com/bot.html)" ], "rejected": [] }, "url": "https://inetdex.com/bot.html" }, { "id": "synapse-crawler", "categories": [ "preview", "social" ], "pattern": { "accepted": [ "Synapse" ], "forbidden": [] }, "addition_date": "2023/09/08", "verification": [], "instances": { "accepted": [ "Synapse (bot; +https://github.com/matrix-org/synapse)" ], "rejected": [] }, "url": "https://github.com/matrix-org/synapse" }, { "id": "t3versions-crawler", "categories": [ "unknown" ], "pattern": { "accepted": [ "t3versionsBot" ], "forbidden": [] }, "addition_date": "2023/09/08", "verification": [], "instances": { "accepted": [ "Mozilla/5.0 (compatible; t3versionsBot/1.0; +https://www.t3versions.com/bot)" ], "rejected": [] }, "url": "https://www.t3versions.com/bot" }, { "id": "deepnoc-crawler", "categories": [ "unknown" ], "pattern": { "accepted": [ "deepnoc" ], "forbidden": [] }, "addition_date": "2023/09/08", "verification": [], "instances": { "accepted": [ "deepnoc - https://deepnoc.com/bot" ], "rejected": [] }, "url": "https://deepnoc.com/bot" }, { "id": "cocolyze-crawler", "categories": [ "unknown" ], "pattern": { "accepted": [ "Cocolyzebot" ], "forbidden": [] }, "addition_date": "2023/09/08", "verification": [], "instances": { "accepted": [ "Mozilla/5.0 (compatible; Cocolyzebot/1.0; https://cocolyze.com/bot)" ], "rejected": [] }, "url": "https://cocolyze.com/bot" }, { "id": "hypestat-crawler", "categories": [ "unknown" ], "pattern": { "accepted": [ "hypestat" ], "forbidden": [] }, "addition_date": "2023/09/08", "verification": [], "instances": { "accepted": [ "Mozilla/5.0 (compatible; hypestat/1.0; +https://hypestat.com/bot)" ], "rejected": [] }, "url": "https://hypestat.com/bot" }, { "id": "torus-crawler", "categories": [ "unknown" ], "pattern": { "accepted": [ "ReverseEngineeringBot" ], "forbidden": [] }, "addition_date": "2023/09/08", "verification": [], "instances": { "accepted": [ "Mozilla/5.0 (compatible; ReverseEngineeringBot/0.1; +https://torus.company/bot.html)" ], "rejected": [] }, "url": "https://torus.company/bot.html" }, { "id": "sempitech-crawler", "categories": [ "unknown" ], "pattern": { "accepted": [ "sempi\\.tech" ], "forbidden": [] }, "addition_date": "2023/09/08", "verification": [], "instances": { "accepted": [ "Mozilla/5.0 (compatible; Semanticbot/1.0; +http://sempi.tech/bot.html)" ], "rejected": [] }, "url": "http://sempi.tech/bot.html" }, { "id": "iframely-preview", "categories": [ "preview", "social" ], "pattern": { "accepted": [ "Iframely" ], "forbidden": [] }, "addition_date": "2023/09/08", "verification": [], "instances": { "accepted": [ "Iframely/1.3.1 (+https://iframely.com/docs/about) Atlassian" ], "rejected": [] }, "url": "https://iframely.com/docs/about" }, { "id": "ruby-metainspector", "categories": [ "programmatic" ], "pattern": { "accepted": [ "MetaInspector" ], "forbidden": [] }, "addition_date": "2023/09/08", "verification": [], "instances": { "accepted": [ "MetaInspector/5.6.0 (+https://github.com/jaimeiniesta/metainspector)" ], "rejected": [] }, "url": "https://github.com/jaimeiniesta/metainspector" }, { "id": "javascript-node-fetch", "categories": [ "programmatic" ], "pattern": { "accepted": [ "node-fetch" ], "forbidden": [] }, "addition_date": "2023/09/08", "verification": [], "instances": { "accepted": [ "node-fetch/1.0 (+https://github.com/bitinn/node-fetch)" ], "rejected": [] }, "url": "https://github.com/bitinn/node-fetch" }, { "id": "l9explore", "categories": [ "tool" ], "pattern": { "accepted": [ "l9explore" ], "forbidden": [] }, "addition_date": "2023/09/08", "verification": [], "instances": { "accepted": [ "l9explore/1.2.2", "lkxscan/v0.1.0 (+https://leakix.net) l9explore/v1.0.0 (+https://github.com/LeakIX/l9explore)" ], "rejected": [] }, "url": "https://github.com/LeakIX/l9explore" }, { "id": "python-opengraph", "categories": [ "programmatic" ], "pattern": { "accepted": [ "python-opengraph" ], "forbidden": [] }, "addition_date": "2023/09/08", "verification": [], "instances": { "accepted": [ "python-opengraph-jaywink/0.2.0 (+https://github.com/jaywink/python-opengraph)" ], "rejected": [] }, "url": "https://github.com/jaywink/python-opengraph" }, { "id": "opengraphcheck-crawler", "categories": [ "unknown" ], "pattern": { "accepted": [ "OpenGraphCheck" ], "forbidden": [] }, "addition_date": "2023/09/08", "verification": [], "instances": { "accepted": [ "OpenGraphCheck/2.1 (+https://opengraphcheck.com)" ], "rejected": [] }, "url": "https://opengraphcheck.com" }, { "id": "google-web-snippet", "categories": [ "google", "preview", "social" ], "pattern": { "accepted": [ "developers\\.google\\.com\\/\\+\\/web\\/snippet" ], "forbidden": [] }, "addition_date": "2023/09/08", "verification": [], "instances": { "accepted": [ "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.2924.87 Safari/537.36 Google-PageRenderer Google (+https://developers.google.com/+/web/snippet/)", "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.2924.87 Safari/537.36 Google (+https://developers.google.com/+/web/snippet/" ], "rejected": [] }, "url": "https://developers.google.com/+/web/snippet" }, { "id": "senuto-crawler", "categories": [ "unknown" ], "pattern": { "accepted": [ "SenutoBot" ], "forbidden": [] }, "addition_date": "2023/09/08", "verification": [], "instances": { "accepted": [ "SenutoBot/1.0 (compatible; SenutoBot/1.0; +https://www.senuto.com/)" ], "rejected": [] }, "url": "https://www.senuto.com" }, { "id": "macocu-crawler", "categories": [ "academic" ], "pattern": { "accepted": [ "MaCoCu" ], "forbidden": [] }, "addition_date": "2023/09/08", "verification": [], "instances": { "accepted": [ "Mozilla/5.0 (compatible; MaCoCu; +https://www.clarin.si/info/macocu-massive-collection-and-curation-of-monolingual-and-bilingual-data/)" ], "rejected": [] }, "url": "https://www.clarin.si/info/macocu-massive-collection-and-curation-of-monolingual-and-bilingual-data" }, { "id": "newsblur-aggregator", "categories": [ "feedfetcher" ], "pattern": { "accepted": [ "NewsBlur" ], "forbidden": [] }, "addition_date": "2023/09/08", "verification": [], "instances": { "accepted": [ "NewsBlur Feed Fetcher - 1 subscriber - http://www.newsblur.com/site/0000000/webpage (Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/14.0.1 Safari/605.1.15)" ], "rejected": [] }, "url": "http://www.newsblur.com" }, { "id": "inoreader-aggregator", "categories": [ "feedfetcher" ], "pattern": { "accepted": [ "inoreader" ], "forbidden": [] }, "addition_date": "2023/09/08", "verification": [], "instances": { "accepted": [ "Mozilla/5.0 (compatible; inoreader.com; 1 subscribers)" ], "rejected": [] }, "url": "http://inoreader.com" }, { "id": "netsystemsresearch-crawler", "categories": [ "unknown" ], "pattern": { "accepted": [ "NetSystemsResearch" ], "forbidden": [] }, "addition_date": "2023/09/08", "verification": [], "instances": { "accepted": [ "NetSystemsResearch studies the availability of various services across the internet. Our website is netsystemsresearch.com" ], "rejected": [] }, "url": "http://netsystemsresearch.com" }, { "id": "pagething-crawler", "categories": [ "unknown" ], "pattern": { "accepted": [ "PageThing" ], "forbidden": [] }, "addition_date": "2023/09/08", "verification": [], "instances": { "accepted": [ "PageThing http://pagething.com curl www" ], "rejected": [] }, "url": "http://pagething.com" }, { "id": "wordpress-crawler", "categories": [ "unknown" ], "pattern": { "accepted": [ "WordPress\\/" ], "forbidden": [] }, "addition_date": "2023/10/24", "verification": [], "instances": { "accepted": [ "WordPress/X.X.X; https://example.com" ], "rejected": [] }, "url": "https://wordpress.org", "aliases": [ "WordPressBot" ] }, { "id": "phxbot", "categories": [ "unknown" ], "pattern": { "accepted": [ "PhxBot" ], "forbidden": [] }, "addition_date": "2024/01/06", "verification": [], "instances": { "accepted": [ "PhxBot/0.1 (phxbot@protonmail.com)" ], "rejected": [] } }, { "id": "imagesift-crawler", "categories": [ "ai" ], "pattern": { "accepted": [ "ImagesiftBot" ], "forbidden": [] }, "addition_date": "2024/01/06", "verification": [], "instances": { "accepted": [ "Mozilla/5.0 (compatible; ImagesiftBot; +imagesift.com)" ], "rejected": [] }, "url": "https://imagesift.com/about" }, { "id": "paloaltonetworks-crawler", "categories": [ "unknown" ], "pattern": { "accepted": [ "Expanse" ], "forbidden": [] }, "addition_date": "2024/02/01", "verification": [], "instances": { "accepted": [ "Expanse, a Palo Alto Networks company, searches across the global IPv4 space multiple times per day to identify customers' presences on the Internet. If you would like to be excluded from our scans, please send IP addresses/domains to: scaninfo@paloaltonetworks.com" ], "rejected": [] }, "url": "https://www.paloaltonetworks.com/cortex/cortex-xpanse" }, { "id": "driftnet-crawler", "categories": [ "unknown" ], "pattern": { "accepted": [ "InternetMeasurement" ], "forbidden": [] }, "addition_date": "2024/02/01", "verification": [], "instances": { "accepted": [ "Mozilla/5.0 (compatible; InternetMeasurement/1.0; +https://internet-measurement.com/)" ], "rejected": [] }, "url": "https://internet-measurement.com" }, { "id": "builtwith-crawler", "categories": [ "unknown" ], "pattern": { "accepted": [ "^BW\\/" ], "forbidden": [] }, "addition_date": "2024/02/08", "verification": [], "instances": { "accepted": [ "BW/1.1; bit.ly/3eZNDnO", "BW/1.1; rb.gy/oupwis" ], "rejected": [] }, "url": "https://builtwith.com/biup" }, { "id": "geedo-crawler", "categories": [ "search-engine" ], "pattern": { "accepted": [ "GeedoBot" ], "forbidden": [] }, "addition_date": "2024/02/11", "verification": [], "instances": { "accepted": [ "Mozilla/5.0 (compatible; GeedoBot; +http://www.geedo.com/bot.html)" ], "rejected": [] }, "url": "http://www.geedo.com" }, { "id": "audisto-crawler", "categories": [ "unknown" ], "pattern": { "accepted": [ "Audisto Crawler" ], "forbidden": [] }, "addition_date": "2024/03/14", "verification": [], "instances": { "accepted": [ "Audisto Crawler (mobile; +https://audisto.com/bot)", "Audisto Crawler (desktop; +https://audisto.com/bot)", "Audisto Crawler (mobile; essential; +https://audisto.com/bot)", "Audisto Crawler (desktop; essential; +https://audisto.com/bot)" ], "rejected": [] }, "url": "https://audisto.com/help/crawler/bot/" }, { "id": "perplexity-crawler", "categories": [ "ai" ], "pattern": { "accepted": [ "PerplexityBot\\/" ], "forbidden": [] }, "addition_date": "2024/03/14", "verification": [ { "type": "cidr", "sources": [ { "type": "http-json", "url": "https://www.perplexity.ai/perplexitybot.json", "selector": "$.prefixes[*].ipv4Prefix" } ] } ], "instances": { "accepted": [ "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; PerplexityBot/1.0; +https://perplexity.ai/perplexitybot)" ], "rejected": [] }, "url": "https://docs.perplexity.ai/docs/perplexitybot" }, { "id": "anthropic-crawler", "categories": [ "ai" ], "pattern": { "accepted": [ "[cC]laude(?:[bB]ot|-[Ww]eb)" ], "forbidden": [] }, "addition_date": "2024/04/19", "verification": [], "instances": { "accepted": [ "claudebot", "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; ClaudeBot/1.0; +claudebot@anthropic.com)", "Claude-Web", "claude-web" ], "rejected": [] }, "url": "https://www.anthropic.com/" }, { "id": "monsido-crawler", "categories": [ "optimizer" ], "pattern": { "accepted": [ "Monsidobot" ], "forbidden": [] }, "addition_date": "2024/05/14", "verification": [], "instances": { "accepted": [ "Mozilla/5.0 (compatible; Monsidobot/2.2; +http://monsido.com/bot.html; info@monsido.com)" ], "rejected": [] }, "url": "http://monsido.com/bot.html" }, { "id": "groupme-crawler", "categories": [ "social" ], "pattern": { "accepted": [ "GroupMeBot" ], "forbidden": [] }, "addition_date": "2024/05/19", "verification": [], "instances": { "accepted": [ "GroupMeBot/1.0" ], "rejected": [] }, "url": "https://groupme.com/" }, { "id": "vercel-crawler", "categories": [ "vercel" ], "pattern": { "accepted": [ "Vercelbot" ], "forbidden": [] }, "addition_date": "2024/08/30", "verification": [], "instances": { "accepted": [ "Vercelbot (+https://vercel.com)" ], "rejected": [] }, "url": "https://github.com/vercel/vercel/discussions/5095#discussioncomment-58705" }, { "id": "vercel-monitor-preview", "categories": [ "monitor", "preview", "vercel" ], "pattern": { "accepted": [ "vercel-screenshot" ], "forbidden": [] }, "addition_date": "2024/08/30", "verification": [], "instances": { "accepted": [], "rejected": [] } }, { "id": "coda-server-fetcher", "categories": [ "programmatic" ], "pattern": { "accepted": [ "Coda-Server-Fetcher" ], "forbidden": [] }, "addition_date": "2024/09/10", "verification": [], "instances": { "accepted": [], "rejected": [] }, "url": "https://coda.io/product/packs" }, { "id": "ai2-crawler", "categories": [ "ai" ], "pattern": { "accepted": [ "AI2Bot\\s" ], "forbidden": [] }, "addition_date": "2024/09/16", "verification": [], "instances": { "accepted": [ "Mozilla/5.0 (compatible) AI2Bot (+https://www.allenai.org/crawler)" ], "rejected": [] }, "url": "https://allenai.org/crawler" }, { "id": "ai2-crawler-dolma", "categories": [ "ai" ], "pattern": { "accepted": [ "Ai2Bot-Dolma" ], "forbidden": [] }, "addition_date": "2024/09/16", "verification": [], "instances": { "accepted": [ "Mozilla/5.0 (compatible) Ai2Bot-Dolma (+https://www.allenai.org/crawler)" ], "rejected": [] }, "url": "https://wok.oblomov.eu/tecnologia/preparing-end-open-web/" }, { "id": "friendlycrawler", "categories": [ "ai" ], "pattern": { "accepted": [ "FriendlyCrawler" ], "forbidden": [] }, "addition_date": "2024/09/16", "verification": [], "instances": { "accepted": [ "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/605.1.15 (KHTML, like Gecko; compatible; FriendlyCrawler/1.0) Chrome/120.0.6099.216 Safari/605.1.15" ], "rejected": [] }, "url": "https://community.cloudflare.com/t/excessive-crawling-from-friendlycrawler/601127" }, { "id": "google-crawler-cloudvertex", "categories": [ "ai", "google" ], "pattern": { "accepted": [ "Google-CloudVertexBot" ], "forbidden": [] }, "addition_date": "2024/09/16", "verification": [ { "type": "cidr", "sources": [ { "type": "http-json", "url": "https://developers.google.com/static/search/apis/ipranges/googlebot.json", "selector": "$.prefixes[*][\\\"ipv6Prefix\\\",\\\"ipv4Prefix\\\"]" } ] }, { "type": "dns", "masks": [ "crawl-***-***-***-***.googlebot.com", "geo-crawl-***-***-***-***.geo.googlebot.com" ] } ], "instances": { "accepted": [ "Google-CloudVertexBot" ], "rejected": [] }, "url": "https://developers.google.com/search/docs/crawling-indexing/google-common-crawlers#google-cloudvertexbot" }, { "id": "meta-crawler", "categories": [ "ai", "meta" ], "pattern": { "accepted": [ "[Mm]eta-[Ee]xternal[Aa]gent" ], "forbidden": [] }, "addition_date": "2024/09/16", "verification": [], "instances": { "accepted": [ "meta-externalagent/1.1 (+https://developers.facebook.com/docs/sharing/webmasters/crawler)", "meta-externalagent/1.1" ], "rejected": [] }, "url": "https://developers.facebook.com/docs/sharing/webmasters/web-crawlers/#identify-2" }, { "id": "meta-externalads", "categories": [ "advertising", "meta" ], "pattern": { "accepted": [ "meta-externalads\\/" ], "forbidden": [] }, "addition_date": "2026/02/06", "verification": [], "instances": { "accepted": [ "meta-externalads/1.1 (+https://developers.facebook.com/docs/sharing/webmasters/crawler)", "meta-externalads/1.1" ], "rejected": [] }, "url": "https://developers.facebook.com/docs/sharing/webmasters/web-crawlers" }, { "id": "meta-crawler-user", "categories": [ "ai", "meta", "preview" ], "pattern": { "accepted": [ "meta-externalfetcher" ], "forbidden": [] }, "addition_date": "2024/09/16", "verification": [ { "type": "cidr", "sources": [ { "type": "http-csv", "url": "http://www.facebook.com/peering/geofeed" } ] } ], "instances": { "accepted": [ "meta-externalfetcher/1.1 (+https://developers.facebook.com/docs/sharing/webmasters/crawler)", "meta-externalfetcher/1.1" ], "rejected": [] }, "url": "https://developers.facebook.com/docs/sharing/webmasters/web-crawlers/#identify-3" }, { "id": "openai-crawler-search", "categories": [ "ai", "search-engine" ], "pattern": { "accepted": [ "OAI-SearchBot" ], "forbidden": [] }, "addition_date": "2024/09/16", "verification": [ { "type": "cidr", "sources": [ { "type": "http-json", "url": "https://openai.com/searchbot.json", "selector": "$.prefixes[*][\\\"ipv6Prefix\\\",\\\"ipv4Prefix\\\"]" } ] } ], "instances": { "accepted": [ "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko); compatible; OAI-SearchBot/1.0; +https://openai.com/searchbot" ], "rejected": [] }, "url": "https://platform.openai.com/docs/bots", "aliases": [ "OAISearchBot" ] }, { "id": "timpi-crawler", "categories": [ "ai" ], "pattern": { "accepted": [ "Timpibot" ], "forbidden": [] }, "addition_date": "2024/09/16", "verification": [], "instances": { "accepted": [ "Timpibot/0.8 (+http://www.timpi.io)" ], "rejected": [] }, "url": "https://herrbischoff.com/2022/07/timpibot-is-yet-another-badly-behaved-crawler/" }, { "id": "webzio-crawler", "categories": [ "search-engine" ], "pattern": { "accepted": [ "webzio\\s" ], "forbidden": [] }, "addition_date": "2024/09/16", "verification": [], "instances": { "accepted": [ "webzio (+https://webz.io/bot.html)" ], "rejected": [] }, "url": "https://webz.io/bot.html" }, { "id": "webzio-crawler-ai", "categories": [ "ai" ], "pattern": { "accepted": [ "webzio-extended" ], "forbidden": [] }, "addition_date": "2024/09/16", "verification": [], "instances": { "accepted": [ "webzio-extended (+https://webz.io/bot.html)" ], "rejected": [] }, "url": "https://webz.io/bot.html" }, { "id": "cohere-crawler", "categories": [ "ai" ], "pattern": { "accepted": [ "cohere-ai" ], "forbidden": [] }, "addition_date": "2024/09/16", "verification": [], "instances": { "accepted": [], "rejected": [] } }, { "id": "iask-crawler", "categories": [ "ai", "search-engine" ], "pattern": { "accepted": [ "iaskspider" ], "forbidden": [] }, "addition_date": "2024/09/16", "verification": [], "instances": { "accepted": [], "rejected": [] }, "url": "https://neil-clarke.com/block-the-bots-that-feed-ai-models-by-scraping-your-website/" }, { "id": "img2dataset", "categories": [ "ai", "tool" ], "pattern": { "accepted": [ "img2dataset" ], "forbidden": [] }, "addition_date": "2024/09/16", "verification": [], "instances": { "accepted": [ "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:72.0) Gecko/20100101 Firefox/72.0 (compatible; img2downloader; +https://github.com/rom1504/img2dataset)" ], "rejected": [] }, "url": "https://github.com/rom1504/img2dataset" }, { "id": "datadog-monitor-synthetics", "categories": [ "monitor" ], "pattern": { "accepted": [ "Datadog\\/{0,1}Synthetics" ], "forbidden": [] }, "addition_date": "2024/09/19", "verification": [ { "type": "cidr", "sources": [ { "type": "http-json", "url": "https://ip-ranges.datadoghq.com/synthetics.json", "selector": "$.synthetics[\\\"prefixes_ipv4\\\",\\\"prefixes_ipv6\\\"][*]" } ] } ], "instances": { "accepted": [ "Datadog/Synthetics", "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:72.0) Gecko/20100101 Firefox/72.0 DatadogSynthetics" ], "rejected": [] }, "url": "https://docs.datadoghq.com/synthetics/guide/identify_synthetics_bots/?tab=singleandmultistepapitests#user-agent" }, { "id": "hydrozen-monitor", "categories": [ "monitor" ], "pattern": { "accepted": [ "Hydrozen\\.io\\/" ], "forbidden": [] }, "addition_date": "2025/02/02", "verification": [ { "type": "cidr", "sources": [ { "type": "http-json", "url": "https://hydrozen.io/config/iplist.json", "selector": "$.prefixes[*][\\\"ipv6Prefix\\\",\\\"ipv4Prefix\\\"]" } ] } ], "instances": { "accepted": [ "Hydrozen.io/1.0" ], "rejected": [] }, "url": "https://docs.hydrozen.io/overview/misc/user-agent-and-ip-list" }, { "id": "imessage-preview", "categories": [ "apple", "preview", "social" ], "pattern": { "accepted": [ "facebookexternalhit", "Twitterbot", "Facebot" ], "forbidden": [] }, "addition_date": "2025/03/12", "verification": [], "instances": { "accepted": [ "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_1) AppleWebKit/601.2.4 (KHTML, like Gecko) Version/9.0.1 Safari/601.2.4 facebookexternalhit/1.1 Facebot Twitterbot/1.0" ], "rejected": [] }, "url": "https://deviceandbrowserinfo.com/learning_zone/articles/facebookexternalhit" }, { "id": "stripe-webhook", "categories": [ "webhook" ], "pattern": { "accepted": [ "Stripe\\/" ], "forbidden": [] }, "addition_date": "2025/03/13", "verification": [ { "type": "ip", "sources": [ { "type": "http-json", "url": "https://stripe.com/files/ips/ips_webhooks.json", "selector": "$.WEBHOOKS[*]" } ] } ], "instances": { "accepted": [ "Stripe/1.0 (+https://stripe.com/docs/webhooks)" ], "rejected": [] }, "url": "https://github.com/fnando/browser/issues/258" }, { "id": "stripe-crawler", "categories": [ "programmatic" ], "pattern": { "accepted": [ "Stripebot\\/" ], "forbidden": [] }, "addition_date": "2026/02/08", "verification": [ { "type": "dns", "masks": [ "@.crawl.stripe.com" ] } ], "instances": { "accepted": [ "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36 (Stripebot/1.0; +https://docs.stripe.com/stripebot-crawler)" ], "rejected": [] }, "url": "https://docs.stripe.com/stripebot-crawler" }, { "id": "postman", "categories": [ "tool" ], "pattern": { "accepted": [ "PostmanRuntime\\/" ], "forbidden": [] }, "addition_date": "2025/03/13", "verification": [], "instances": { "accepted": [ "PostmanRuntime/7.28.4" ], "rejected": [] }, "url": "https://user-agents.net/string/postmanruntime-7-28-4" }, { "id": "tiktok-crawler", "categories": [ "social", "ai" ], "pattern": { "accepted": [ "TikTokSpider" ], "forbidden": [] }, "addition_date": "2025/03/18", "verification": [], "instances": { "accepted": [ "Mozilla/5.0 (Linux; Android 5.0) AppleWebKit/537.36 (KHTML, like Gecko) Mobile Safari/537.36 (compatible; TikTokSpider; ttspider-feedback@tiktok.com)" ], "rejected": [] }, "url": "https://www.webmasterworld.com/search_engine_spiders/5118606.htm" }, { "id": "geedo-crawler-products", "categories": [ "search-engine" ], "pattern": { "accepted": [ "GeedoProductSearch" ], "forbidden": [] }, "addition_date": "2025/03/18", "verification": [ { "type": "cidr", "sources": [ { "type": "http-json", "url": "https://geedo.com/product-search/product-search-ip.json", "selector": "$.prefixes[*][\\\"ipv4Prefix\\\"]" } ] }, { "type": "dns", "masks": [ "@.geedo.com" ] } ], "instances": { "accepted": [ "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; GeedoProductSearch; +http://www.geedo.com/product-search.html) Chrome/79.0.3945.88 Safari/537.36" ], "rejected": [] }, "url": "https://geedo.com/product-search/" }, { "id": "microsoft-preview", "categories": [ "microsoft", "preview" ], "pattern": { "accepted": [ "MicrosoftPreview\\/" ], "forbidden": [] }, "addition_date": "2025/03/18", "verification": [ { "type": "cidr", "sources": [ { "type": "http-json", "url": "https://www.bing.com/toolbox/bingbot.json", "selector": "$.prefixes[*][\\\"ipv6Prefix\\\",\\\"ipv4Prefix\\\"]" } ] }, { "type": "dns", "masks": [ "@.search.msn.com" ] } ], "instances": { "accepted": [ "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; MicrosoftPreview/2.0; +https://aka.ms/MicrosoftPreview) Chrome/W.X.Y.Z Safari/537.36", "Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/W.X.Y.Z Mobile Safari/537.36 (compatible; MicrosoftPreview/2.0; +https://aka.ms/MicrosoftPreview)" ], "rejected": [] }, "url": "https://www.bing.com/webmasters/help/which-crawlers-does-bing-use-8c184ec0" }, { "id": "summaly-crawler", "categories": [ "tool" ], "pattern": { "accepted": [ "SummalyBot" ], "forbidden": [] }, "addition_date": "2025/03/18", "verification": [], "instances": { "accepted": [ "SummalyBot/5.1.0" ], "rejected": [] }, "url": "https://github.com/misskey-dev/summaly" }, { "id": "lemmy-crawler", "categories": [ "social" ], "pattern": { "accepted": [ "Lemmy" ], "forbidden": [] }, "addition_date": "2025/03/18", "verification": [], "instances": { "accepted": [ "Lemmy/0.19.8; +https://leminal.space" ], "rejected": [] }, "url": "https://leminal.space" }, { "id": "ipip-crawler", "categories": [ "tool" ], "pattern": { "accepted": [ "HTTP Banner Detection" ], "forbidden": [] }, "addition_date": "2025/03/18", "verification": [], "instances": { "accepted": [ "HTTP Banner Detection (https://security.ipip.net)" ], "rejected": [] }, "url": "https://security.ipip.net" }, { "id": "yext-bot", "categories": [ "tool" ], "pattern": { "accepted": [ "YextBot" ], "forbidden": [] }, "addition_date": "2026/02/06", "verification": [], "instances": { "accepted": [ "Mozilla/5.0 (iPhone; CPU iPhone OS 10_0 like Mac OS X) AppleWebKit/602.1.38 (KHTML, like Gecko) Version/10.0 Mobile/14A5297c YextBot/Java Safari/602.1", "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) HeadlessChrome/87.0.4280.88 YextBot/Java Safari/537.36" ], "rejected": [] }, "url": "https://www.yext.com/platform/crawl" }, { "id": "adagio-crawler", "categories": [ "advertising" ], "pattern": { "accepted": [ "Adagiobot\\/" ], "forbidden": [] }, "addition_date": "2026/02/07", "verification": [ { "type": "ip", "ips": [ "35.204.201.174", "34.125.202.46" ] } ], "instances": { "accepted": [ "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Adagiobot/1.0) Chrome/112.0.5615.137 Safari/537.36" ], "rejected": [] }, "url": "https://adagio-io.gitbook.io/adagio-documentation/general-configuration/update-your-app-ads.txt-file" }, { "id": "algolia-crawler", "categories": [ "search-engine" ], "pattern": { "accepted": [ "Algolia Crawler\\/" ], "forbidden": [] }, "addition_date": "2026/02/07", "verification": [ { "type": "ip", "ips": [ "34.66.202.43" ] } ], "instances": { "accepted": [ "Algolia Crawler/1.0.0" ], "rejected": [] }, "url": "https://support.algolia.com/hc/en/us/articles/17223698862993-Which-IP-address-of-the-Crawler-can-I-add-to-my-allowlist" }, { "id": "quantcast-crawler", "categories": [ "advertising" ], "pattern": { "accepted": [ "Quantcastbot\\/" ], "forbidden": [] }, "addition_date": "2026/02/07", "verification": [ { "type": "ip", "ips": [ "52.34.88.131", "54.71.154.151", "192.184.74.253", "192.184.75.253", "192.184.76.253", "192.184.77.253", "103.229.11.253" ] } ], "instances": { "accepted": [ "Quantcastbot/1.0 (+https://www.quantcast.com/bot)", "Quantcastbot/2.0 (+https://www.quantcast.com/bot)" ], "rejected": [] }, "url": "https://www.quantcast.com/bot" }, { "id": "adyen-webhook", "categories": [ "webhook" ], "pattern": { "accepted": [ "Adyen HttpClient" ], "forbidden": [] }, "addition_date": "2026/02/07", "verification": [ { "type": "dns", "masks": [ "out.adyen.com" ] } ], "instances": { "accepted": [ "Adyen HttpClient 1.0" ], "rejected": [] }, "url": "https://docs.adyen.com/development-resources/webhooks/secure-webhooks#domain-and-ip-addresses" }, { "id": "amazon-adbot", "categories": [ "amazon", "advertising" ], "pattern": { "accepted": [ "AmazonAdBot\\/" ], "forbidden": [] }, "addition_date": "2026/02/07", "verification": [ { "type": "dns", "masks": [ "@.amazonadbot.com" ] } ], "instances": { "accepted": [ "Mozilla/5.0 (compatible; AmazonAdBot/1.0; +https://adbot.amazon.com)" ], "rejected": [] }, "url": "https://adbot.amazon.com/" }, { "id": "ccbot-crawler", "categories": [ "archive" ], "pattern": { "accepted": [ "CCBot\\/" ], "forbidden": [] }, "addition_date": "2026/02/07", "verification": [ { "type": "dns", "masks": [ "@.crawl.commoncrawl.org" ] } ], "instances": { "accepted": [ "CCBot/2.0 (https://commoncrawl.org/faq/)" ], "rejected": [] }, "url": "https://commoncrawl.org/faq/" }, { "id": "pinterest-crawler", "categories": [ "social" ], "pattern": { "accepted": [ "Pinterest(bot)?\\/" ], "forbidden": [] }, "addition_date": "2026/02/07", "verification": [ { "type": "dns", "masks": [ "@.pinterest.com", "@.pinterestcrawler.com" ] } ], "instances": { "accepted": [ "Pinterest/0.2 (+https://www.pinterest.com/bot.html)", "Mozilla/5.0 (compatible; Pinterestbot/1.0; +https://www.pinterest.com/bot.html)", "Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2272.96 Mobile Safari/537.36 (compatible; Pinterestbot/1.0; +https://www.pinterest.com/bot.html)" ], "rejected": [] }, "url": "https://help.pinterest.com/en/business/article/pinterestbot" }, { "id": "ahrefs-site-audit", "categories": [ "advertising" ], "pattern": { "accepted": [ "AhrefsSiteAudit\\/" ], "forbidden": [] }, "addition_date": "2026/02/07", "verification": [ { "type": "cidr", "sources": [ { "type": "http-json", "url": "https://api.ahrefs.com/v3/public/crawler-ip-ranges", "selector": "$.prefixes[*].ipv4Prefix" } ] } ], "instances": { "accepted": [ "Mozilla/5.0 (compatible; AhrefsSiteAudit/6.1; +http://ahrefs.com/robot/site-audit)", "Mozilla/5.0 (Linux; Android 13) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/108.0.5359.128 Mobile Safari/537.36 (compatible; AhrefsSiteAudit/6.1; +http://ahrefs.com/robot/site-audit)" ], "rejected": [] }, "url": "http://ahrefs.com/robot/site-audit" }, { "id": "perplexity-user", "categories": [ "ai" ], "pattern": { "accepted": [ "Perplexity-User\\/" ], "forbidden": [] }, "addition_date": "2026/02/07", "verification": [ { "type": "cidr", "sources": [ { "type": "http-json", "url": "https://www.perplexity.ai/perplexity-user.json", "selector": "$.prefixes[*].ipv4Prefix" } ] } ], "instances": { "accepted": [ "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Perplexity-User/1.0; +https://perplexity.ai/perplexity-user)" ], "rejected": [] }, "url": "https://docs.perplexity.ai/docs/resources/perplexity-crawlers" }, { "id": "newrelic-monitor", "categories": [ "monitor" ], "pattern": { "accepted": [ "NewRelicbot\\/" ], "forbidden": [] }, "addition_date": "2026/02/07", "verification": [ { "type": "cidr", "sources": [ { "type": "http-json", "url": "https://s3.amazonaws.com/nr-synthetics-assets/nat-ip-dnsname/production/ip-ranges.json", "selector": "$.*[*]" } ] } ], "instances": { "accepted": [ "Mozilla/5.0 (compatible; NewRelicbot/2.1; +http://www.newrelic.com)" ], "rejected": [] }, "url": "http://www.newrelic.com" }, { "id": "amazon-route53-health-check", "categories": [ "amazon", "monitor" ], "pattern": { "accepted": [ "Amazon Route 53 Health Check Service" ], "forbidden": [] }, "addition_date": "2026/02/07", "verification": [ { "type": "cidr", "sources": [ { "type": "http-json", "url": "https://ip-ranges.amazonaws.com/ip-ranges.json", "selector": "$.prefixes[?(@.service==\"ROUTE53_HEALTHCHECKS\")].ip_prefix" } ] } ], "instances": { "accepted": [ "Amazon Route 53 Health Check Service; ref:47d9bc51-39d6-4cd9-9a7f-4c981c5db165; report http://amzn.to/1vsZADi" ], "rejected": [] }, "url": "http://amzn.to/1vsZADi" }, { "id": "checkly-monitor", "categories": [ "monitor" ], "pattern": { "accepted": [ "Checkly\\/" ], "forbidden": [] }, "addition_date": "2026/02/07", "verification": [ { "type": "ip", "sources": [ { "type": "http-json", "url": "https://api.checklyhq.com/v1/static-ips", "selector": "$[*]" } ] } ], "instances": { "accepted": [ "Checkly/1.0" ], "rejected": [] }, "url": "https://www.checklyhq.com/docs/platform/allowlisting-traffic/" }, { "id": "statuscake-monitor", "categories": [ "monitor" ], "pattern": { "accepted": [ "StatusCake" ], "forbidden": [] }, "addition_date": "2026/02/07", "verification": [ { "type": "ip", "sources": [ { "type": "http-json", "url": "https://app.statuscake.com/Workfloor/Locations.php?format=json", "selector": "$[*].ip" } ] } ], "instances": { "accepted": [ "StatusCake/1.0" ], "rejected": [] }, "url": "https://www.statuscake.com/kb/knowledge-base/what-are-your-ips/" }, { "id": "sentry-uptime-monitor", "categories": [ "monitor" ], "pattern": { "accepted": [ "SentryUptimeBot\\/1\\.0" ], "forbidden": [] }, "addition_date": "2026/02/07", "verification": [ { "type": "ip", "sources": [ { "type": "http-text", "url": "https://sentry.io/api/0/uptime-ips/" } ] } ], "instances": { "accepted": [ "SentryUptimeBot/1.0 (+http://docs.sentry.io/product/uptime-monitoring/)" ], "rejected": [] }, "url": "https://docs.sentry.io/security-legal-pii/security/ip-ranges/#uptime-monitoring" }, { "id": "facebook-catalog", "categories": [ "meta", "social" ], "pattern": { "accepted": [ "facebookcatalog\\/" ], "forbidden": [] }, "addition_date": "2024/10/03", "verification": [ { "type": "cidr", "sources": [ { "type": "http-csv", "url": "http://www.facebook.com/peering/geofeed" } ] } ], "instances": { "accepted": [ "facebookcatalog/1.0" ], "rejected": [] }, "url": "https://developers.facebook.com/docs/sharing/webmasters/web-crawlers" }, { "id": "keybase-bot", "categories": [ "preview", "social" ], "pattern": { "accepted": [ "KeybaseBot" ], "forbidden": [] }, "addition_date": "2024/10/21", "verification": [], "instances": { "accepted": [ "Mozilla/5.0 (compatible; KeybaseBot; +https://keybase.io)" ], "rejected": [] }, "url": "https://book.keybase.io/docs/chat/link-previews" }, { "id": "academicbot-rtu", "categories": [ "academic" ], "pattern": { "accepted": [ "AcademicBotRTU" ], "forbidden": [] }, "addition_date": "2024/10/17", "verification": [], "instances": { "accepted": [ "AcademicBotRTU (https://academicbot.rtu.lv; mailto:caps@rtu.lv)" ], "rejected": [] }, "url": "https://academicbot.rtu.lv" }, { "id": "cookiehub-scan", "categories": [ "tool" ], "pattern": { "accepted": [ "CookieHubScan" ], "forbidden": [] }, "addition_date": "2024/11/29", "verification": [ { "type": "ip", "sources": [ { "type": "http-json", "url": "https://dash.cookiehub.com/scanner-ip/json", "selector": "$[*]" } ] } ], "instances": { "accepted": [ "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36 CookieHubScan/3.0" ], "rejected": [] }, "url": "https://www.cookiehub.com/" }, { "id": "cloudflare-healthchecks", "categories": [ "monitor" ], "pattern": { "accepted": [ "Cloudflare-Healthchecks" ], "forbidden": [] }, "addition_date": "2024/12/17", "verification": [ { "type": "cidr", "sources": [ { "type": "http-text", "url": "https://www.cloudflare.com/ips-v4/" }, { "type": "http-text", "url": "https://www.cloudflare.com/ips-v6/" } ] } ], "instances": { "accepted": [ "Mozilla/5.0 (compatible; Cloudflare-Healthchecks/1.0; +https://www.cloudflare.com/; healthcheck-id: AAAAAAAAAAAAAAAA)" ], "rejected": [] }, "url": "https://developers.cloudflare.com/health-checks/" }, { "id": "cloudflare-traffic-manager", "categories": [ "monitor" ], "pattern": { "accepted": [ "Cloudflare-Traffic-Manager" ], "forbidden": [] }, "addition_date": "2024/12/17", "verification": [ { "type": "cidr", "sources": [ { "type": "http-text", "url": "https://www.cloudflare.com/ips-v4/" }, { "type": "http-text", "url": "https://www.cloudflare.com/ips-v6/" } ] } ], "instances": { "accepted": [ "Mozilla/5.0 (compatible; Cloudflare-Traffic-Manager/1.0; +https://www.cloudflare.com/traffic-manager/; pool-id: AAAAAAAAAAAAAAAA)" ], "rejected": [] }, "url": "https://developers.cloudflare.com/load-balancing/monitors/" }, { "id": "cloudflare-prefetch", "categories": [ "optimizer" ], "pattern": { "accepted": [ "CloudFlare-Prefetch" ], "forbidden": [] }, "addition_date": "2024/12/17", "verification": [ { "type": "cidr", "sources": [ { "type": "http-text", "url": "https://www.cloudflare.com/ips-v4/" }, { "type": "http-text", "url": "https://www.cloudflare.com/ips-v6/" } ] } ], "instances": { "accepted": [ "Mozilla/5.0 (compatible; CloudFlare-Prefetch/0.1; +http://www.cloudflare.com/)" ], "rejected": [] }, "url": "https://developers.cloudflare.com/speed/optimization/content/prefetch-urls/" }, { "id": "cloudflare-ssl-detector", "categories": [ "monitor" ], "pattern": { "accepted": [ "Cloudflare-SSLDetector" ], "forbidden": [] }, "addition_date": "2024/12/17", "verification": [ { "type": "cidr", "sources": [ { "type": "http-text", "url": "https://www.cloudflare.com/ips-v4/" }, { "type": "http-text", "url": "https://www.cloudflare.com/ips-v6/" } ] } ], "instances": { "accepted": [ "Cloudflare-SSLDetector" ], "rejected": [] }, "url": "https://developers.cloudflare.com/ssl/origin-configuration/ssl-tls-recommender/" }, { "id": "cloudflare-security-center", "categories": [ "monitor" ], "pattern": { "accepted": [ "https:\\/\\/developers\\.cloudflare\\.com\\/security-center\\/" ], "forbidden": [] }, "addition_date": "2024/12/17", "verification": [ { "type": "cidr", "sources": [ { "type": "http-text", "url": "https://www.cloudflare.com/ips-v4/" }, { "type": "http-text", "url": "https://www.cloudflare.com/ips-v6/" } ] } ], "instances": { "accepted": [ "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/92.0.4515.107 Safari/537.36 (compatible; +https://developers.cloudflare.com/security-center/)" ], "rejected": [] }, "url": "https://developers.cloudflare.com/security-center/" }, { "id": "oncrawl", "categories": [ "optimizer", "tool" ], "pattern": { "accepted": [ "OnCrawl\\/" ], "forbidden": [] }, "addition_date": "2025/03/27", "verification": [ { "type": "ip", "ips": [ "35.187.15.33", "35.187.69.66", "104.199.98.197", "35.199.9.128", "35.230.188.244", "35.245.139.31" ] } ], "instances": { "accepted": [ "Mozilla/5.0 (compatible; OnCrawl/1.0; +http://www.oncrawl.com)", "Mozilla/5.0 (iPhone; CPU iPhone OS 8_3 like Mac OS X) AppleWebKit/600.1.4 (KHTML, like Gecko) Version/8.0 Mobile/12F70 Safari/600.1.4 (compatible; OnCrawl/1.0; +http://www.oncrawl.com)" ], "rejected": [] }, "url": "https://help.oncrawl.com/en/articles/2288662-what-ips-does-oncrawl-use-to-crawl-a-website" }, { "id": "got", "categories": [ "tool" ], "pattern": { "accepted": [ "sindresorhus\\/got" ], "forbidden": [] }, "addition_date": "2025/04/22", "verification": [], "instances": { "accepted": [ "got (https://github.com/sindresorhus/got)" ], "rejected": [] }, "url": "https://github.com/sindresorhus/got" }, { "id": "censys-inspect", "categories": [ "monitor" ], "pattern": { "accepted": [ "CensysInspect\\/" ], "forbidden": [] }, "addition_date": "2025/04/22", "verification": [ { "type": "ip", "ips": [ "66.132.159.0/24", "162.142.125.0/24", "167.94.138.0/24", "167.94.145.0/24", "167.94.146.0/24", "167.248.133.0/24", "199.45.154.0/24", "199.45.155.0/24", "206.168.34.0/24", "206.168.35.0/24", "2602:80d:1000:b0cc:e::/80", "2620:96:e000:b0cc:e::/80", "2602:80d:1003::/112", "2602:80d:1004::/112" ] } ], "instances": { "accepted": [ "Mozilla/5.0 (compatible; CensysInspect/1.1; +https://about.censys.io/)" ], "rejected": [] }, "url": "https://docs.censys.com/docs/opt-out-of-data-collection" }, { "id": "sbintuitions-bot", "categories": [ "ai" ], "pattern": { "accepted": [ "SBIntuitionsBot\\/" ], "forbidden": [] }, "addition_date": "2025/04/23", "verification": [], "instances": { "accepted": [ "Mozilla/5.0 (compatible; SBIntuitionsBot/0.1; +https://www.sbintuitions.co.jp/bot/)" ], "rejected": [] }, "url": "https://www.sbintuitions.co.jp/bot/" }, { "id": "sitebulb", "categories": [ "optimizer", "tool" ], "pattern": { "accepted": [ "sitebulb" ], "forbidden": [] }, "addition_date": "2025/04/30", "verification": [], "instances": { "accepted": [ "Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.6439.0 Mobile Safari/537.36 +https://sitebulb.com" ], "rejected": [] }, "url": "https://sitebulb.com/" } ]