Copied: velocity/tools/trunk/velocity-tools-browser/src/test/resources/devices.txt (from r1765879, velocity/tools/trunk/velocity-tools-browser/src/test/resources/robots.txt) URL: http://svn.apache.org/viewvc/velocity/tools/trunk/velocity-tools-browser/src/test/resources/devices.txt?p2=velocity/tools/trunk/velocity-tools-browser/src/test/resources/devices.txt&p1=velocity/tools/trunk/velocity-tools-browser/src/test/resources/robots.txt&r1=1765879&r2=1768140&rev=1768140&view=diff ============================================================================== --- velocity/tools/trunk/velocity-tools-browser/src/test/resources/robots.txt (original) +++ velocity/tools/trunk/velocity-tools-browser/src/test/resources/devices.txt Sat Nov 5 01:39:24 2016 @@ -1,715 +1,3791 @@ -AppEngine-Google # AppEngine-Google; (+http://code.google.com/appengine; appid: webetrex) -AppEngine-Google # AppEngine-Google; (+http://code.google.com/appengine; appid: unblock4myspace) -AppEngine-Google # AppEngine-Google; (+http://code.google.com/appengine; appid: tunisproxy) -AppEngine-Google # AppEngine-Google; (+http://code.google.com/appengine; appid: proxy-in-rs) -AppEngine-Google # AppEngine-Google; (+http://code.google.com/appengine; appid: proxy-ba-k) -AppEngine-Google # AppEngine-Google; (+http://code.google.com/appengine; appid: moelonepyaeshan) -AppEngine-Google # AppEngine-Google; (+http://code.google.com/appengine; appid: mirrorrr) -AppEngine-Google # AppEngine-Google; (+http://code.google.com/appengine; appid: mapremiereapplication) -AppEngine-Google # AppEngine-Google; (+http://code.google.com/appengine; appid: longbows-hideout) -AppEngine-Google # AppEngine-Google; (+http://code.google.com/appengine; appid: eduas23) -AppEngine-Google # AppEngine-Google; (+http://code.google.com/appengine; appid: craigserver) -AppEngine-Google # AppEngine-Google; ( http://code.google.com/appengine; appid: proxy-ba-k) -ABACHOBot # ABACHOBot -Accoona-AI-Agent 1.1.2 # Accoona-AI-Agent/1.1.2 (aicrawler at accoonabot dot com) -Accoona-AI-Agent 1.1.2 # Accoona-AI-Agent/1.1.2 -Accoona-AI-Agent 1.1.1 # Accoona-AI-Agent/1.1.1 (crawler at accoona dot com) -AddSugarSpiderBot # Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.0) AddSugarSpiderBot www.idealobserver.com -AnyApexBot 1.0 # Mozilla/5.0 (compatible; AnyApexBot/1.0; +http://www.anyapex.com/bot.html) -Arachmo # Mozilla/4.0 (compatible; Arachmo) -B-l-i-t-z-B-O-T # Mozilla/4.0 (compatible; B-l-i-t-z-B-O-T) -Baiduspider 2.0 # Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html) -Baiduspider # Baiduspider+(+http://www.baidu.com/search/spider_jp.html) -Baiduspider # Baiduspider+(+http://www.baidu.com/search/spider.htm) -Baiduspider # BaiDuSpider -BecomeBot 3.0 # Mozilla/5.0 (compatible; BecomeBot/3.0; MSIE 6.0 compatible; +http://www.become.com/site_owners.html) -BecomeBot 2.3 # Mozilla/5.0 (compatible; BecomeBot/2.3; MSIE 6.0 compatible; +http://www.become.com/site_owners.html) -BeslistBot 1.0 # Mozilla/5.0 (compatible; BeslistBot; nl; BeslistBot 1.0; http://www.beslist.nl/ -BillyBobBot 1.0 # BillyBobBot/1.0 (+http://www.billybobbot.com/crawler/) -Bimbot 1.0 # Bimbot/1.0 -Bingbot 2.0 # Mozilla/5.0 (compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm) -Bingbot 2.0 # Mozilla/5.0 (compatible; bingbot/2.0 +http://www.bing.com/bingbot.htm) -BlitzBOT # Mozilla/4.0 (compatible; BlitzBot) -BlitzBOT # [email protected] (Mozilla compatible) -BlitzBOT # [email protected] (Mozilla compatible) -boitho.com-dc 0.85 # boitho.com-dc/0.85 ( http://www.boitho.com/dcbot.html ) -boitho.com-dc 0.83 # boitho.com-dc/0.83 ( http://www.boitho.com/dcbot.html ) -boitho.com-dc 0.82 # boitho.com-dc/0.82 ( http://www.boitho.com/dcbot.html ) -boitho.com-dc 0.81 # boitho.com-dc/0.81 ( http://www.boitho.com/dcbot.html ) -boitho.com-dc 0.79 # boitho.com-dc/0.79 ( http://www.boitho.com/dcbot.html ) -boitho.com-robot 1.1 # boitho.com-robot/1.1 -boitho.com-robot 1.0 # boitho.com-robot/1.0 -btbot 0.4 # btbot/0.4 (+http://www.btbot.com/btbot.html) -CatchBot 2.0 # CatchBot/2.0; +http://www.catchbot.com -CatchBot 1.0 # CatchBot/1.0; +http://www.catchbot.com -CatchBot 1.0 # CatchBot/1.0; http://www.catchbot.com -Cerberian Drtrs 3.2 # Mozilla/4.0 (compatible; Cerberian Drtrs Version-3.2-Build-1) -Cerberian Drtrs 3.2 # Mozilla/4.0 (compatible; Cerberian Drtrs Version-3.2-Build-0) -Charlotte 1.1 # Mozilla/5.0 (compatible; Charlotte/1.1; http://www.searchme.com/support/) -Charlotte 1.0t # Mozilla/5.0 (compatible; Charlotte/1.0t; http://www.searchme.com/support/) -Charlotte 1.0b # Mozilla/5.0 (compatible; Charlotte/1.0b; http://www.searchme.com/support/) -Charlotte 1.0b # Mozilla/5.0 (compatible; Charlotte/1.0b; http://www.betaspider.com/) -Charlotte 0.9t # Mozilla/5.0 (X11; U; Linux i686 (x86_64); en-US; rv:1.8.1.11) Gecko/20080109 (Charlotte/0.9t; http://www.searchme.com/support/) (Charlotte/0.9t; http://www.searchme.com/support/) -Charlotte 0.9t # Mozilla/5.0 (X11; U; Linux i686 (x86_64); en-US; rv:1.8.1.11) Gecko/20080109 (Charlotte/0.9t; http://www.searchme.com/support/) -Charlotte 0.9t # Mozilla/5.0 (compatible; Charlotte/0.9t; http://www.searchme.com/support/) -Charlotte 0.9t # Mozilla/5.0 (compatible; Charlotte/0.9t; +http://www.searchme.com/support/) -ConveraCrawler 0.9e # ConveraCrawler/0.9e (+http://ews.converasearch.com/crawl.htm) -ConveraCrawler 0.9d # ConveraCrawler/0.9d (+http://www.authoritativeweb.com/crawl) -ConveraCrawler 0.9d # ConveraCrawler/0.9d ( http://www.authoritativeweb.com/crawl) -ConveraCrawler 0.9 # ConveraCrawler/0.9 (+http://www.authoritativeweb.com/crawl) -cosmos 0.9 # cosmos/0.9_([email protected]) -DataparkSearch 4.37 # DataparkSearch/4.37-23012006 ( http://www.dataparksearch.org/) -DataparkSearch 4.36 # DataparkSearch/4.36 ( http://www.dataparksearch.org/) -DataparkSearch 4.35 # DataparkSearch/4.35-02122005 ( http://www.dataparksearch.org/) -DataparkSearch 4.35 # DataparkSearch/4.35 ( http://www.dataparksearch.org/) -DiamondBot # DiamondBot -Discobot 1.0 # Mozilla/5.0 (compatible; discobot/1.0; +http://discoveryengine.com/discobot.html) -Dotbot 1.1 # Mozilla/5.0 (compatible; DotBot/1.1; http://www.dotnetdotcom.org/, [email protected]) -Dotbot 1.0.1 # DotBot/1.0.1 (http://www.dotnetdotcom.org/#info, [email protected]) -EmeraldShield.com WebBot # EmeraldShield.com WebBot (http://www.emeraldshield.com/webbot.aspx) -envolk[ITS]spider 1.6 # envolk[ITS]spider/1.6 (+http://www.envolk.com/envolkspider.html) -envolk[ITS]spider 1.6 # envolk[ITS]spider/1.6 ( http://www.envolk.com/envolkspider.html) -EsperanzaBot # EsperanzaBot(+http://www.esperanza.to/bot/) -Exabot 2.0 # Exabot/2.0 -FAST Enterprise Crawler 6 # FAST Enterprise Crawler 6 used by Schibsted ([email protected]) -FAST Enterprise Crawler 6 # FAST Enterprise Crawler 6 / Scirus [email protected]; http://www.scirus.com/srsapp/contactus/ -FAST Enterprise Crawler 6 # FAST Enteprise Crawler/6 (www dot fastsearch dot com) -FAST-WebCrawler 3.8 # FAST-WebCrawler/3.8 (atw-crawler at fast dot no; http://fast.no/support/crawler.asp) -FAST-WebCrawler 3.7 # FAST-WebCrawler/3.7/FirstPage (atw-crawler at fast dot no;http://fast.no/support/crawler.asp) -FAST-WebCrawler 3.7 # FAST-WebCrawler/3.7 (atw-crawler at fast dot no; http://fast.no/support/crawler.asp) -FAST-WebCrawler 3.6 # FAST-WebCrawler/3.6/FirstPage (atw-crawler at fast dot no;http://fast.no/support/crawler.asp) -FAST-WebCrawler 3.6 # FAST-WebCrawler/3.6 (atw-crawler at fast dot no; http://fast.no/support/crawler.asp) -FAST-WebCrawler 3.6 # FAST-WebCrawler/3.6 -FAST-WebCrawler 3.x # FAST-WebCrawler/3.x Multimedia (mm dash crawler at fast dot no) -FAST-WebCrawler 3.x # FAST-WebCrawler/3.x Multimedia -FDSE robot # Mozilla/4.0 (compatible: FDSE robot) -FindLinks 2.0.1 # findlinks/2.0.1 (+http://wortschatz.uni-leipzig.de/findlinks/) -FindLinks 1.1.6-beta6 # findlinks/1.1.6-beta6 (+http://wortschatz.uni-leipzig.de/findlinks/) -FindLinks 1.1.6-beta4 # findlinks/1.1.6-beta4 (+http://wortschatz.uni-leipzig.de/findlinks/) -FindLinks 1.1.6-beta1 # findlinks/1.1.6-beta1 (+http://wortschatz.uni-leipzig.de/findlinks/) -FindLinks 1.1.5-beta7 # findlinks/1.1.5-beta7 (+http://wortschatz.uni-leipzig.de/findlinks/) -FindLinks 1.1.4-beta1 # findlinks/1.1.4-beta1 (+http://wortschatz.uni-leipzig.de/findlinks/) -FindLinks 1.1.3-beta9 # findlinks/1.1.3-beta9 (+http://wortschatz.uni-leipzig.de/findlinks/) -FindLinks 1.1.3-beta8 # findlinks/1.1.3-beta8 (+http://wortschatz.uni-leipzig.de/findlinks/) -FindLinks 1.1.3-beta6 # findlinks/1.1.3-beta6 (+http://wortschatz.uni-leipzig.de/findlinks/) -FindLinks 1.1.3-beta4 # findlinks/1.1.3-beta4 (+http://wortschatz.uni-leipzig.de/findlinks/) -FindLinks 1.1.3-beta2 # findlinks/1.1.3-beta2 (+http://wortschatz.uni-leipzig.de/findlinks/) -FindLinks 1.1.3-beta1 # findlinks/1.1.3-beta1 (+http://wortschatz.uni-leipzig.de/findlinks/) -FindLinks 1.1.2-a5 # findlinks/1.1.2-a5 (+http://wortschatz.uni-leipzig.de/findlinks/) -FindLinks 1.1.1-a5 # findlinks/1.1.1-a5 (+http://wortschatz.uni-leipzig.de/findlinks/) -FindLinks 1.1.1-a1 # findlinks/1.1.1-a1 (+http://wortschatz.uni-leipzig.de/findlinks/) -FindLinks 1.1.1 # findlinks/1.1.1 (+http://wortschatz.uni-leipzig.de/findlinks/) -FindLinks 1.1-a9 # findlinks/1.1-a9 (+http://wortschatz.uni-leipzig.de/findlinks/) -FindLinks 1.1-a8 # findlinks/1.1-a8 (+http://wortschatz.uni-leipzig.de/findlinks/) -FindLinks 1.1-a8 # findlinks/1.1-a8 ( http://wortschatz.uni-leipzig.de/findlinks/) -FindLinks 1.1-a7 # findlinks/1.1-a7 (+http://wortschatz.uni-leipzig.de/findlinks/) -FindLinks 1.1-a5 # findlinks/1.1-a5 (+http://wortschatz.uni-leipzig.de/findlinks/) -FindLinks 1.1-a4 # findlinks/1.1-a4 (+http://wortschatz.uni-leipzig.de/findlinks/) -FindLinks 1.1-a3 # findlinks/1.1-a3 (+http://wortschatz.uni-leipzig.de/findlinks/) -FindLinks 1.1 # findlinks/1.1 (+http://wortschatz.uni-leipzig.de/findlinks/) -FindLinks 1.06 # findlinks/1.06 (+http://wortschatz.uni-leipzig.de/findlinks/) -FindLinks 1.0.9 # findlinks/1.0.9 (+http://wortschatz.uni-leipzig.de/findlinks/) -FindLinks 1.0.8 # findlinks/1.0.8 (+http://wortschatz.uni-leipzig.de/findlinks/) -FindLinks 1.0 # findlinks/1.0 (+http://wortschatz.uni-leipzig.de/findlinks/) -FurlBot Furl Search 2.0 # Mozilla/4.0 compatible FurlBot/Furl Search 2.0 (FurlBot; http://www.furl.net; [email protected]) -FyberSpider # FyberSpider (+http://www.fybersearch.com/fyberspider.php) -FyberSpider # FyberSpider -g2crawler # g2Crawler [email protected] -Gaisbot 3.0+ # Gaisbot/3.0+([email protected];+http://gais.cs.ccu.edu.tw/robot.php) -Gaisbot 3.0 # Gaisbot/3.0+([email protected];+http://gais.cs.ccu.edu.tw/robot.php) -Gaisbot 3.0 # Gaisbot/3.0 ([email protected]; http://gais.cs.ccu.edu.tw/robot.php) -GalaxyBot 1.0 # GalaxyBot/1.0 (http://www.galaxy.com/galaxybot.html) -genieBot # genieBot (http://64.5.245.11/faq/faq.html) -genieBot # genieBot ((http://64.5.245.11/faq/faq.html)) -Gigabot 3.0 # Gigabot/3.0 (http://www.gigablast.com/spider.html) -Gigabot 2.0 # Gigabot/2.0/gigablast.com/spider.html -Gigabot 2.0 # Gigabot/2.0 (http://www.gigablast.com/spider.html) -Gigabot 2.0 # Gigabot/2.0 -Gigabot 1.0 # Gigabot/1.0 -Girafabot # Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.2; SV1; .NET CLR 1.1.4322; Girafabot [girafa.com]) -Girafabot # Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 4.0; Girafabot; girafabot at girafa dot com; http://www.girafa.com) -Girafabot # Mozilla/4.0 (compatible; MSIE 5.0; Windows NT; Girafabot; girafabot at girafa dot com; http://www.girafa.com) -Googlebot 2.1 # Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html) -Googlebot 2.1 # Googlebot/2.1 (+http://www.googlebot.com/bot.html) -Googlebot 2.1 # Googlebot/2.1 (+http://www.google.com/bot.html) -Googlebot-Image 1.0 # Googlebot-Image/1.0 -GurujiBot 1.0 # Mozilla/5.0 GurujiBot/1.0 (+http://www.guruji.com/en/WebmasterFAQ.html) -GurujiBot 1.0 # Mozilla/5.0 GurujiBot/1.0 ( http://www.guruji.com/en/WebmasterFAQ.html) -GurujiBot 1.0 # Mozilla/5.0 (compatible; GurujiBot/1.0; +http://www.guruji.com/en/WebmasterFAQ.html) -GurujiBot 1.0 # GurujiBot/1.0 (+http://www.guruji.com/WebmasterFAQ.html) -GurujiBot 1.0 # GurujiBot/1.0 (+http://www.guruji.com/en/WebmasterFAQ.html) -HappyFunBot 1.1 # HappyFunBot/1.1 ( http://www.happyfunsearch.com/bot.html) -Holmes 3.9 # holmes/3.9 (someurl.co.cc) -Holmes 3.12.4 # holmes/3.12.4 (http://morfeo.centrum.cz/bot) -Holmes 3.12.3 # holmes/3.12.3 (http://morfeo.centrum.cz/bot) -Holmes 3.12.2 # holmes/3.12.2 (http://morfeo.centrum.cz/bot) -Holmes 3.12.1 # holmes/3.12.1 (http://morfeo.centrum.cz/bot) -htdig 3.1.6 # htdig/3.1.6 ([email protected]) -htdig 3.1.6 # htdig/3.1.6 ([email protected]) -htdig 3.1.5 # htdig/3.1.5 ([email protected]) -htdig 3.1.5 # htdig/3.1.5 (root@localhost) -htdig 3.1.5 # htdig/3.1.5 ([email protected]) -htdig 3.1.5 # htdig/3.1.5 -iaskspider 2.0 # iaskspider/2.0(+http://iask.com/help/help_index.html) -iaskspider # iaskspider -ia_archiver 8.9 # ia_archiver/8.9 (Windows NT 3.1; en-US;) -ia_archiver 8.9 # ia_archiver/8.9 (Windows 3.9; en-US;) -ia_archiver 8.9 # ia_archiver/8.9 (Linux 1.0; en-US;) -ia_archiver 8.8 # ia_archiver/8.8 (Windows XP 7.2; en-US;) -ia_archiver 8.8 # ia_archiver/8.8 (Windows XP 3.0; en-US;) -ia_archiver 8.2 # ia_archiver/8.2 (Windows 7.6; en-US;) -ia_archiver 8.1 # ia_archiver/8.1 (Windows 2000 1.9; en-US;) -ia_archiver 8.0 # ia_archiver/8.0 (Windows 2.4; en-US;) -ia_archiver # ia_archiver -iCCrawler # iCCrawler (http://www.iccenter.net/bot.htm) -ichiro 4.0 # ichiro/4.0 (http://help.goo.ne.jp/door/crawler.html) -ichiro 3.0 # ichiro/3.0 (http://help.goo.ne.jp/door/crawler.html) -ichiro 2.0 # ichiro/2.0+(http://help.goo.ne.jp/door/crawler.html) -ichiro 2.0 # ichiro/2.0 ([email protected]) -ichiro 2.0 # ichiro/2.0 (http://help.goo.ne.jp/door/crawler.html) -igdeSpyder # igdeSpyder (compatible; igde.ru; +http://igde.ru/doc/tech.html) -IRLbot 3.0 # IRLbot/3.0 (compatible; MSIE 6.0; http://irl.cs.tamu.edu/crawler/) -IRLbot 3.0 # IRLbot/3.0 (compatible; MSIE 6.0; http://irl.cs.tamu.edu/crawler) -IRLbot 2.0 # IRLbot/2.0 (compatible; MSIE 6.0; http://irl.cs.tamu.edu/crawler) -IRLbot 2.0 # IRLbot/2.0 (+http://irl.cs.tamu.edu/crawler) -IRLbot 2.0 # IRLbot/2.0 ( http://irl.cs.tamu.edu/crawler) -IssueCrawler # IssueCrawler -Jaxified Bot # Jaxified Bot (+http://www.jaxified.com/crawler/) -Jyxobot 1 # Jyxobot/1 -KoepaBot # Mozilla/5.0 (compatible; KoepaBot BETA; http://www.koepa.nl/bot.html) -LapozzBot 1.4 # LapozzBot/1.4 (+http://robot.lapozz.com) -Larbin 5.0 # Mozilla/5.0 [email protected] -Larbin 2.6.3 # larbin_2.6.3 [email protected] -Larbin 2.6.3 # larbin_2.6.3 [email protected] -Larbin 2.6.3 # larbin_2.6.3 [email protected] -Larbin 2.6.3 # larbin_2.6.3 [email protected] -Larbin 2.6.3 # larbin_2.6.3 [email protected] -Larbin 2.6.3 # larbin_2.6.3 [email protected] -Larbin 2.6.3 # larbin_2.6.3 [email protected] -Larbin 2.6.3 # larbin_2.6.3 ([email protected]) -Larbin 2.6.3 # larbin_2.6.3 ([email protected]) -Larbin 2.6.3 # larbin_2.6.3 ([email protected]) -Larbin 2.6.3 # larbin_2.6.3 ([email protected]) -Larbin 2.6.2 # larbin_2.6.2 [email protected] -Larbin 2.6.2 # larbin_2.6.2 [email protected] -Larbin 2.6.2 # larbin_2.6.2 listonATccDOTgatechDOTedu -Larbin 2.6.2 # larbin_2.6.2 [email protected] -Larbin 2.6.2 # larbin_2.6.2 [email protected] -Larbin 2.6.2 # larbin_2.6.2 [email protected] -Larbin 2.6.2 # larbin_2.6.2 [email protected] -Larbin 2.6.2 # larbin_2.6.2 ([email protected]) -Larbin 2.6.2 # larbin_2.6.2 ([email protected]) -Larbin 2.6.2 # larbin_2.6.2 ([email protected]) -Larbin 2.6.2 # larbin_2.6.2 ([email protected]) -Larbin 2.6.1 # larbin_2.6.1 [email protected] -Larbin 2.5.0 # larbin_2.5.0 ([email protected]) -Larbin xy250 # larbin_xy250 [email protected] -LDSpider # ldspider (http://code.google.com/p/ldspider/wiki/Robots) -LexxeBot 1.0 # LexxeBot/1.0 ([email protected]) -Linguee Bot # Linguee Bot (http://www.linguee.com/bot; [email protected]) -LinkWalker 2.0 # LinkWalker/2.0 -LinkWalker # LinkWalker -lmspider # lmspider [email protected] -lmspider # lmspider ([email protected]) -lwp-trivial 1.41 # lwp-trivial/1.41 -lwp-trivial 1.38 # lwp-trivial/1.38 -lwp-trivial 1.36 # lwp-trivial/1.36 -lwp-trivial 1.35 # lwp-trivial/1.35 -lwp-trivial 1.33 # lwp-trivial/1.33 -magpie-crawler 1.1 # magpie-crawler/1.1 (U; Linux amd64; en-GB; +http://www.brandwatch.net) -Mediapartners-Google 2.1 # Mediapartners-Google/2.1 -MJ12bot 1.2.4 # Mozilla/5.0 (compatible; MJ12bot/v1.2.4; http://www.majestic12.co.uk/bot.php?+) -MJ12bot 1.2.3 # Mozilla/5.0 (compatible; MJ12bot/v1.2.3; http://www.majestic12.co.uk/bot.php?+) -MJ12bot 1.0.8 # MJ12bot/v1.0.8 (http://majestic12.co.uk/bot.php?+) -MJ12bot 1.0.7 # MJ12bot/v1.0.7 (http://majestic12.co.uk/bot.php?+) -MJ12bot 1.0.6 # MJ12bot/v1.0.6 (http://majestic12.co.uk/bot.php?+) -MJ12bot 1.0.5 # MJ12bot/v1.0.5 (http://majestic12.co.uk/bot.php?+) -Mnogosearch 3.1.21 # Mnogosearch-3.1.21 -MojeekBot 2.0 # Mozilla/5.0 (compatible; MojeekBot/2.0; http://www.mojeek.com/bot.html) -MojeekBot 0.2 # MojeekBot/0.2 (archi; http://www.mojeek.com/bot.html) -Moreoverbot 5.1 # Moreoverbot/5.1 ( http://w.moreover.com; [email protected]) Mozilla/5.0 -Moreoverbot 5.00 # Moreoverbot/5.00 (+http://www.moreover.com; [email protected]) -Moreoverbot 5.00 # Moreoverbot/5.00 (+http://www.moreover.com) -Morning Paper 1.0 # Morning Paper 1.0 (robots.txt compliant!) -msnbot 2.1 # msnbot/2.1 -msnbot 2.0b # msnbot/2.0b -msnbot 1.1 # msnbot/1.1 (+http://search.msn.com/msnbot.htm) -msnbot 1.1 # msnbot/1.1 -msnbot 1.0 # msnbot/1.0 (+http://search.msn.com/msnbot.htm) -msnbot 0.9 # msnbot/0.9 (+http://search.msn.com/msnbot.htm) -msnbot 0.11 # msnbot/0.11 ( http://search.msn.com/msnbot.htm) -msnbot 0.1 # MSNBOT/0.1 (http://search.msn.com/msnbot.htm) -MSRBot # MSRBOT (http://research.microsoft.com/research/sv/msrbot/) -MSRBot # MSRBOT -MVAClient # MVAClient -mxbot 1.0 # Mozilla/5.0 (compatible; mxbot/1.0; +http://www.chainn.com/mxbot.html) -mxbot 1.0 # Mozilla/5.0 (compatible; mxbot/1.0; http://www.chainn.com/mxbot.html) -NetResearchServer 4.0 # NetResearchServer/4.0(loopimprovements.com/robot.html) -NetResearchServer 3.5 # NetResearchServer/3.5(loopimprovements.com/robot.html) -NetResearchServer 2.8 # NetResearchServer/2.8(loopimprovements.com/robot.html) -NetResearchServer 2.7 # NetResearchServer/2.7(loopimprovements.com/robot.html) -NetResearchServer 2.5 # NetResearchServer/2.5(loopimprovements.com/robot.html) -NetResearchServer # NetResearchServer(http://www.look.com) -NetSeer Crawler 2.0 # Mozilla/5.0 (compatible; NetSeer crawler/2.0; +http://www.netseer.com/crawler.html; [email protected]) -NewsGator 2.5 # NewsGator/2.5 (http://www.newsgator.com; Microsoft Windows NT 5.1.2600.0; .NET CLR 1.1.4322.2032) -NewsGator 2.0 # NewsGator/2.0 Bot (http://www.newsgator.com) -NG-Search 0.9.8 # NG-Search/0.9.8 (http://www.ng-search.com) -NG-Search 0.86 # NG-Search/0.86 (+http://www.ng-search.com) -NG-Search 0.86 # NG-Search/0.86 ( http://www.ng-search.com) -nicebot # nicebot -noxtrumbot 1.0 # noxtrumbot/1.0 ([email protected]) -Nusearch Spider # Nusearch Spider (www.nusearch.com) -Nusearch Spider # nuSearch Spider (compatible; MSIE 4.01; Windows NT) -NutchCVS 0.8-dev # NutchCVS/0.8-dev (Nutch; http://lucene.apache.org/nutch/bot.html; [email protected]) -NutchCVS 0.7.2 # NutchCVS/0.7.2 (Nutch; http://lucene.apache.org/nutch/bot.html; [email protected]) -NutchCVS 0.7.1 # NutchCVS/0.7.1 (Nutch; http://lucene.apache.org/nutch/bot.html; [email protected]) -NutchCVS 0.7.1 # NutchCVS/0.7.1 (Nutch running at UW; http://crawlers.cs.washington.edu/; [email protected]) -NutchCVS 0.7 # NutchCVS/0.7 (Nutch; http://lucene.apache.org/nutch/bot.html; [email protected]) -NutchCVS 0.06-dev # NutchCVS/0.06-dev (Nutch; http://www.nutch.org/docs/en/bot.html; [email protected]) -NutchCVS 0.06-dev # NutchCVS/0.06-dev (Nutch; http://www.nutch.org/docs/en/bot.html; [email protected]) -NutchCVS 0.05 # NutchCVS/0.05 (Nutch; http://www.nutch.org/docs/en/bot.html; [email protected]) -Nymesis 1.0 # Nymesis/1.0 (http://nymesis.com) -obot # Mozilla/4.0 (compatible; MSIE 5.5; Windows NT 4.0; obot) -oegp 1.3.0 # oegp v. 1.3.0 -omgilibot 0.4 # omgilibot/0.4 +http://omgili.com -omgilibot 0.3 # omgilibot/0.3 +http://www.omgili.com/Crawler.html -omgilibot 0.3 # omgilibot/0.3 http://www.omgili.com/Crawler.html -OmniExplorer_Bot 6.70 # OmniExplorer_Bot/6.70 (+http://www.omni-explorer.com) WorldIndexer -OmniExplorer_Bot 6.65a # OmniExplorer_Bot/6.65a (+http://www.omni-explorer.com) WorldIndexer -OmniExplorer_Bot 6.63b # OmniExplorer_Bot/6.63b (+http://www.omni-explorer.com) WorldIndexer -OmniExplorer_Bot 6.62 # OmniExplorer_Bot/6.62 (+http://www.omni-explorer.com) WorldIndexer -OmniExplorer_Bot 6.60 # OmniExplorer_Bot/6.60 (+http://www.omni-explorer.com) WorldIndexer -OmniExplorer_Bot 6.47 # OmniExplorer_Bot/6.47 (+http://www.omni-explorer.com) WorldIndexer -OmniExplorer_Bot 5.91c # OmniExplorer_Bot/5.91c (+http://www.omni-explorer.com) WorldIndexer -OmniExplorer_Bot 5.28 # OmniExplorer_Bot/5.28 (+http://www.omni-explorer.com) WorldIndexer -OmniExplorer_Bot 5.25 # OmniExplorer_Bot/5.25 (+http://www.omni-explorer.com) WorldIndexer -OmniExplorer_Bot 5.20 # OmniExplorer_Bot/5.20 (+http://www.omni-explorer.com) WorldIndexer -OmniExplorer_Bot 5.01 # OmniExplorer_Bot/5.01 (+http://www.omni-explorer.com) WorldIndexer -OmniExplorer_Bot 4.80 # OmniExplorer_Bot/4.80 (+http://www.omni-explorer.com) WorldIndexer -OmniExplorer_Bot 4.32 # OmniExplorer_Bot/4.32 (+http://www.omni-explorer.com) WorldIndexer -OOZBOT 0.20 # OOZBOT/0.20 ( -- ; http://www.setooz.com/oozbot.html ; agentname at setooz dot_com ) -OOZBOT 0.17 # OOZBOT/0.17 (--; http://www.setooz.com/oozbot.html; pvvpr at iiit dot ac dot in) -Orbiter # Orbiter (+http://www.dailyorbit.com/bot.htm) -PageBitesHyperBot 600 # PageBitesHyperBot/600 (http://www.pagebites.com/) -Peew 1.0 # Mozilla/5.0 (compatible; Peew/1.0; http://www.peew.de/crawler/) -polybot 1.0 # polybot 1.0 (http://cis.poly.edu/polybot/) -Pompos 1.3 # Pompos/1.3 http://dir.com/pompos.html -Pompos 1.2 # Pompos/1.2 http://pompos.iliad.fr -Pompos 1.1 # Pompos/1.1 http://pompos.iliad.fr -PostPost 1.0 # PostPost/1.0 (+http://postpo.st/crawlers) -Psbot 0.1 # psbot/0.1 (+http://www.picsearch.com/bot.html) -PycURL 7.23.1 # PycURL/7.23.1 -PycURL 7.19.7 # PycURL/7.19.7 -PycURL 7.19.5 # PycURL/7.19.5 -PycURL 7.19.3 # PycURL/7.19.3 -PycURL 7.19.0 # PycURL/7.19.0 -PycURL 7.18.2 # PycURL/7.18.2 -PycURL 7.18.0 # PycURL/7.18.0 -PycURL 7.16.4 # PycURL/7.16.4 -PycURL 7.15.5 # PycURL/7.15.5 -PycURL 7.13.2 # PycURL/7.13.2 -PycURL # PycURL -Qseero 1.0.0 # Qseero v1.0.0 -Radian6 # radian6_default_(www.radian6.com/crawler) -RAMPyBot 0.1 # RAMPyBot - www.giveRAMP.com/0.1 (RAMPyBot - www.giveRAMP.com; http://www.giveramp.com/bot.html; [email protected]) -RufusBot # RufusBot (Rufus Web Miner; http://64.124.122.252/feedback.html) -SandCrawler # SandCrawler - Compatibility Testing -SBIder 0.8-dev # SBIder/0.8-dev (SBIder; http://www.sitesell.com/sbider.html; http://support.sitesell.com/contact-support.html) -ScoutJet # Mozilla/5.0 (compatible; ScoutJet; http://www.scoutjet.com/) -Scrubby 2.2 # Scrubby/2.2 (http://www.scrubtheweb.com/) -Scrubby 2.2 # Mozilla/5.0 (compatible; Scrubby/2.2; +http://www.scrubtheweb.com/) -Scrubby 2.2 # Mozilla/5.0 (compatible; Scrubby/2.2; http://www.scrubtheweb.com/) -Scrubby 2.1 # Scrubby/2.1 (http://www.scrubtheweb.com/) -Scrubby 2.1 # Mozilla/5.0 (compatible; Scrubby/2.1; +http://www.scrubtheweb.com/abs/meta-check.html) -SearchSight 2.0 # SearchSight/2.0 (http://SearchSight.com/) -Seekbot 1.0 # Seekbot/1.0 (http://www.seekbot.net/bot.html) RobotsTxtFetcher/1.2 -Seekbot 1.0 # Seekbot/1.0 (http://www.seekbot.net/bot.html) HTTPFetcher/2.1 -Seekbot 1.0 # Seekbot/1.0 (http://www.seekbot.net/bot.html) HTTPFetcher/0.3 -Seekbot 1.0 # Seekbot/1.0 (http://www.seekbot.net/bot.html) -semanticdiscovery 0.1 # semanticdiscovery/0.1 -Sensis Web Crawler # Sensis Web Crawler (search_comments\at\sensis\dot\com\dot\au) -SEOChat::Bot 1.1 # Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.0) SEOChat::Bot v1.1 -SeznamBot 2.0 # SeznamBot/2.0 (+http://fulltext.seznam.cz/) -SeznamBot 2.0 # SeznamBot/2.0 (+http://fulltext.sblog.cz/robot/) -Shim-Crawler # Shim-Crawler(Mozilla-compatible; http://www.logos.ic.i.u-tokyo.ac.jp/crawler/; [email protected]) -Shim-Crawler # Shim-Crawler -ShopWiki 1.0 # ShopWiki/1.0 ( +http://www.shopwiki.com/wiki/Help:Bot) -Shoula robot # Mozilla/4.0 (compatible: Shoula robot) -Sitebot 0.1 # Mozilla/5.0 (compatible; SiteBot/0.1; +http://www.sitebot.org/robot/) -Sitebot 0.1 # Mozilla/5.0 (compatible; SiteBot/0.1; http://www.sitebot.org/robot/) -Snappy 1.1 # Snappy/1.1 ( http://www.urltrends.com/ ) -sogou spider # sogou spider -Sosospider # Sosospider+(+http://help.soso.com/webspider.htm) -Speedy Spider 5.0 # Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US) Speedy Spider (http://www.entireweb.com/about/search_tech/speedy_spider/) -Speedy Spider 5.0 # Mozilla/5.0 (compatible; Speedy Spider; http://www.entireweb.com/about/search_tech/speedy_spider/) -Speedy Spider 1.3 # Speedy Spider (Entireweb; Beta/1.3; http://www.entireweb.com/about/search_tech/speedyspider/) -Speedy Spider 1.2 # Speedy Spider (Entireweb; Beta/1.2; http://www.entireweb.com/about/search_tech/speedyspider/) -Speedy Spider 1.1 # Speedy Spider (Entireweb; Beta/1.1; http://www.entireweb.com/about/search_tech/speedyspider/) -Speedy Spider 1.0 # Speedy Spider (Entireweb; Beta/1.0; http://www.entireweb.com/about/search_tech/speedyspider/) -Speedy Spider 1.0 # Speedy Spider (Beta/1.0; www.entireweb.com) -Speedy Spider # Speedy Spider (http://www.entireweb.com/about/search_tech/speedy_spider/) -Speedy Spider # Speedy Spider (http://www.entireweb.com/about/search_tech/speedyspider/) -Speedy Spider # Speedy Spider (http://www.entireweb.com) -Sqworm 2.9.85-BETA # Sqworm/2.9.85-BETA (beta_release; 20011115-775; i686-pc-linux-gnu) -StackRambler 2.0 # StackRambler/2.0 (MSIE incompatible) -StackRambler 2.0 # StackRambler/2.0 -suggybot 0.01a # Mozilla/5.0 (compatible; suggybot v0.01a, http://blog.suggy.com/was-ist-suggy/suggy-webcrawler/) -SurveyBot 2.3 # SurveyBot/2.3+(Whois+Source) -SurveyBot 2.3 # SurveyBot/2.3 (Whois Source) -SynooBot 0.7.1 # SynooBot/0.7.1 (SynooBot; http://www.synoo.de/bot.html; [email protected]) -Teoma # Mozilla/2.0 (compatible; Ask Jeeves/Teoma; +http://sp.ask.com/docs/about/tech_crawling.html) -Teoma # Mozilla/2.0 (compatible; Ask Jeeves/Teoma; +http://about.ask.com/en/docs/about/webmasters.shtml) -Teoma # Mozilla/2.0 (compatible; Ask Jeeves/Teoma) -TerrawizBot 1.0 # TerrawizBot/1.0 (+http://www.terrawiz.com/bot.html) -TheSuBot 0.2 # TheSuBot/0.2 (www.thesubot.de) -TheSuBot 0.1 # TheSuBot/0.1 (www.thesubot.de) -Thumbnail.CZ robot 1.1 # Thumbnail.CZ robot 1.1 (http://thumbnail.cz/why-no-robots-txt.html) -TinEye 1.1 # TinEye/1.1 (http://tineye.com/crawler.html) -TinEye # TinEye -truwoGPS 1.0 # truwoGPS/1.0 (GNU/Linux; U; i686; en-US; +http://www.lan4lano.net/browser.html ) -TurnitinBot 2.1 # TurnitinBot/2.1 (http://www.turnitin.com/robot/crawlerinfo.html) -TurnitinBot 2.0 # TurnitinBot/2.0 http://www.turnitin.com/robot/crawlerinfo.html -TurnitinBot 1.5 # TurnitinBot/1.5 http://www.turnitin.com/robot/crawlerinfo.html -TurnitinBot 1.5 # TurnitinBot/1.5 (http://www.turnitin.com/robot/crawlerinfo.html) -TurnitinBot 1.5 # TurnitinBot/1.5 http://www.turnitin.com/robot/crawlerinfo.html -TurnitinBot 1.5 # TurnitinBot/1.5 (http://www.turnitin.com/robot/crawlerinfo.html) -TweetedTimes Bot 1.0 # Mozilla/5.0 (compatible; TweetedTimes Bot/1.0; http://tweetedtimes.com) -TwengaBot # TwengaBot -updated 0.1-beta # updated/0.1-beta (updated; http://www.updated.com; [email protected]) -Urlfilebot 2.2 # Mozilla/5.0 (compatible; Urlfilebot/2.2; +http://urlfile.com/bot.html) -Vagabondo 4.0Beta # Mozilla/4.0 (compatible; Vagabondo/4.0Beta; webcrawler at wise-guys dot nl; http://webagent.wise-guys.nl/; http://www.wise-guys.nl/) -Vagabondo 2.2 # Mozilla/4.0 (compatible; Vagabondo/2.2; webcrawler at wise-guys dot nl; http://webagent.wise-guys.nl/) -Vagabondo 2.1 # Mozilla/5.0 (compatible; Vagabondo/2.1; webcrawler at wise-guys dot nl; http://webagent.wise-guys.nl/) -Vagabondo 2.0 MT # Mozilla/3.0 (Vagabondo/2.0 MT; [email protected]; http://aanmelden.ilse.nl/?aanmeld_mode=webhints) -VoilaBot 1.2 # Mozilla/4.0 (compatible; MSIE 5.0; Windows 95) VoilaBot BETA 1.2 (http://www.voila.com/) -Vortex 2.2 # Vortex/2.2 (+http://marty.anstey.ca/robots/vortex/) -Vortex 2.2 # Vortex/2.2 ( http://marty.anstey.ca/robots/vortex/) -Vortex 1.2 # VORTEX/1.2 ( http://marty.anstey.ca/robots/vortex/) -voyager 2.0 # voyager/2.0 (http://www.kosmix.com/crawler.html) -voyager 1.0 # voyager/1.0 -VYU2 # VYU2 (GNU; OpenRISC) -webcollage 1.93 # webcollage/1.93 -webcollage 1.129 # webcollage/1.129 -webcollage 1.125 # webcollage/1.125 -webcollage 1.117 # webcollage/1.117 -webcollage 1.114 # webcollage/1.114 -Websquash.com # Websquash.com (Add url robot) -wf84 # http://www.almaden.ibm.com/cs/crawler [wf84] -WoFindeIch Robot 1.0 # WoFindeIch Robot 1.0(+http://www.search.wofindeich.com/robot.php) -WoFindeIch Robot 1.0 # WoFindeIch Robot 1.0( http://www.search.wofindeich.com/robot.php) -WomlpeFactory 0.1 # WomlpeFactory/0.1 (+http://www.Womple.com/bot.html) -Xaldon_WebSpider 2.0.b1 # Xaldon_WebSpider/2.0.b1 -Xaldon_WebSpider 2.0.b1 # Xaldon_WebSpider/2.0.b1 -yacy # yacybot (x86 Windows XP 5.1; java 1.6.0_12; Europe/de) http://yacy.net/bot.html -yacy # yacybot (x86 Windows XP 5.1; java 1.6.0_11; Europe/de) http://yacy.net/bot.html -yacy # yacybot (x86 Windows XP 5.1; java 1.6.0; Europe/de) http://yacy.net/yacy/bot.html -yacy # yacybot (x86 Windows 2000 5.0; java 1.6.0_16; Europe/de) http://yacy.net/bot.html -yacy # yacybot (ppc Mac OS X 10.5.2; java 1.5.0_13; Europe/de) http://yacy.net/bot.html -yacy # yacybot (ppc Mac OS X 10.4.10; java 1.5.0_07; Europe/de) http://yacy.net/bot.html -yacy # yacybot (i386 Mac OS X 10.5.7; java 1.5.0_16; Europe/de) http://yacy.net/bot.html -yacy # yacybot (i386 Linux 2.6.9-023stab046.2-smp; java 1.6.0_05; Europe/en) http://yacy.net/bot.html -yacy # yacybot (i386 Linux 2.6.8-022stab070.5-enterprise; java 1.4.2-03; Europe/en) yacy.net -yacy # yacybot (i386 Linux 2.6.31-16-generic; java 1.6.0_15; Europe/en) http://yacy.net/bot.html -yacy # yacybot (i386 Linux 2.6.26-2-686; java 1.6.0_0; Europe/en) http://yacy.net/bot.html -yacy # yacybot (i386 Linux 2.6.24-28-generic; java 1.6.0_20; Europe/en) http://yacy.net/bot.html -yacy # yacybot (i386 Linux 2.6.24-24-generic; java 1.6.0_07; Europe/en) http://yacy.net/bot.html -yacy # yacybot (i386 Linux 2.6.24-23-generic; java 1.6.0_16; Europe/en) http://yacy.net/bot.html -yacy # yacybot (i386 Linux 2.6.23; java 1.6.0_17; Europe/en) http://yacy.net/bot.html -yacy # yacybot (i386 Linux 2.6.23; java 1.6.0_04; Europe/en) http://yacy.net/bot.html -yacy # yacybot (i386 Linux 2.6.22-14-generic; java 1.6.0_03; Europe/de) http://yacy.net/bot.html -yacy # yacybot (amd64 Windows 7 6.1; java 1.6.0_17; Europe/de) http://yacy.net/bot.html -yacy # yacybot (amd64 Linux 2.6.28-18-generic; java 1.6.0_0; Europe/en) http://yacy.net/bot.html -yacy # yacybot (amd64 Linux 2.6.16-2-amd64-k8-smp; java 1.5.0_10; Europe/en) http://yacy.net/yacy/bot.html -Yahoo! Slurp # Mozilla/5.0 (compatible; Yahoo! Slurp; http://help.yahoo.com/help/us/ysearch/slurp) -Yahoo! Slurp China # Mozilla/5.0 (compatible; Yahoo! Slurp China; http://misc.yahoo.com.cn/help.html) -YahooSeeker 1.2 # YahooSeeker/1.2 (compatible; Mozilla 4.0; MSIE 5.5; yahooseeker at yahoo-inc dot com ; http://help.yahoo.com/help/us/shop/merchant/) -YahooSeeker-Testing 3.9 # YahooSeeker-Testing/v3.9 (compatible; Mozilla 4.0; MSIE 5.5; http://search.yahoo.com/) -YandexBot 3.0 # Mozilla/5.0 (compatible; YandexBot/3.0; +http://yandex.com/bots) -YandexImages 3.0 # Mozilla/5.0 (compatible; YandexImages/3.0; +http://yandex.com/bots) -Yasaklibot 1.2 # Yasaklibot/v1.2 (http://www.Yasakli.com/bot.php) -Yeti 1.0 # Yeti/1.0 (NHN Corp.; http://help.naver.com/robots/) -Yeti 1.0 # Yeti/1.0 (+http://help.naver.com/robots/) -YodaoBot 1.0 # Mozilla/5.0 (compatible; YodaoBot/1.0; http://www.yodao.com/help/webmaster/spider/; ) -yoogliFetchAgent 0.1 # yoogliFetchAgent/0.1 -YoudaoBot 1.0 # Mozilla/5.0 (compatible; YoudaoBot/1.0; http://www.youdao.com/help/webmaster/spider/; ) -Zao 0.1 # Zao/0.1 (http://www.kototoi.org/zao/) -Zao 0.1 # Zao/0.1 (http://www.kototoi.org/zao/) -Zealbot 1.0 # Mozilla/4.0 (compatible; Zealbot 1.0) -zspider 0.9-dev # zspider/0.9-dev http://feedback.redkolibri.com/ -ZyBorg 1.0 # Mozilla/4.0 compatible ZyBorg/1.0 DLC ([email protected]; http://www.WISEnutbot.com) -ZyBorg 1.0 # Mozilla/4.0 compatible ZyBorg/1.0 Dead Link Checker ([email protected]; http://www.WISEnutbot.com) -ZyBorg 1.0 # Mozilla/4.0 compatible ZyBorg/1.0 Dead Link Checker ([email protected]; http://www.WISEnutbot.com) -ZyBorg 1.0 # Mozilla/4.0 compatible ZyBorg/1.0 ([email protected]; http://www.WISEnutbot.com) -ZyBorg 1.0 # Mozilla/4.0 compatible ZyBorg/1.0 ([email protected]; http://www.WISEnutbot.com) -ZyBorg 1.0 # Mozilla/4.0 compatible ZyBorg/1.0 ([email protected]; http://www.WISEnutbot.com) -EmailSiphon # EmailSiphon -Bloglines 3.1 # Bloglines/3.1 (http://www.bloglines.com) -Bloglines 3.0-rho # Bloglines/3.0-rho (http://www.bloglines.com; 3 subscribers) -everyfeed-spider 2.0 # everyfeed-spider/2.0 (http://www.everyfeed.com) -FeedFetcher-Google # Feedfetcher-Google; (+http://www.google.com/feedfetcher.html; feed-id=8639390370582375869) -FeedFetcher-Google # Feedfetcher-Google; (+http://www.google.com/feedfetcher.html; feed-id=375807642710480585) -FeedFetcher-Google # Feedfetcher-Google; (+http://www.google.com/feedfetcher.html; feed-id=1992026586303346733) -FeedFetcher-Google # Feedfetcher-Google; (+http://www.google.com/feedfetcher.html; feed-id=13645798864011762265) -FeedFetcher-Google # FeedFetcher-Google; (+http://www.google.com/feedfetcher.html) -GreatNews 1.0 # GreatNews/1.0 -Gregarius 0.5.2 # Gregarius/0.5.2 (+http://devlog.gregarius.net/docs/ua) -MagpieRSS 0.7 # MagpieRSS/0.7 ( http://magpierss.sf.net) -NFReader 1.4.1.0 # NFReader/1.4.1.0 (http://www.gaijin.at/) -UniversalFeedParser 3.3 # UniversalFeedParser/3.3 +http://feedparser.org/ -BinGet 1.00.A # BinGet/1.00.A (http://www.bin-co.com/php/scripts/load/) -cURL 7.9.8 # curl/7.9.8 (i686-pc-linux-gnu) libcurl 7.9.8 (OpenSSL 0.9.6b) (ipv6 enabled) -cURL 7.8 # curl/7.8 (i386-redhat-linux-gnu) libcurl 7.8 (OpenSSL 0.9.6b) (ipv6 enabled) -cURL 7.7.x # curl/7.7.x (i386--freebsd4.3) libcurl 7.7.x (SSL 0.9.6) (ipv6 enabled) -cURL 7.7.2 # curl/7.7.2 (powerpc-apple-darwin6.0) libcurl 7.7.2 (OpenSSL 0.9.6b) -cURL 7.21.4 # curl/7.21.4 (universal-apple-darwin11.0) libcurl/7.21.4 OpenSSL/0.9.8r zlib/1.2.5 -cURL 7.21.3 # curl/7.21.3 (x86_64-unknown-linux-gnu) libcurl/7.21.3 OpenSSL/1.0.0c zlib/1.2.5 -cURL 7.21.3 # curl/7.21.3 (x86_64-redhat-linux-gnu) libcurl/7.21.3 NSS/3.13.1.0 zlib/1.2.5 libidn/1.19 libssh2/1.2.7 -cURL 7.21.2 # curl/7.21.2 (i386-pc-win32) libcurl/7.21.2 OpenSSL/0.9.8o zlib/1.2.5 -cURL 7.21.1 # curl/7.21.1 (i686-pc-linux-gnu) libcurl/7.21.1 OpenSSL/1.0.0a zlib/1.2.5 -cURL 7.21.0 # curl/7.21.0 (x86_64-pc-linux-gnu) libcurl/7.21.0 OpenSSL/0.9.8o zlib/1.2.3.4 libidn/1.18 libssh2/1.2.5 -cURL 7.21.0 # curl/7.21.0 (x86_64-pc-linux-gnu) libcurl/7.21.0 OpenSSL/0.9.8o zlib/1.2.3.4 libidn/1.18 -cURL 7.21.0 # curl/7.21.0 (x86_64-pc-linux-gnu) libcurl/7.21.0 OpenSSL/0.9.8o zlib/1.2.3.4 libidn/1.15 libssh2/1.2.5 -cURL 7.21.0 # curl/7.21.0 (x86_64-apple-darwin10.2.0) libcurl/7.21.0 OpenSSL/1.0.0a zlib/1.2.5 libidn/1.19 -cURL 7.21.0 # curl/7.21.0 (i686-pc-linux-gnu) libcurl/7.21.0 OpenSSL/0.9.8o zlib/1.2.3.4 libidn/1.18 -cURL 7.21.0 # curl/7.21.0 (i486-pc-linux-gnu) libcurl/7.21.0 OpenSSL/0.9.8o zlib/1.2.3.4 libidn/1.18 libssh2/1.2.6 -cURL 7.20.0 # curl/7.20.0 (i686-pc-linux-gnu) libcurl/7.20.0 OpenSSL/0.9.8n zlib/1.2.4 -cURL 7.20.0 # curl/7.20.0 (i386-apple-darwin9.8.0) libcurl/7.20.0 OpenSSL/0.9.8m zlib/1.2.3 libidn/1.16 -cURL 7.19.7 # curl/7.19.7 (x86_64-redhat-linux-gnu) libcurl/7.35.0 OpenSSL/1.0.1e zlib/1.2.3 libidn/1.18 libssh2/1.4.3 -cURL 7.19.7 # curl/7.19.7 (universal-apple-darwin10.0) libcurl/7.19.7 OpenSSL/0.9.8l zlib/1.2.3 -cURL 7.19.7 # curl/7.19.7 (i486-pc-linux-gnu) libcurl/7.19.7 OpenSSL/0.9.8o zlib/1.2.3.3 libidn/1.15 -cURL 7.19.7 # curl/7.19.7 (i486-pc-linux-gnu) libcurl/7.19.7 OpenSSL/0.9.8k zlib/1.2.3.3 libidn/1.15 -cURL 7.19.7 # curl/7.19.7 (i386-redhat-linux-gnu) libcurl/7.19.7 NSS/3.12.5.0 zlib/1.2.3 libidn/1.9 libssh2/1.2.2 -cURL 7.19.7 # curl/7.19.7 (i386-apple-darwin9.8.0) libcurl/7.19.7 zlib/1.2.3 -cURL 7.19.6 # curl/7.19.6 (i686-pc-cygwin) libcurl/7.19.6 OpenSSL/0.9.8n zlib/1.2.3 libidn/1.18 libssh2/1.2 -cURL 7.19.6 # curl/7.19.6 (i386-redhat-linux-gnu) libcurl/7.19.6 NSS/3.12.4.5 zlib/1.2.3 libidn/1.9 libssh2/1.2 -cURL 7.19.6 # curl/7.19.6 (i386-pc-win32) libcurl/7.19.6 OpenSSL/0.9.8k zlib/1.2.3 -cURL 7.19.5 # curl/7.19.5 (i586-pc-mingw32msvc) libcurl/7.19.5 zlib/1.2.3 -cURL 7.19.5 # curl/7.19.5 (i486-pc-linux-gnu) libcurl/7.19.5 OpenSSL/0.9.8g zlib/1.2.3.3 libidn/1.15 -cURL 7.19.4 # curl/7.19.4 (universal-apple-darwin10.0) libcurl/7.19.4 OpenSSL/0.9.8k zlib/1.2.3 -cURL 7.19.4 # curl/7.19.4 (i686-pc-cygwin) libcurl/7.19.4 OpenSSL/0.9.8k zlib/1.2.3 libidn/1.9 libssh2/1.0 -cURL 7.19.2 # curl/7.19.2 (i386-pc-win32) libcurl/7.19.2 OpenSSL/0.9.8i zlib/1.2.3 libidn/1.11 libssh2/0.18 -cURL 7.19.2 # curl/7.19.2 (i386-pc-win32) libcurl/7.19.2 OpenSSL/0.9.8c zlib/1.2.3 -cURL 7.19.0 # curl/7.19.0 (x86_64-suse-linux-gnu) libcurl/7.19.0 OpenSSL/0.9.8h zlib/1.2.3 libidn/1.10 -cURL 7.18.2 # curl/7.18.2 (x86_64-pc-linux-gnu) libcurl/7.18.2 OpenSSL/0.9.8g zlib/1.2.3.3 libidn/1.8 libssh2/0.18 -cURL 7.18.1 # curl/7.18.1 (i686-suse-linux-gnu) libcurl/7.18.1 OpenSSL/0.9.8g zlib/1.2.3 libidn/1.8 -cURL 7.18.0 # curl/7.18.0 (x86_64-pc-linux-gnu) libcurl/7.18.0 OpenSSL/0.9.8g zlib/1.2.3.3 libidn/1.1 -cURL 7.17.1 # curl/7.17.1 (x86_64-pc-linux-gnu) libcurl/7.17.1 OpenSSL/0.9.8g zlib/1.2.3 -cURL 7.16.4 # curl/7.16.4 (i486-pc-linux-gnu) libcurl/7.16.4 OpenSSL/0.9.8e zlib/1.2.3.3 libidn/1.0 -cURL 7.16.3 # curl/7.16.3 (powerpc-apple-darwin8.0) libcurl/7.16.3 OpenSSL/0.9.7l zlib/1.2.3 -cURL 7.16.2 # curl/7.16.2 (x86_64-redhat-linux-gnu) libcurl/7.16.2 OpenSSL/0.9.8b zlib/1.2.3 libidn/0.6.8 -cURL 7.16.1 # curl/7.16.1 (i386-pc-win32) libcurl/7.16.1 OpenSSL/0.9.8h zlib/1.2.3 -cURL 7.15.5 # curl/7.15.5 (x86_64-redhat-linux-gnu) libcurl/7.15.5 OpenSSL/0.9.8b zlib/1.2.3 libidn/0.6.5 -cURL 7.15.4 # curl/7.15.4 (i686-pc-linux-gnu) libcurl/7.15.4 OpenSSL/0.9.7e zlib/1.2.3 -cURL 7.15.3 # curl/7.15.3 (sparc64--netbsd) libcurl/7.15.3 OpenSSL/0.9.7d zlib/1.1.4 libidn/0.6.3 -cURL 7.15.1 # curl/7.15.1 (x86_64-suse-linux) libcurl/7.15.1 OpenSSL/0.9.8a zlib/1.2.3 libidn/0.6.0 -cURL 7.15.1 # curl/7.15.1 (i486-pc-linux-gnu) libcurl/7.15.1 OpenSSL/0.9.8a zlib/1.2.3 libidn/0.5.18 -cURL 7.15.0 # curl/7.15.0 (i386-portbld-freebsd5.4) libcurl/7.15.0 OpenSSL/0.9.7e zlib/1.2.1 -cURL 7.14.0 # curl/7.14.0 (i386-portbld-freebsd5.4) libcurl/7.14.0 OpenSSL/0.9.7e zlib/1.2.1 -cURL 7.13.2 # curl/7.13.2 (i386-pc-linux-gnu) libcurl/7.13.2 OpenSSL/0.9.7e zlib/1.2.2 libidn/0.5.13 -cURL 7.13.1 # cURL: curl/7.13.1 (powerpc-apple-darwin8.0) libcurl/7.13.1 OpenSSL/0.9.7b zlib/1.2.2 -cURL 7.13.1 # curl/7.13.1 (powerpc-apple-darwin8.0) libcurl/7.13.1 OpenSSL/0.9.7l zlib/1.2.3 -cURL 7.12.1 # curl/7.12.1 (i686-redhat-linux-gnu) libcurl/7.12.1 OpenSSL/0.9.7a zlib/1.2.1.2 libidn/0.5.6 -cURL 7.11.1 # curl/7.11.1 (i686-redhat-linux-gnu) libcurl/7.11.1 OpenSSL/0.9.7a ipv6 zlib/1.2.1.2 -cURL 7.11.1 # curl/7.11.1 (i386-redhat-linux-gnu) libcurl/7.11.1 OpenSSL/0.9.7a ipv6 zlib/1.2.1.2 -cURL 7.10.6 # curl/7.10.6 (i386-redhat-linux-gnu) libcurl/7.10.6 OpenSSL/0.9.7a ipv6 zlib/1.1.4 -Java 1.6.0_26 # Java/1.6.0_26 -Java 1.6.0_13 # Java/1.6.0_13 -Java 1.6.0_12 # Java/1.6.0_12 -Java 1.6.0_11 # Java/1.6.0_11 -Java 1.6.0_04 # Java/1.6.0_04 -Java 1.6.0_03 # Java/1.6.0_03 -Java 1.6.0_02 # Java/1.6.0_02 -Java 1.6.0-beta # Java/1.6.0-beta -Java 1.5.0_11 # Java/1.5.0_11 -Java 1.5.0_08 # Java/1.5.0_08 -Java 1.5.0_06 # Java/1.5.0_06 -Java 1.5.0_05 # Java/1.5.0_05 -Java 1.5.0_04 # Java/1.5.0_04 -Java 1.5.0_03 # Java/1.5.0_03 -Java 1.5.0_02 # Java/1.5.0_02 -Java 1.5.0_01 # Java/1.5.0_01 -Java 1.5.0 # Java/1.5.0 -Java 1.4.2_11 # Java/1.4.2_11 -Java 1.4.2_10 # Java/1.4.2_10 -Java 1.4.2_09 # Java/1.4.2_09 -Java 1.4.2_08 # Java/1.4.2_08 -Java 1.4.2_07 # Java/1.4.2_07 -Java 1.4.2_05 # Java/1.4.2_05 -Java 1.4.2_04 # Java/1.4.2_04 -Java 1.4.2_03 # Java1.4.2_03 -Java 1.4.2_03 # Java/1.4.2_03 -Java 1.4.2_01 # Java/1.4.2_01 -Java 1.4.2 # Java/1.4.2 -Java 1.4.1_04 # Java/1.4.1_04 -Java 1.4.1_03 # Java/1.4.1_03 -Java 1.4.1_02 # Java/1.4.1_02 -Java 1.4.1_01a # Java/1.4.1_01a -Java 1.4.1_01 # Java/1.4.1_01 -Java 1.4.1-p3 # Java/1.4.1-p3 -Java 1.4.1 # Java/1.4.1 -Java 1.4.0_03 # Java1.4.0_03 -Java 1.4.0_02 # Java1.4.0_02 -Java 1.4.0_01 # Java1.4.0_01 -Java 1.4.0 # Java1.4.0 -Java 1.3.1_06 # Java1.3.1_06 -Java 1.3.1_04 # Java1.3.1_04 -Java 1.3.1 # Java1.3.1 -Java 1.3.0 # Java1.3.0 -Java 1.2.2-JDeveloper # Java1.2.2-JDeveloper -Java 1.2.2 # Java1.2.2 -Java 1.2.1 # Java1.2.1 -libwww-perl 5.821 # libwww-perl/5.821 -libwww-perl 5.820 # libwww-perl/5.820 -libwww-perl 5.816 # libwww-perl/5.816 -libwww-perl 5.814 # libwww-perl/5.814 -libwww-perl 5.808 # libwww-perl/5.808 -libwww-perl 5.805 # libwww-perl/5.805 -libwww-perl 5.803 # libwww-perl/5.803 -libwww-perl 5.800 # libwww-perl/5.800 (+http://passoire.afraid.org/mylittlewebsurvey/index.html) -libwww-perl 5.800 # libwww-perl/5.800 -libwww-perl 5.76 # libwww-perl/5.76 -libwww-perl 5.75 # libwww-perl/5.75 -libwww-perl 5.69 # libwww-perl/5.69 -libwww-perl 5.65 # libwww-perl/5.65 -libwww-perl 5.65 # libwww-perl/5.65 -libwww-perl 5.64 # libwww-perl/5.64 -libwww-perl 5.63 # libwww-perl/5.63 -libwww-perl 5.53 # libwww-perl/5.53 -libwww-perl 5.50 # libwww-perl/5.50 -libwww-perl 5.48 # libwww-perl/5.48 -libwww-perl 5.36 # libwww-perl/5.36 -Microsoft URL Control 6.01.9782 # Microsoft URL Control - 6.01.9782 -Peach 1.01 # Peach/1.01 (Ubuntu 8.04 LTS; U; en) -PHP 5.2.9 # PHP/5.2.9 -PHP 5.2.8 # PHP/5.2.8 -PHP 5.2.14 # PHP/5.2.14 -PHP 5.2.11 # PHP/5.2.11 -PHP 5.2.10 # PHP/5.2.10 -pxyscand 2.1 # pxyscand/2.1 -Python-urllib 3.1 # Python-urllib/3.1 -Python-urllib 3.0 # Python-urllib/3.0 -Python-urllib 2.7 # Python-urllib/2.7 -Python-urllib 2.6 # Python-urllib/2.6 -Python-urllib 2.5 # Python-urllib/2.5 -Python-urllib 2.4 # Python-urllib/2.4 -Python-urllib 2.1 # Python-urllib/2.1 -Python-urllib 2.0a1 # Python-urllib/2.0a1 -Python-urllib 1.17 # Python-urllib/1.17 -Python-urllib 1.16 # Python-urllib/1.16 -Python-urllib 1.15 # Python-urllib/1.15 -AbiLogicBot 1.0 # Mozilla/5.0 (compatible; AbiLogicBot/1.0; +http://www.abilogic.com/bot.html) -AbiLogicBot 1.0 # Mozilla/5.0 (compatible; AbiLogicBot/1.0; +http://www.abilogic.com) -Link Valet 1.1 # Link Valet Online 1.1 -Link Validity Check # Link Validity Check From: http://www.w3dir.com/cgi-bin (Using: Hot Links SQL by Mrcgiguy.com) -LinkExaminer 1.01 # LinkExaminer/1.01 (Windows) -LinksManager.com_bot 5.0 # Mozilla/5.0 (compatible; LinksManager.com_bot http://linksmanager.com/linkchecker.html) -LinksManager.com_bot # Mozilla/5.0 (compatible; LinksManager.com_bot +http://linksmanager.com/linkchecker.html) -Mojoo Robot # Mojoo Robot (http://www.mojoo.com/) -Notifixious # Notifixious/LinkChecker (http://notifixio.us) -online link validator # online link validator (http://www.dead-links.com/) -Ploetz + Zeller 1.0 # Ploetz + Zeller (http://www.ploetz-zeller.de) Link Validator v1.0 ([email protected]) for ARIS Business Architect -Reciprocal Link System PRO # InfoWizards Reciprocal Link System PRO - (http://www.infowizards.com) -REL Link Checker Lite 1.0 # REL Link Checker Lite 1.0 -SiteBar 3.3.8 # SiteBar/3.3.8 (Bookmark Server; http://sitebar.org/) -SiteBar 3.3.5 # SiteBar/3.3.5 (Bookmark Server; http://sitebar.org/) -Vivante Link Checker # Vivante Link Checker (http://www.vivante.com) -W3C-checklink 4.5 # W3C-checklink/4.5 [4.160] libwww-perl/5.823 -W3C-checklink 4.5 # W3C-checklink/4.5 [4.154] libwww-perl/5.823 -W3C-checklink 4.3 # W3C-checklink/4.3 [4.42] libwww-perl/5.820 -W3C-checklink 4.3 # W3C-checklink/4.3 [4.42] libwww-perl/5.808 -W3C-checklink 4.3 # W3C-checklink/4.3 [4.42] libwww-perl/5.805 -W3C-checklink 4.2.1 # W3C-checklink/4.2.1 [4.21] libwww-perl/5.803 -W3C-checklink 4.2 # W3C-checklink/4.2 [4.20] libwww-perl/5.803 -W3C-checklink 3.6.2.3 # W3C-checklink/3.6.2.3 libwww-perl/5.64 -W3C-checklink 2.90 # W3C-checklink/2.90 libwww-perl/5.64 -Xenu Link Sleuth 1.2i # Xenu Link Sleuth 1.2i -Xenu Link Sleuth 1.2h # Xenu Link Sleuth 1.2h -Xenu Link Sleuth 1.2g # Xenu Link Sleuth 1.2g -Xenu Link Sleuth 1.2f # Xenu Link Sleuth 1.2f -Xenu Link Sleuth 1.2e # Xenu Link Sleuth 1.2e -Xenu Link Sleuth 1.2d # Xenu Link Sleuth 1.2d -Xenu Link Sleuth 1.2c # Xenu Link Sleuth 1.2c -Xenu Link Sleuth 1.2b # Xenu Link Sleuth 1.2b -Xenu Link Sleuth # Xenu Link Sleuth/1.3.7 -Offline Explorer 2.5 # Offline Explorer/2.5 -Offline Explorer 1.9 # Offline Explorer/1.9 -Offline Explorer 1.4 # Offline Explorer/1.4 -SuperBot 4.4.0.60 # SuperBot/4.4.0.60 (Windows XP) -Web Downloader 6.9 # Web Downloader/6.9 -Web Downloader 6.8 # Web Downloader/6.8 -Web Downloader 5.7 # Web Downloader/5.7 -Web Downloader 4.5 # Web Downloader/4.5 -WebCopier 4.6 # WebCopier v4.6 -WebCopier 4.5.2 # WebCopier v4.5.2 -WebCopier 3.3 # WebCopier v3.3 -WebCopier 3.2a # WebCopier v3.2a -WebCopier 3.2a # WebCopier v3.2a -WebCopier 3.2 # WebCopier v3.2 -WebCopier 3.0 # WebCopier v3.0 -WebCopier 2.8 # WebCopier v2.8 -WebZIP 3.5 # WebZIP/3.5 (http://www.spidersoft.com) -Wget 1.9cvs-stable # Wget/1.9+cvs-stable (Red Hat modified) -Wget 1.9.1 # Wget/1.9.1 -Wget 1.9 # Wget/1.9 -Wget 1.8.2 # Wget/1.8.2 -Wget 1.8.1 # Wget/1.8.1 -Wget 1.7 # Wget/1.7 -Wget 1.6 # Wget/1.6 -Wget 1.5.3 # Wget/1.5.3 -Wget 1.15 # Wget/1.15 (darwin13.0.2) -Wget 1.11.4 # Wget/1.11.4 (Red Hat modified) -Wget 1.11.1 # Wget/1.11.1 (Red Hat modified) -Wget 1.10.2 # Wget/1.10.2 (Red Hat modified) -Wget 1.10.2 # Wget/1.10.2 -Wget 1.10.1-beta1 # Wget/1.10.1-beta1 -Wget 1.10.1 # Wget/1.10.1 (Red Hat modified) -Wget 1.10.1 # Wget/1.10.1 -Wget # wget -!Susie # !Susie (http://www.sync2it.com/susie) -Amaya 11.3.1 # amaya/11.3.1 libwww/5.4.1 -Amaya 11.2 # amaya/11.2 libwww/5.4.0 -Amaya 11.1 # amaya/11.1 libwww/5.4.0 -Amaya 10.1 # amaya/10.1 libwww/5.4.0 -Amaya 10 # amaya/10 libwww/5.4.0 -Amaya 9.55 # amaya/9.55 libwww/5.4.0 -Amaya 9.54 # amaya/9.54 libwww/5.4.0 -Amaya 9.52 # amaya/9.52 libwww/5.4.0 -Amaya 9.51 # amaya/9.51 libwww/5.4.0 -Amaya 8.8.5 # amaya/8.8.5 libwww/5.4.0 -Amaya 5.4.0 # amaya/11.2 amaya/5.4.0 -Amaya 5.4.0 # amaya/11.1 amaya/5.4.0 -DomainsDB.net MetaCrawler 0.9.7c # DomainsDB.net MetaCrawler v.0.9.7c (http://domainsdb.net/) -GSiteCrawler 1.20 # GSiteCrawler/v1.20 rev. 273 (http://gsitecrawler.com/) -GSiteCrawler 1.12 # GSiteCrawler/v1.12 rev. 260 (http://gsitecrawler.com/) -GSiteCrawler 1.06 # GSiteCrawler/v1.06 rev. 251 (http://gsitecrawler.com/) -lftp 4.3.8 # lftp/4.3.8 -lftp 4.3.5 # lftp/4.3.5 -MetaURI # MetaURI API/2.0 metauri.com -Nitro PDF # Nitro PDF Download -Snoopy 1.2 # Snoopy v1.2 -URD-MAGPIE 0.73 # URD-MAGPIE/0.73 (Cached) -WebCapture 3.0 # Mozilla/4.0 (compatible; WebCapture 3.0; Windows) -WebCapture 3.0 # Mozilla/4.0 (compatible; WebCapture 3.0; Macintosh) -WebCapture 2.0 # Mozilla/3.0 (compatible; WebCapture 2.0; Windows) -WebCapture 1.0 # Mozilla/3.0 (compatible; WebCapture 1.0; Windows) -Windows-Media-Player 11.0.5721.5145 # Windows-Media-Player/11.0.5721.5145 -Windows-Media-Player 10.00.00.xxxx # Windows-Media-Player/10.00.00.xxxx -Windows-Media-Player 10.00.00.4036 # Windows-Media-Player/10.00.00.4036 -Windows-Media-Player 10.00.00.3646 # Windows-Media-Player/10.00.00.3646 -Windows-Media-Player 9.00.00.4503 # Windows-Media-Player/9.00.00.4503 -Windows-Media-Player 9.00.00.3250 # Windows-Media-Player/9.00.00.3250 -CSSCheck 1.2.2 # CSSCheck/1.2.2 -Cynthia 1.0 # Cynthia 1.0 -HTMLParser 1.6 # HTMLParser/1.6 -P3P Validator # P3P Validator -W3C_Validator 1.654 # W3C_Validator/1.654 -W3C_Validator 1.606 # W3C_Validator/1.606 -W3C_Validator 1.591 # W3C_Validator/1.591 -W3C_Validator 1.575 # W3C_Validator/1.575 -W3C_Validator 1.555 # W3C_Validator/1.555 -W3C_Validator 1.432.2.5 # W3C_Validator/1.432.2.5 -W3C_Validator 1.432.2.22 # W3C_Validator/1.432.2.22 -W3C_Validator 1.432.2.19 # W3C_Validator/1.432.2.19 -W3C_Validator 1.432.2.10 # W3C_Validator/1.432.2.10 -W3C_Validator 1.305.2.12 # W3C_Validator/1.305.2.12 libwww-perl/5.64 -WDG_Validator 1.6.2 # WDG_Validator/1.6.2 +robot # AppEngine-Google; (+http://code.google.com/appengine; appid: webetrex) +robot # AppEngine-Google; (+http://code.google.com/appengine; appid: unblock4myspace) +robot # AppEngine-Google; (+http://code.google.com/appengine; appid: tunisproxy) +robot # AppEngine-Google; (+http://code.google.com/appengine; appid: proxy-in-rs) +robot # AppEngine-Google; (+http://code.google.com/appengine; appid: proxy-ba-k) +robot # AppEngine-Google; (+http://code.google.com/appengine; appid: moelonepyaeshan) +robot # AppEngine-Google; (+http://code.google.com/appengine; appid: mirrorrr) +robot # AppEngine-Google; (+http://code.google.com/appengine; appid: mapremiereapplication) +robot # AppEngine-Google; (+http://code.google.com/appengine; appid: longbows-hideout) +robot # AppEngine-Google; (+http://code.google.com/appengine; appid: eduas23) +robot # AppEngine-Google; (+http://code.google.com/appengine; appid: craigserver) +robot # AppEngine-Google; ( http://code.google.com/appengine; appid: proxy-ba-k) +robot # ABACHOBot +robot # Accoona-AI-Agent/1.1.2 (aicrawler at accoonabot dot com) +robot # Accoona-AI-Agent/1.1.2 +robot # Accoona-AI-Agent/1.1.1 (crawler at accoona dot com) +robot # Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.0) AddSugarSpiderBot www.idealobserver.com +robot # Mozilla/5.0 (compatible; AnyApexBot/1.0; +http://www.anyapex.com/bot.html) +robot # Mozilla/4.0 (compatible; Arachmo) +robot # Mozilla/4.0 (compatible; B-l-i-t-z-B-O-T) +robot # Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html) +robot # Baiduspider+(+http://www.baidu.com/search/spider_jp.html) +robot # Baiduspider+(+http://www.baidu.com/search/spider.htm) +robot # BaiDuSpider +robot # Mozilla/5.0 (compatible; BecomeBot/3.0; MSIE 6.0 compatible; +http://www.become.com/site_owners.html) +robot # Mozilla/5.0 (compatible; BecomeBot/2.3; MSIE 6.0 compatible; +http://www.become.com/site_owners.html) +robot # Mozilla/5.0 (compatible; BeslistBot; nl; BeslistBot 1.0; http://www.beslist.nl/ +robot # BillyBobBot/1.0 (+http://www.billybobbot.com/crawler/) +robot # Bimbot/1.0 +robot # Mozilla/5.0 (compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm) +robot # Mozilla/5.0 (compatible; bingbot/2.0 +http://www.bing.com/bingbot.htm) +robot # Mozilla/4.0 (compatible; BlitzBot) +robot # [email protected] (Mozilla compatible) +robot # [email protected] (Mozilla compatible) +robot # boitho.com-dc/0.85 ( http://www.boitho.com/dcbot.html ) +robot # boitho.com-dc/0.83 ( http://www.boitho.com/dcbot.html ) +robot # boitho.com-dc/0.82 ( http://www.boitho.com/dcbot.html ) +robot # boitho.com-dc/0.81 ( http://www.boitho.com/dcbot.html ) +robot # boitho.com-dc/0.79 ( http://www.boitho.com/dcbot.html ) +robot # boitho.com-robot/1.1 +robot # boitho.com-robot/1.0 +robot # btbot/0.4 (+http://www.btbot.com/btbot.html) +robot # CatchBot/2.0; +http://www.catchbot.com +robot # CatchBot/1.0; +http://www.catchbot.com +robot # CatchBot/1.0; http://www.catchbot.com +robot # Mozilla/4.0 (compatible; Cerberian Drtrs Version-3.2-Build-1) +robot # Mozilla/4.0 (compatible; Cerberian Drtrs Version-3.2-Build-0) +robot # Mozilla/5.0 (compatible; Charlotte/1.1; http://www.searchme.com/support/) +robot # Mozilla/5.0 (compatible; Charlotte/1.0t; http://www.searchme.com/support/) +robot # Mozilla/5.0 (compatible; Charlotte/1.0b; http://www.searchme.com/support/) +robot # Mozilla/5.0 (compatible; Charlotte/1.0b; http://www.betaspider.com/) +robot # Mozilla/5.0 (X11; U; Linux i686 (x86_64); en-US; rv:1.8.1.11) Gecko/20080109 (Charlotte/0.9t; http://www.searchme.com/support/) (Charlotte/0.9t; http://www.searchme.com/support/) +robot # Mozilla/5.0 (X11; U; Linux i686 (x86_64); en-US; rv:1.8.1.11) Gecko/20080109 (Charlotte/0.9t; http://www.searchme.com/support/) +robot # Mozilla/5.0 (compatible; Charlotte/0.9t; http://www.searchme.com/support/) +robot # Mozilla/5.0 (compatible; Charlotte/0.9t; +http://www.searchme.com/support/) +robot # ConveraCrawler/0.9e (+http://ews.converasearch.com/crawl.htm) +robot # ConveraCrawler/0.9d (+http://www.authoritativeweb.com/crawl) +robot # ConveraCrawler/0.9d ( http://www.authoritativeweb.com/crawl) +robot # ConveraCrawler/0.9 (+http://www.authoritativeweb.com/crawl) +robot # cosmos/0.9_([email protected]) +robot # DataparkSearch/4.37-23012006 ( http://www.dataparksearch.org/) +robot # DataparkSearch/4.36 ( http://www.dataparksearch.org/) +robot # DataparkSearch/4.35-02122005 ( http://www.dataparksearch.org/) +robot # DataparkSearch/4.35 ( http://www.dataparksearch.org/) +robot # DiamondBot +robot # Mozilla/5.0 (compatible; discobot/1.0; +http://discoveryengine.com/discobot.html) +robot # Mozilla/5.0 (compatible; DotBot/1.1; http://www.dotnetdotcom.org/, [email protected]) +robot # DotBot/1.0.1 (http://www.dotnetdotcom.org/#info, [email protected]) +robot # EmeraldShield.com WebBot (http://www.emeraldshield.com/webbot.aspx) +robot # envolk[ITS]spider/1.6 (+http://www.envolk.com/envolkspider.html) +robot # envolk[ITS]spider/1.6 ( http://www.envolk.com/envolkspider.html) +robot # EsperanzaBot(+http://www.esperanza.to/bot/) +robot # Exabot/2.0 +robot # FAST Enterprise Crawler 6 used by Schibsted ([email protected]) +robot # FAST Enterprise Crawler 6 / Scirus [email protected]; http://www.scirus.com/srsapp/contactus/ +robot # FAST Enteprise Crawler/6 (www dot fastsearch dot com) +robot # FAST-WebCrawler/3.8 (atw-crawler at fast dot no; http://fast.no/support/crawler.asp) +robot # FAST-WebCrawler/3.7/FirstPage (atw-crawler at fast dot no;http://fast.no/support/crawler.asp) +robot # FAST-WebCrawler/3.7 (atw-crawler at fast dot no; http://fast.no/support/crawler.asp) +robot # FAST-WebCrawler/3.6/FirstPage (atw-crawler at fast dot no;http://fast.no/support/crawler.asp) +robot # FAST-WebCrawler/3.6 (atw-crawler at fast dot no; http://fast.no/support/crawler.asp) +robot # FAST-WebCrawler/3.6 +robot # FAST-WebCrawler/3.x Multimedia (mm dash crawler at fast dot no) +robot # FAST-WebCrawler/3.x Multimedia +robot # Mozilla/4.0 (compatible: FDSE robot) +robot # findlinks/2.0.1 (+http://wortschatz.uni-leipzig.de/findlinks/) +robot # findlinks/1.1.6-beta6 (+http://wortschatz.uni-leipzig.de/findlinks/) +robot # findlinks/1.1.6-beta4 (+http://wortschatz.uni-leipzig.de/findlinks/) +robot # findlinks/1.1.6-beta1 (+http://wortschatz.uni-leipzig.de/findlinks/) +robot # findlinks/1.1.5-beta7 (+http://wortschatz.uni-leipzig.de/findlinks/) +robot # findlinks/1.1.4-beta1 (+http://wortschatz.uni-leipzig.de/findlinks/) +robot # findlinks/1.1.3-beta9 (+http://wortschatz.uni-leipzig.de/findlinks/) +robot # findlinks/1.1.3-beta8 (+http://wortschatz.uni-leipzig.de/findlinks/) +robot # findlinks/1.1.3-beta6 (+http://wortschatz.uni-leipzig.de/findlinks/) +robot # findlinks/1.1.3-beta4 (+http://wortschatz.uni-leipzig.de/findlinks/) +robot # findlinks/1.1.3-beta2 (+http://wortschatz.uni-leipzig.de/findlinks/) +robot # findlinks/1.1.3-beta1 (+http://wortschatz.uni-leipzig.de/findlinks/) +robot # findlinks/1.1.2-a5 (+http://wortschatz.uni-leipzig.de/findlinks/) +robot # findlinks/1.1.1-a5 (+http://wortschatz.uni-leipzig.de/findlinks/) +robot # findlinks/1.1.1-a1 (+http://wortschatz.uni-leipzig.de/findlinks/) +robot # findlinks/1.1.1 (+http://wortschatz.uni-leipzig.de/findlinks/) +robot # findlinks/1.1-a9 (+http://wortschatz.uni-leipzig.de/findlinks/) +robot # findlinks/1.1-a8 (+http://wortschatz.uni-leipzig.de/findlinks/) +robot # findlinks/1.1-a8 ( http://wortschatz.uni-leipzig.de/findlinks/) +robot # findlinks/1.1-a7 (+http://wortschatz.uni-leipzig.de/findlinks/) +robot # findlinks/1.1-a5 (+http://wortschatz.uni-leipzig.de/findlinks/) +robot # findlinks/1.1-a4 (+http://wortschatz.uni-leipzig.de/findlinks/) +robot # findlinks/1.1-a3 (+http://wortschatz.uni-leipzig.de/findlinks/) +robot # findlinks/1.1 (+http://wortschatz.uni-leipzig.de/findlinks/) +robot # findlinks/1.06 (+http://wortschatz.uni-leipzig.de/findlinks/) +robot # findlinks/1.0.9 (+http://wortschatz.uni-leipzig.de/findlinks/) +robot # findlinks/1.0.8 (+http://wortschatz.uni-leipzig.de/findlinks/) +robot # findlinks/1.0 (+http://wortschatz.uni-leipzig.de/findlinks/) +robot # Mozilla/4.0 compatible FurlBot/Furl Search 2.0 (FurlBot; http://www.furl.net; [email protected]) +robot # FyberSpider (+http://www.fybersearch.com/fyberspider.php) +robot # FyberSpider +robot # g2Crawler [email protected] +robot # Gaisbot/3.0+([email protected];+http://gais.cs.ccu.edu.tw/robot.php) +robot # Gaisbot/3.0+([email protected];+http://gais.cs.ccu.edu.tw/robot.php) +robot # Gaisbot/3.0 ([email protected]; http://gais.cs.ccu.edu.tw/robot.php) +robot # GalaxyBot/1.0 (http://www.galaxy.com/galaxybot.html) +robot # genieBot (http://64.5.245.11/faq/faq.html) +robot # genieBot ((http://64.5.245.11/faq/faq.html)) +robot # Gigabot/3.0 (http://www.gigablast.com/spider.html) +robot # Gigabot/2.0/gigablast.com/spider.html +robot # Gigabot/2.0 (http://www.gigablast.com/spider.html) +robot # Gigabot/2.0 +robot # Gigabot/1.0 +robot # Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.2; SV1; .NET CLR 1.1.4322; Girafabot [girafa.com]) +robot # Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 4.0; Girafabot; girafabot at girafa dot com; http://www.girafa.com) +robot # Mozilla/4.0 (compatible; MSIE 5.0; Windows NT; Girafabot; girafabot at girafa dot com; http://www.girafa.com) +robot # Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html) +robot # Googlebot/2.1 (+http://www.googlebot.com/bot.html) +robot # Googlebot/2.1 (+http://www.google.com/bot.html) +robot # Googlebot-Image/1.0 +robot # Mozilla/5.0 GurujiBot/1.0 (+http://www.guruji.com/en/WebmasterFAQ.html) +robot # Mozilla/5.0 GurujiBot/1.0 ( http://www.guruji.com/en/WebmasterFAQ.html) +robot # Mozilla/5.0 (compatible; GurujiBot/1.0; +http://www.guruji.com/en/WebmasterFAQ.html) +robot # GurujiBot/1.0 (+http://www.guruji.com/WebmasterFAQ.html) +robot # GurujiBot/1.0 (+http://www.guruji.com/en/WebmasterFAQ.html) +robot # HappyFunBot/1.1 ( http://www.happyfunsearch.com/bot.html) +robot # holmes/3.9 (someurl.co.cc) +robot # holmes/3.12.4 (http://morfeo.centrum.cz/bot) +robot # holmes/3.12.3 (http://morfeo.centrum.cz/bot) +robot # holmes/3.12.2 (http://morfeo.centrum.cz/bot) +robot # holmes/3.12.1 (http://morfeo.centrum.cz/bot) +robot # htdig/3.1.6 ([email protected]) +robot # htdig/3.1.6 ([email protected]) +robot # htdig/3.1.5 ([email protected]) +robot # htdig/3.1.5 (root@localhost) +robot # htdig/3.1.5 ([email protected]) +robot # htdig/3.1.5 +robot # iaskspider/2.0(+http://iask.com/help/help_index.html) +robot # iaskspider +robot # ia_archiver/8.9 (Windows NT 3.1; en-US;) +robot # ia_archiver/8.9 (Windows 3.9; en-US;) +robot # ia_archiver/8.9 (Linux 1.0; en-US;) +robot # ia_archiver/8.8 (Windows XP 7.2; en-US;) +robot # ia_archiver/8.8 (Windows XP 3.0; en-US;) +robot # ia_archiver/8.2 (Windows 7.6; en-US;) +robot # ia_archiver/8.1 (Windows 2000 1.9; en-US;) +robot # ia_archiver/8.0 (Windows 2.4; en-US;) +robot # ia_archiver +robot # iCCrawler (http://www.iccenter.net/bot.htm) +robot # ichiro/4.0 (http://help.goo.ne.jp/door/crawler.html) +robot # ichiro/3.0 (http://help.goo.ne.jp/door/crawler.html) +robot # ichiro/2.0+(http://help.goo.ne.jp/door/crawler.html) +robot # ichiro/2.0 ([email protected]) +robot # ichiro/2.0 (http://help.goo.ne.jp/door/crawler.html) +robot # igdeSpyder (compatible; igde.ru; +http://igde.ru/doc/tech.html) +robot # IRLbot/3.0 (compatible; MSIE 6.0; http://irl.cs.tamu.edu/crawler/) +robot # IRLbot/3.0 (compatible; MSIE 6.0; http://irl.cs.tamu.edu/crawler) +robot # IRLbot/2.0 (compatible; MSIE 6.0; http://irl.cs.tamu.edu/crawler) +robot # IRLbot/2.0 (+http://irl.cs.tamu.edu/crawler) +robot # IRLbot/2.0 ( http://irl.cs.tamu.edu/crawler) +robot # IssueCrawler +robot # Jaxified Bot (+http://www.jaxified.com/crawler/) +robot # Jyxobot/1 +robot # Mozilla/5.0 (compatible; KoepaBot BETA; http://www.koepa.nl/bot.html) +robot # LapozzBot/1.4 (+http://robot.lapozz.com) +robot # Mozilla/5.0 [email protected] +robot # larbin_2.6.3 [email protected] +robot # larbin_2.6.3 [email protected] +robot # larbin_2.6.3 [email protected] +robot # larbin_2.6.3 [email protected] +robot # larbin_2.6.3 [email protected] +robot # larbin_2.6.3 [email protected] +robot # larbin_2.6.3 [email protected] +robot # larbin_2.6.3 ([email protected]) +robot # larbin_2.6.3 ([email protected]) +robot # larbin_2.6.3 ([email protected]) +robot # larbin_2.6.3 ([email protected]) +robot # larbin_2.6.2 [email protected] +robot # larbin_2.6.2 [email protected] +robot # larbin_2.6.2 listonATccDOTgatechDOTedu +robot # larbin_2.6.2 [email protected] +robot # larbin_2.6.2 [email protected] +robot # larbin_2.6.2 [email protected] +robot # larbin_2.6.2 [email protected] +robot # larbin_2.6.2 ([email protected]) +robot # larbin_2.6.2 ([email protected]) +robot # larbin_2.6.2 ([email protected]) +robot # larbin_2.6.2 ([email protected]) +robot # larbin_2.6.1 [email protected] +robot # larbin_2.5.0 ([email protected]) +robot # larbin_xy250 [email protected] +robot # ldspider (http://code.google.com/p/ldspider/wiki/Robots) +robot # LexxeBot/1.0 ([email protected]) +robot # Linguee Bot (http://www.linguee.com/bot; [email protected]) +robot # LinkWalker/2.0 +robot # LinkWalker +robot # lmspider [email protected] +robot # lmspider ([email protected]) +robot # lwp-trivial/1.41 +robot # lwp-trivial/1.38 +robot # lwp-trivial/1.36 +robot # lwp-trivial/1.35 +robot # lwp-trivial/1.33 +robot # magpie-crawler/1.1 (U; Linux amd64; en-GB; +http://www.brandwatch.net) +robot # Mediapartners-Google/2.1 +robot # Mozilla/5.0 (compatible; MJ12bot/v1.2.4; http://www.majestic12.co.uk/bot.php?+) +robot # Mozilla/5.0 (compatible; MJ12bot/v1.2.3; http://www.majestic12.co.uk/bot.php?+) +robot # MJ12bot/v1.0.8 (http://majestic12.co.uk/bot.php?+) +robot # MJ12bot/v1.0.7 (http://majestic12.co.uk/bot.php?+) +robot # MJ12bot/v1.0.6 (http://majestic12.co.uk/bot.php?+) +robot # MJ12bot/v1.0.5 (http://majestic12.co.uk/bot.php?+) +robot # Mnogosearch-3.1.21 +robot # Mozilla/5.0 (compatible; MojeekBot/2.0; http://www.mojeek.com/bot.html) +robot # MojeekBot/0.2 (archi; http://www.mojeek.com/bot.html) +robot # Moreoverbot/5.1 ( http://w.moreover.com; [email protected]) Mozilla/5.0 +robot # Moreoverbot/5.00 (+http://www.moreover.com; [email protected]) +robot # Moreoverbot/5.00 (+http://www.moreover.com) +robot # Morning Paper 1.0 (robots.txt compliant!) +robot # msnbot/2.1 +robot # msnbot/2.0b +robot # msnbot/1.1 (+http://search.msn.com/msnbot.htm) +robot # msnbot/1.1 +robot # msnbot/1.0 (+http://search.msn.com/msnbot.htm) +robot # msnbot/0.9 (+http://search.msn.com/msnbot.htm) +robot # msnbot/0.11 ( http://search.msn.com/msnbot.htm) +robot # MSNBOT/0.1 (http://search.msn.com/msnbot.htm) +robot # MSRBOT (http://research.microsoft.com/research/sv/msrbot/) +robot # MSRBOT +robot # MVAClient +robot # Mozilla/5.0 (compatible; mxbot/1.0; +http://www.chainn.com/mxbot.html) +robot # Mozilla/5.0 (compatible; mxbot/1.0; http://www.chainn.com/mxbot.html) +robot # NetResearchServer/4.0(loopimprovements.com/robot.html) +robot # NetResearchServer/3.5(loopimprovements.com/robot.html) +robot # NetResearchServer/2.8(loopimprovements.com/robot.html) +robot # NetResearchServer/2.7(loopimprovements.com/robot.html) +robot # NetResearchServer/2.5(loopimprovements.com/robot.html) +robot # NetResearchServer(http://www.look.com) +robot # Mozilla/5.0 (compatible; NetSeer crawler/2.0; +http://www.netseer.com/crawler.html; [email protected]) +robot # NewsGator/2.5 (http://www.newsgator.com; Microsoft Windows NT 5.1.2600.0; .NET CLR 1.1.4322.2032) +robot # NewsGator/2.0 Bot (http://www.newsgator.com) +robot # NG-Search/0.9.8 (http://www.ng-search.com) +robot # NG-Search/0.86 (+http://www.ng-search.com) +robot # NG-Search/0.86 ( http://www.ng-search.com) +robot # nicebot +robot # noxtrumbot/1.0 ([email protected]) +robot # Nusearch Spider (www.nusearch.com) +robot # nuSearch Spider (compatible; MSIE 4.01; Windows NT) +robot # NutchCVS/0.8-dev (Nutch; http://lucene.apache.org/nutch/bot.html; [email protected]) +robot # NutchCVS/0.7.2 (Nutch; http://lucene.apache.org/nutch/bot.html; [email protected]) +robot # NutchCVS/0.7.1 (Nutch; http://lucene.apache.org/nutch/bot.html; [email protected]) +robot # NutchCVS/0.7.1 (Nutch running at UW; http://crawlers.cs.washington.edu/; [email protected]) +robot # NutchCVS/0.7 (Nutch; http://lucene.apache.org/nutch/bot.html; [email protected]) +robot # NutchCVS/0.06-dev (Nutch; http://www.nutch.org/docs/en/bot.html; [email protected]) +robot # NutchCVS/0.06-dev (Nutch; http://www.nutch.org/docs/en/bot.html; [email protected]) +robot # NutchCVS/0.05 (Nutch; http://www.nutch.org/docs/en/bot.html; [email protected]) +robot # Nymesis/1.0 (http://nymesis.com) +robot # Mozilla/4.0 (compatible; MSIE 5.5; Windows NT 4.0; obot) +robot # oegp v. 1.3.0 +robot # omgilibot/0.4 +http://omgili.com +robot # omgilibot/0.3 +http://www.omgili.com/Crawler.html +robot # omgilibot/0.3 http://www.omgili.com/Crawler.html +robot # OmniExplorer_Bot/6.70 (+http://www.omni-explorer.com) WorldIndexer +robot # OmniExplorer_Bot/6.65a (+http://www.omni-explorer.com) WorldIndexer +robot # OmniExplorer_Bot/6.63b (+http://www.omni-explorer.com) WorldIndexer +robot # OmniExplorer_Bot/6.62 (+http://www.omni-explorer.com) WorldIndexer +robot # OmniExplorer_Bot/6.60 (+http://www.omni-explorer.com) WorldIndexer +robot # OmniExplorer_Bot/6.47 (+http://www.omni-explorer.com) WorldIndexer +robot # OmniExplorer_Bot/5.91c (+http://www.omni-explorer.com) WorldIndexer +robot # OmniExplorer_Bot/5.28 (+http://www.omni-explorer.com) WorldIndexer +robot # OmniExplorer_Bot/5.25 (+http://www.omni-explorer.com) WorldIndexer +robot # OmniExplorer_Bot/5.20 (+http://www.omni-explorer.com) WorldIndexer +robot # OmniExplorer_Bot/5.01 (+http://www.omni-explorer.com) WorldIndexer +robot # OmniExplorer_Bot/4.80 (+http://www.omni-explorer.com) WorldIndexer +robot # OmniExplorer_Bot/4.32 (+http://www.omni-explorer.com) WorldIndexer +robot # OOZBOT/0.20 ( -- ; http://www.setooz.com/oozbot.html ; agentname at setooz dot_com ) +robot # OOZBOT/0.17 (--; http://www.setooz.com/oozbot.html; pvvpr at iiit dot ac dot in) +robot # Orbiter (+http://www.dailyorbit.com/bot.htm) +robot # PageBitesHyperBot/600 (http://www.pagebites.com/) +robot # Mozilla/5.0 (compatible; Peew/1.0; http://www.peew.de/crawler/) +robot # polybot 1.0 (http://cis.poly.edu/polybot/) +robot # Pompos/1.3 http://dir.com/pompos.html +robot # Pompos/1.2 http://pompos.iliad.fr +robot # Pompos/1.1 http://pompos.iliad.fr +robot # PostPost/1.0 (+http://postpo.st/crawlers) +robot # psbot/0.1 (+http://www.picsearch.com/bot.html) +robot # PycURL/7.23.1 +robot # PycURL/7.19.7 +robot # PycURL/7.19.5 +robot # PycURL/7.19.3 +robot # PycURL/7.19.0 +robot # PycURL/7.18.2 +robot # PycURL/7.18.0 +robot # PycURL/7.16.4 +robot # PycURL/7.15.5 +robot # PycURL/7.13.2 +robot # PycURL +robot # Qseero v1.0.0 +robot # radian6_default_(www.radian6.com/crawler) +robot # RAMPyBot - www.giveRAMP.com/0.1 (RAMPyBot - www.giveRAMP.com; http://www.giveramp.com/bot.html; [email protected]) +robot # RufusBot (Rufus Web Miner; http://64.124.122.252/feedback.html) +robot # SandCrawler - Compatibility Testing +robot # SBIder/0.8-dev (SBIder; http://www.sitesell.com/sbider.html; http://support.sitesell.com/contact-support.html) +robot # Mozilla/5.0 (compatible; ScoutJet; http://www.scoutjet.com/) +robot # Scrubby/2.2 (http://www.scrubtheweb.com/) +robot # Mozilla/5.0 (compatible; Scrubby/2.2; +http://www.scrubtheweb.com/) +robot # Mozilla/5.0 (compatible; Scrubby/2.2; http://www.scrubtheweb.com/) +robot # Scrubby/2.1 (http://www.scrubtheweb.com/) +robot # Mozilla/5.0 (compatible; Scrubby/2.1; +http://www.scrubtheweb.com/abs/meta-check.html) +robot # SearchSight/2.0 (http://SearchSight.com/) +robot # Seekbot/1.0 (http://www.seekbot.net/bot.html) RobotsTxtFetcher/1.2 +robot # Seekbot/1.0 (http://www.seekbot.net/bot.html) HTTPFetcher/2.1 +robot # Seekbot/1.0 (http://www.seekbot.net/bot.html) HTTPFetcher/0.3 +robot # Seekbot/1.0 (http://www.seekbot.net/bot.html) +robot # semanticdiscovery/0.1 +robot # Sensis Web Crawler (search_comments\at\sensis\dot\com\dot\au) +robot # Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.0) SEOChat::Bot v1.1 +robot # SeznamBot/2.0 (+http://fulltext.seznam.cz/) +robot # SeznamBot/2.0 (+http://fulltext.sblog.cz/robot/) +robot # Shim-Crawler(Mozilla-compatible; http://www.logos.ic.i.u-tokyo.ac.jp/crawler/; [email protected]) +robot # Shim-Crawler +robot # ShopWiki/1.0 ( +http://www.shopwiki.com/wiki/Help:Bot) +robot # Mozilla/4.0 (compatible: Shoula robot) +robot # Mozilla/5.0 (compatible; SiteBot/0.1; +http://www.sitebot.org/robot/) +robot # Mozilla/5.0 (compatible; SiteBot/0.1; http://www.sitebot.org/robot/) +robot # Snappy/1.1 ( http://www.urltrends.com/ ) +robot # sogou spider +robot # Sosospider+(+http://help.soso.com/webspider.htm) +robot # Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US) Speedy Spider (http://www.entireweb.com/about/search_tech/speedy_spider/) +robot # Mozilla/5.0 (compatible; Speedy Spider; http://www.entireweb.com/about/search_tech/speedy_spider/) +robot # Speedy Spider (Entireweb; Beta/1.3; http://www.entireweb.com/about/search_tech/speedyspider/) +robot # Speedy Spider (Entireweb; Beta/1.2; http://www.entireweb.com/about/search_tech/speedyspider/) +robot # Speedy Spider (Entireweb; Beta/1.1; http://www.entireweb.com/about/search_tech/speedyspider/) +robot # Speedy Spider (Entireweb; Beta/1.0; http://www.entireweb.com/about/search_tech/speedyspider/) +robot # Speedy Spider (Beta/1.0; www.entireweb.com) +robot # Speedy Spider (http://www.entireweb.com/about/search_tech/speedy_spider/) +robot # Speedy Spider (http://www.entireweb.com/about/search_tech/speedyspider/) +robot # Speedy Spider (http://www.entireweb.com) +robot # Sqworm/2.9.85-BETA (beta_release; 20011115-775; i686-pc-linux-gnu) +robot # StackRambler/2.0 (MSIE incompatible) +robot # StackRambler/2.0 +robot # Mozilla/5.0 (compatible; suggybot v0.01a, http://blog.suggy.com/was-ist-suggy/suggy-webcrawler/) +robot # SurveyBot/2.3+(Whois+Source) +robot # SurveyBot/2.3 (Whois Source) +robot # SynooBot/0.7.1 (SynooBot; http://www.synoo.de/bot.html; [email protected]) +robot # Mozilla/2.0 (compatible; Ask Jeeves/Teoma; +http://sp.ask.com/docs/about/tech_crawling.html) +robot # Mozilla/2.0 (compatible; Ask Jeeves/Teoma; +http://about.ask.com/en/docs/about/webmasters.shtml) +robot # Mozilla/2.0 (compatible; Ask Jeeves/Teoma) +robot # TerrawizBot/1.0 (+http://www.terrawiz.com/bot.html) +robot # TheSuBot/0.2 (www.thesubot.de) +robot # TheSuBot/0.1 (www.thesubot.de) +robot # Thumbnail.CZ robot 1.1 (http://thumbnail.cz/why-no-robots-txt.html) +robot # TinEye/1.1 (http://tineye.com/crawler.html) +robot # TinEye +robot # truwoGPS/1.0 (GNU/Linux; U; i686; en-US; +http://www.lan4lano.net/browser.html ) +robot # TurnitinBot/2.1 (http://www.turnitin.com/robot/crawlerinfo.html) +robot # TurnitinBot/2.0 http://www.turnitin.com/robot/crawlerinfo.html +robot # TurnitinBot/1.5 http://www.turnitin.com/robot/crawlerinfo.html +robot # TurnitinBot/1.5 (http://www.turnitin.com/robot/crawlerinfo.html) +robot # TurnitinBot/1.5 http://www.turnitin.com/robot/crawlerinfo.html +robot # TurnitinBot/1.5 (http://www.turnitin.com/robot/crawlerinfo.html) +robot # Mozilla/5.0 (compatible; TweetedTimes Bot/1.0; http://tweetedtimes.com) +robot # TwengaBot +robot # updated/0.1-beta (updated; http://www.updated.com; [email protected]) +robot # Mozilla/5.0 (compatible; Urlfilebot/2.2; +http://urlfile.com/bot.html) +robot # Mozilla/4.0 (compatible; Vagabondo/4.0Beta; webcrawler at wise-guys dot nl; http://webagent.wise-guys.nl/; http://www.wise-guys.nl/) +robot # Mozilla/4.0 (compatible; Vagabondo/2.2; webcrawler at wise-guys dot nl; http://webagent.wise-guys.nl/) +robot # Mozilla/5.0 (compatible; Vagabondo/2.1; webcrawler at wise-guys dot nl; http://webagent.wise-guys.nl/) +robot # Mozilla/3.0 (Vagabondo/2.0 MT; [email protected]; http://aanmelden.ilse.nl/?aanmeld_mode=webhints) +robot # Mozilla/4.0 (compatible; MSIE 5.0; Windows 95) VoilaBot BETA 1.2 (http://www.voila.com/) +robot # Vortex/2.2 (+http://marty.anstey.ca/robots/vortex/) +robot # Vortex/2.2 ( http://marty.anstey.ca/robots/vortex/) +robot # VORTEX/1.2 ( http://marty.anstey.ca/robots/vortex/) +robot # voyager/2.0 (http://www.kosmix.com/crawler.html) +robot # voyager/1.0 +robot # VYU2 (GNU; OpenRISC) +robot # webcollage/1.93 +robot # webcollage/1.129 +robot # webcollage/1.125 +robot # webcollage/1.117 +robot # webcollage/1.114 +robot # Websquash.com (Add url robot) +robot # http://www.almaden.ibm.com/cs/crawler [wf84] +robot # WoFindeIch Robot 1.0(+http://www.search.wofindeich.com/robot.php) +robot # WoFindeIch Robot 1.0( http://www.search.wofindeich.com/robot.php) +robot # WomlpeFactory/0.1 (+http://www.Womple.com/bot.html) +robot # Xaldon_WebSpider/2.0.b1 +robot # Xaldon_WebSpider/2.0.b1 +robot # yacybot (x86 Windows XP 5.1; java 1.6.0_12; Europe/de) http://yacy.net/bot.html +robot # yacybot (x86 Windows XP 5.1; java 1.6.0_11; Europe/de) http://yacy.net/bot.html +robot # yacybot (x86 Windows XP 5.1; java 1.6.0; Europe/de) http://yacy.net/yacy/bot.html +robot # yacybot (x86 Windows 2000 5.0; java 1.6.0_16; Europe/de) http://yacy.net/bot.html +robot # yacybot (ppc Mac OS X 10.5.2; java 1.5.0_13; Europe/de) http://yacy.net/bot.html +robot # yacybot (ppc Mac OS X 10.4.10; java 1.5.0_07; Europe/de) http://yacy.net/bot.html +robot # yacybot (i386 Mac OS X 10.5.7; java 1.5.0_16; Europe/de) http://yacy.net/bot.html +robot # yacybot (i386 Linux 2.6.9-023stab046.2-smp; java 1.6.0_05; Europe/en) http://yacy.net/bot.html +robot # yacybot (i386 Linux 2.6.8-022stab070.5-enterprise; java 1.4.2-03; Europe/en) yacy.net +robot # yacybot (i386 Linux 2.6.31-16-generic; java 1.6.0_15; Europe/en) http://yacy.net/bot.html +robot # yacybot (i386 Linux 2.6.26-2-686; java 1.6.0_0; Europe/en) http://yacy.net/bot.html +robot # yacybot (i386 Linux 2.6.24-28-generic; java 1.6.0_20; Europe/en) http://yacy.net/bot.html +robot # yacybot (i386 Linux 2.6.24-24-generic; java 1.6.0_07; Europe/en) http://yacy.net/bot.html +robot # yacybot (i386 Linux 2.6.24-23-generic; java 1.6.0_16; Europe/en) http://yacy.net/bot.html +robot # yacybot (i386 Linux 2.6.23; java 1.6.0_17; Europe/en) http://yacy.net/bot.html +robot # yacybot (i386 Linux 2.6.23; java 1.6.0_04; Europe/en) http://yacy.net/bot.html +robot # yacybot (i386 Linux 2.6.22-14-generic; java 1.6.0_03; Europe/de) http://yacy.net/bot.html +robot # yacybot (amd64 Windows 7 6.1; java 1.6.0_17; Europe/de) http://yacy.net/bot.html +robot # yacybot (amd64 Linux 2.6.28-18-generic; java 1.6.0_0; Europe/en) http://yacy.net/bot.html +robot # yacybot (amd64 Linux 2.6.16-2-amd64-k8-smp; java 1.5.0_10; Europe/en) http://yacy.net/yacy/bot.html +robot # Mozilla/5.0 (compatible; Yahoo! Slurp; http://help.yahoo.com/help/us/ysearch/slurp) +robot # Mozilla/5.0 (compatible; Yahoo! Slurp China; http://misc.yahoo.com.cn/help.html) +robot # YahooSeeker/1.2 (compatible; Mozilla 4.0; MSIE 5.5; yahooseeker at yahoo-inc dot com ; http://help.yahoo.com/help/us/shop/merchant/) +robot # YahooSeeker-Testing/v3.9 (compatible; Mozilla 4.0; MSIE 5.5; http://search.yahoo.com/) +robot # Mozilla/5.0 (compatible; YandexBot/3.0; +http://yandex.com/bots) +robot # Mozilla/5.0 (compatible; YandexImages/3.0; +http://yandex.com/bots) +robot # Yasaklibot/v1.2 (http://www.Yasakli.com/bot.php) +robot # Yeti/1.0 (NHN Corp.; http://help.naver.com/robots/) +robot # Yeti/1.0 (+http://help.naver.com/robots/) +robot # Mozilla/5.0 (compatible; YodaoBot/1.0; http://www.yodao.com/help/webmaster/spider/; ) +robot # yoogliFetchAgent/0.1 +robot # Mozilla/5.0 (compatible; YoudaoBot/1.0; http://www.youdao.com/help/webmaster/spider/; ) +robot # Zao/0.1 (http://www.kototoi.org/zao/) +robot # Zao/0.1 (http://www.kototoi.org/zao/) +robot # Mozilla/4.0 (compatible; Zealbot 1.0) +robot # zspider/0.9-dev http://feedback.redkolibri.com/ +robot # Mozilla/4.0 compatible ZyBorg/1.0 DLC ([email protected]; http://www.WISEnutbot.com) +robot # Mozilla/4.0 compatible ZyBorg/1.0 Dead Link Checker ([email protected]; http://www.WISEnutbot.com) +robot # Mozilla/4.0 compatible ZyBorg/1.0 Dead Link Checker ([email protected]; http://www.WISEnutbot.com) +robot # Mozilla/4.0 compatible ZyBorg/1.0 ([email protected]; http://www.WISEnutbot.com) +robot # Mozilla/4.0 compatible ZyBorg/1.0 ([email protected]; http://www.WISEnutbot.com) +robot # Mozilla/4.0 compatible ZyBorg/1.0 ([email protected]; http://www.WISEnutbot.com) +robot # EmailSiphon +robot # Bloglines/3.1 (http://www.bloglines.com) +robot # Bloglines/3.0-rho (http://www.bloglines.com; 3 subscribers) +robot # everyfeed-spider/2.0 (http://www.everyfeed.com) +robot # Feedfetcher-Google; (+http://www.google.com/feedfetcher.html; feed-id=8639390370582375869) +robot # Feedfetcher-Google; (+http://www.google.com/feedfetcher.html; feed-id=375807642710480585) +robot # Feedfetcher-Google; (+http://www.google.com/feedfetcher.html; feed-id=1992026586303346733) +robot # Feedfetcher-Google; (+http://www.google.com/feedfetcher.html; feed-id=13645798864011762265) +robot # FeedFetcher-Google; (+http://www.google.com/feedfetcher.html) +robot # GreatNews/1.0 +robot # Gregarius/0.5.2 (+http://devlog.gregarius.net/docs/ua) +robot # MagpieRSS/0.7 ( http://magpierss.sf.net) +robot # NFReader/1.4.1.0 (http://www.gaijin.at/) +robot # UniversalFeedParser/3.3 +http://feedparser.org/ +robot # BinGet/1.00.A (http://www.bin-co.com/php/scripts/load/) +robot # curl/7.9.8 (i686-pc-linux-gnu) libcurl 7.9.8 (OpenSSL 0.9.6b) (ipv6 enabled) +robot # curl/7.8 (i386-redhat-linux-gnu) libcurl 7.8 (OpenSSL 0.9.6b) (ipv6 enabled) +robot # curl/7.7.x (i386--freebsd4.3) libcurl 7.7.x (SSL 0.9.6) (ipv6 enabled) +robot # curl/7.7.2 (powerpc-apple-darwin6.0) libcurl 7.7.2 (OpenSSL 0.9.6b) +robot # curl/7.21.4 (universal-apple-darwin11.0) libcurl/7.21.4 OpenSSL/0.9.8r zlib/1.2.5 +robot # curl/7.21.3 (x86_64-unknown-linux-gnu) libcurl/7.21.3 OpenSSL/1.0.0c zlib/1.2.5 +robot # curl/7.21.3 (x86_64-redhat-linux-gnu) libcurl/7.21.3 NSS/3.13.1.0 zlib/1.2.5 libidn/1.19 libssh2/1.2.7 +robot # curl/7.21.2 (i386-pc-win32) libcurl/7.21.2 OpenSSL/0.9.8o zlib/1.2.5 +robot # curl/7.21.1 (i686-pc-linux-gnu) libcurl/7.21.1 OpenSSL/1.0.0a zlib/1.2.5 +robot # curl/7.21.0 (x86_64-pc-linux-gnu) libcurl/7.21.0 OpenSSL/0.9.8o zlib/1.2.3.4 libidn/1.18 libssh2/1.2.5 +robot # curl/7.21.0 (x86_64-pc-linux-gnu) libcurl/7.21.0 OpenSSL/0.9.8o zlib/1.2.3.4 libidn/1.18 +robot # curl/7.21.0 (x86_64-pc-linux-gnu) libcurl/7.21.0 OpenSSL/0.9.8o zlib/1.2.3.4 libidn/1.15 libssh2/1.2.5 +robot # curl/7.21.0 (x86_64-apple-darwin10.2.0) libcurl/7.21.0 OpenSSL/1.0.0a zlib/1.2.5 libidn/1.19 +robot # curl/7.21.0 (i686-pc-linux-gnu) libcurl/7.21.0 OpenSSL/0.9.8o zlib/1.2.3.4 libidn/1.18 +robot # curl/7.21.0 (i486-pc-linux-gnu) libcurl/7.21.0 OpenSSL/0.9.8o zlib/1.2.3.4 libidn/1.18 libssh2/1.2.6 +robot # curl/7.20.0 (i686-pc-linux-gnu) libcurl/7.20.0 OpenSSL/0.9.8n zlib/1.2.4 +robot # curl/7.20.0 (i386-apple-darwin9.8.0) libcurl/7.20.0 OpenSSL/0.9.8m zlib/1.2.3 libidn/1.16 +robot # curl/7.19.7 (x86_64-redhat-linux-gnu) libcurl/7.35.0 OpenSSL/1.0.1e zlib/1.2.3 libidn/1.18 libssh2/1.4.3 +robot # curl/7.19.7 (universal-apple-darwin10.0) libcurl/7.19.7 OpenSSL/0.9.8l zlib/1.2.3 +robot # curl/7.19.7 (i486-pc-linux-gnu) libcurl/7.19.7 OpenSSL/0.9.8o zlib/1.2.3.3 libidn/1.15 +robot # curl/7.19.7 (i486-pc-linux-gnu) libcurl/7.19.7 OpenSSL/0.9.8k zlib/1.2.3.3 libidn/1.15 +robot # curl/7.19.7 (i386-redhat-linux-gnu) libcurl/7.19.7 NSS/3.12.5.0 zlib/1.2.3 libidn/1.9 libssh2/1.2.2 +robot # curl/7.19.7 (i386-apple-darwin9.8.0) libcurl/7.19.7 zlib/1.2.3 +robot # curl/7.19.6 (i686-pc-cygwin) libcurl/7.19.6 OpenSSL/0.9.8n zlib/1.2.3 libidn/1.18 libssh2/1.2 +robot # curl/7.19.6 (i386-redhat-linux-gnu) libcurl/7.19.6 NSS/3.12.4.5 zlib/1.2.3 libidn/1.9 libssh2/1.2 +robot # curl/7.19.6 (i386-pc-win32) libcurl/7.19.6 OpenSSL/0.9.8k zlib/1.2.3 +robot # curl/7.19.5 (i586-pc-mingw32msvc) libcurl/7.19.5 zlib/1.2.3 +robot # curl/7.19.5 (i486-pc-linux-gnu) libcurl/7.19.5 OpenSSL/0.9.8g zlib/1.2.3.3 libidn/1.15 +robot # curl/7.19.4 (universal-apple-darwin10.0) libcurl/7.19.4 OpenSSL/0.9.8k zlib/1.2.3 +robot # curl/7.19.4 (i686-pc-cygwin) libcurl/7.19.4 OpenSSL/0.9.8k zlib/1.2.3 libidn/1.9 libssh2/1.0 +robot # curl/7.19.2 (i386-pc-win32) libcurl/7.19.2 OpenSSL/0.9.8i zlib/1.2.3 libidn/1.11 libssh2/0.18 +robot # curl/7.19.2 (i386-pc-win32) libcurl/7.19.2 OpenSSL/0.9.8c zlib/1.2.3 +robot # curl/7.19.0 (x86_64-suse-linux-gnu) libcurl/7.19.0 OpenSSL/0.9.8h zlib/1.2.3 libidn/1.10 +robot # curl/7.18.2 (x86_64-pc-linux-gnu) libcurl/7.18.2 OpenSSL/0.9.8g zlib/1.2.3.3 libidn/1.8 libssh2/0.18 +robot # curl/7.18.1 (i686-suse-linux-gnu) libcurl/7.18.1 OpenSSL/0.9.8g zlib/1.2.3 libidn/1.8 +robot # curl/7.18.0 (x86_64-pc-linux-gnu) libcurl/7.18.0 OpenSSL/0.9.8g zlib/1.2.3.3 libidn/1.1 +robot # curl/7.17.1 (x86_64-pc-linux-gnu) libcurl/7.17.1 OpenSSL/0.9.8g zlib/1.2.3 +robot # curl/7.16.4 (i486-pc-linux-gnu) libcurl/7.16.4 OpenSSL/0.9.8e zlib/1.2.3.3 libidn/1.0 +robot # curl/7.16.3 (powerpc-apple-darwin8.0) libcurl/7.16.3 OpenSSL/0.9.7l zlib/1.2.3 +robot # curl/7.16.2 (x86_64-redhat-linux-gnu) libcurl/7.16.2 OpenSSL/0.9.8b zlib/1.2.3 libidn/0.6.8 +robot # curl/7.16.1 (i386-pc-win32) libcurl/7.16.1 OpenSSL/0.9.8h zlib/1.2.3 +robot # curl/7.15.5 (x86_64-redhat-linux-gnu) libcurl/7.15.5 OpenSSL/0.9.8b zlib/1.2.3 libidn/0.6.5 +robot # curl/7.15.4 (i686-pc-linux-gnu) libcurl/7.15.4 OpenSSL/0.9.7e zlib/1.2.3 +robot # curl/7.15.3 (sparc64--netbsd) libcurl/7.15.3 OpenSSL/0.9.7d zlib/1.1.4 libidn/0.6.3 +robot # curl/7.15.1 (x86_64-suse-linux) libcurl/7.15.1 OpenSSL/0.9.8a zlib/1.2.3 libidn/0.6.0 +robot # curl/7.15.1 (i486-pc-linux-gnu) libcurl/7.15.1 OpenSSL/0.9.8a zlib/1.2.3 libidn/0.5.18 +robot # curl/7.15.0 (i386-portbld-freebsd5.4) libcurl/7.15.0 OpenSSL/0.9.7e zlib/1.2.1 +robot # curl/7.14.0 (i386-portbld-freebsd5.4) libcurl/7.14.0 OpenSSL/0.9.7e zlib/1.2.1 +robot # curl/7.13.2 (i386-pc-linux-gnu) libcurl/7.13.2 OpenSSL/0.9.7e zlib/1.2.2 libidn/0.5.13 +robot # cURL: curl/7.13.1 (powerpc-apple-darwin8.0) libcurl/7.13.1 OpenSSL/0.9.7b zlib/1.2.2 +robot # curl/7.13.1 (powerpc-apple-darwin8.0) libcurl/7.13.1 OpenSSL/0.9.7l zlib/1.2.3 +robot # curl/7.12.1 (i686-redhat-linux-gnu) libcurl/7.12.1 OpenSSL/0.9.7a zlib/1.2.1.2 libidn/0.5.6 +robot # curl/7.11.1 (i686-redhat-linux-gnu) libcurl/7.11.1 OpenSSL/0.9.7a ipv6 zlib/1.2.1.2 +robot # curl/7.11.1 (i386-redhat-linux-gnu) libcurl/7.11.1 OpenSSL/0.9.7a ipv6 zlib/1.2.1.2 +robot # curl/7.10.6 (i386-redhat-linux-gnu) libcurl/7.10.6 OpenSSL/0.9.7a ipv6 zlib/1.1.4 +robot # Java/1.6.0_26 +robot # Java/1.6.0_13 +robot # Java/1.6.0_12 +robot # Java/1.6.0_11 +robot # Java/1.6.0_04 +robot # Java/1.6.0_03 +robot # Java/1.6.0_02 +robot # Java/1.6.0-beta +robot # Java/1.5.0_11 +robot # Java/1.5.0_08 +robot # Java/1.5.0_06 +robot # Java/1.5.0_05 +robot # Java/1.5.0_04 +robot # Java/1.5.0_03 +robot # Java/1.5.0_02 +robot # Java/1.5.0_01 +robot # Java/1.5.0 +robot # Java/1.4.2_11 +robot # Java/1.4.2_10 +robot # Java/1.4.2_09 +robot # Java/1.4.2_08 +robot # Java/1.4.2_07 +robot # Java/1.4.2_05 +robot # Java/1.4.2_04 +robot # Java1.4.2_03 +robot # Java/1.4.2_03 +robot # Java/1.4.2_01 +robot # Java/1.4.2 +robot # Java/1.4.1_04 +robot # Java/1.4.1_03 +robot # Java/1.4.1_02 +robot # Java/1.4.1_01a +robot # Java/1.4.1_01 +robot # Java/1.4.1-p3 +robot # Java/1.4.1 +robot # Java1.4.0_03 +robot # Java1.4.0_02 +robot # Java1.4.0_01 +robot # Java1.4.0 +robot # Java1.3.1_06 +robot # Java1.3.1_04 +robot # Java1.3.1 +robot # Java1.3.0 +robot # Java1.2.2-JDeveloper +robot # Java1.2.2 +robot # Java1.2.1 +robot # libwww-perl/5.821 +robot # libwww-perl/5.820 +robot # libwww-perl/5.816 +robot # libwww-perl/5.814 +robot # libwww-perl/5.808 +robot # libwww-perl/5.805 +robot # libwww-perl/5.803 +robot # libwww-perl/5.800 (+http://passoire.afraid.org/mylittlewebsurvey/index.html) +robot # libwww-perl/5.800 +robot # libwww-perl/5.76 +robot # libwww-perl/5.75 +robot # libwww-perl/5.69 +robot # libwww-perl/5.65 +robot # libwww-perl/5.65 +robot # libwww-perl/5.64 +robot # libwww-perl/5.63 +robot # libwww-perl/5.53 +robot # libwww-perl/5.50 +robot # libwww-perl/5.48 +robot # libwww-perl/5.36 +robot # Microsoft URL Control - 6.01.9782 +robot # Peach/1.01 (Ubuntu 8.04 LTS; U; en) +robot # PHP/5.2.9 +robot # PHP/5.2.8 +robot # PHP/5.2.14 +robot # PHP/5.2.11 +robot # PHP/5.2.10 +robot # pxyscand/2.1 +robot # Python-urllib/3.1 +robot # Python-urllib/3.0 +robot # Python-urllib/2.7 +robot # Python-urllib/2.6 +robot # Python-urllib/2.5 +robot # Python-urllib/2.4 +robot # Python-urllib/2.1 +robot # Python-urllib/2.0a1 +robot # Python-urllib/1.17 +robot # Python-urllib/1.16 +robot # Python-urllib/1.15 +robot # Mozilla/5.0 (compatible; AbiLogicBot/1.0; +http://www.abilogic.com/bot.html) +robot # Mozilla/5.0 (compatible; AbiLogicBot/1.0; +http://www.abilogic.com) +robot # Link Valet Online 1.1 +robot # Link Validity Check From: http://www.w3dir.com/cgi-bin (Using: Hot Links SQL by Mrcgiguy.com) +robot # LinkExaminer/1.01 (Windows) +robot # Mozilla/5.0 (compatible; LinksManager.com_bot http://linksmanager.com/linkchecker.html) +robot # Mozilla/5.0 (compatible; LinksManager.com_bot +http://linksmanager.com/linkchecker.html) +robot # Mojoo Robot (http://www.mojoo.com/) +robot # Notifixious/LinkChecker (http://notifixio.us) +robot # online link validator (http://www.dead-links.com/) +robot # Ploetz + Zeller (http://www.ploetz-zeller.de) Link Validator v1.0 ([email protected]) for ARIS Business Architect +robot # InfoWizards Reciprocal Link System PRO - (http://www.infowizards.com) +robot # REL Link Checker Lite 1.0 +robot # SiteBar/3.3.8 (Bookmark Server; http://sitebar.org/) +robot # SiteBar/3.3.5 (Bookmark Server; http://sitebar.org/) +robot # Vivante Link Checker (http://www.vivante.com) +robot # W3C-checklink/4.5 [4.160] libwww-perl/5.823 +robot # W3C-checklink/4.5 [4.154] libwww-perl/5.823 +robot # W3C-checklink/4.3 [4.42] libwww-perl/5.820 +robot # W3C-checklink/4.3 [4.42] libwww-perl/5.808 +robot # W3C-checklink/4.3 [4.42] libwww-perl/5.805 +robot # W3C-checklink/4.2.1 [4.21] libwww-perl/5.803 +robot # W3C-checklink/4.2 [4.20] libwww-perl/5.803 +robot # W3C-checklink/3.6.2.3 libwww-perl/5.64 +robot # W3C-checklink/2.90 libwww-perl/5.64 +robot # Xenu Link Sleuth 1.2i +robot # Xenu Link Sleuth 1.2h +robot # Xenu Link Sleuth 1.2g +robot # Xenu Link Sleuth 1.2f +robot # Xenu Link Sleuth 1.2e +robot # Xenu Link Sleuth 1.2d +robot # Xenu Link Sleuth 1.2c +robot # Xenu Link Sleuth 1.2b +robot # Xenu Link Sleuth/1.3.7 +robot # Offline Explorer/2.5 +robot # Offline Explorer/1.9 +robot # Offline Explorer/1.4 +robot # SuperBot/4.4.0.60 (Windows XP) +robot # Web Downloader/6.9 +robot # Web Downloader/6.8 +robot # Web Downloader/5.7 +robot # Web Downloader/4.5 +robot # WebCopier v4.6 +robot # WebCopier v4.5.2 +robot # WebCopier v3.3 +robot # WebCopier v3.2a +robot # WebCopier v3.2a +robot # WebCopier v3.2 +robot # WebCopier v3.0 +robot # WebCopier v2.8 +robot # WebZIP/3.5 (http://www.spidersoft.com) +robot # Wget/1.9+cvs-stable (Red Hat modified) +robot # Wget/1.9.1 +robot # Wget/1.9 +robot # Wget/1.8.2 +robot # Wget/1.8.1 +robot # Wget/1.7 +robot # Wget/1.6 +robot # Wget/1.5.3 +robot # Wget/1.15 (darwin13.0.2) +robot # Wget/1.11.4 (Red Hat modified) +robot # Wget/1.11.1 (Red Hat modified) +robot # Wget/1.10.2 (Red Hat modified) +robot # Wget/1.10.2 +robot # Wget/1.10.1-beta1 +robot # Wget/1.10.1 (Red Hat modified) +robot # Wget/1.10.1 +robot # wget +robot # !Susie (http://www.sync2it.com/susie) +robot # amaya/11.3.1 libwww/5.4.1 +robot # amaya/11.2 libwww/5.4.0 +robot # amaya/11.1 libwww/5.4.0 +robot # amaya/10.1 libwww/5.4.0 +robot # amaya/10 libwww/5.4.0 +robot # amaya/9.55 libwww/5.4.0 +robot # amaya/9.54 libwww/5.4.0 +robot # amaya/9.52 libwww/5.4.0 +robot # amaya/9.51 libwww/5.4.0 +robot # amaya/8.8.5 libwww/5.4.0 +robot # amaya/11.2 amaya/5.4.0 +robot # amaya/11.1 amaya/5.4.0 +robot # DomainsDB.net MetaCrawler v.0.9.7c (http://domainsdb.net/) +robot # GSiteCrawler/v1.20 rev. 273 (http://gsitecrawler.com/) +robot # GSiteCrawler/v1.12 rev. 260 (http://gsitecrawler.com/) +robot # GSiteCrawler/v1.06 rev. 251 (http://gsitecrawler.com/) +robot # lftp/4.3.8 +robot # lftp/4.3.5 +robot # MetaURI API/2.0 metauri.com +robot # Nitro PDF Download +robot # Snoopy v1.2 +robot # URD-MAGPIE/0.73 (Cached) +robot # Mozilla/4.0 (compatible; WebCapture 3.0; Windows)
[... 3101 lines stripped ...]
