diff --git a/Parser/Client/Browser.php b/Parser/Client/Browser.php index 1a331f3e62..67de82d853 100644 --- a/Parser/Client/Browser.php +++ b/Parser/Client/Browser.php @@ -81,6 +81,7 @@ class Browser extends AbstractClientParser 'AB' => 'ABrowse', 'BW' => 'AdBlock Browser', 'A7' => 'Adult Browser', + 'A9' => 'Airfind Secure Browser', 'AF' => 'ANT Fresco', 'AG' => 'ANTGalio', 'AL' => 'Aloha Browser', @@ -101,6 +102,7 @@ class Browser extends AbstractClientParser 'AV' => 'Amiga Voyager', 'AW' => 'Amiga Aweb', 'PN' => 'APN Browser', + 'RA' => 'Arc', 'AI' => 'Arvin', 'AK' => 'Ask.com', 'AU' => 'Asus Browser', @@ -113,6 +115,7 @@ class Browser extends AbstractClientParser 'A1' => 'AwoX', 'BA' => 'Beaker Browser', 'BM' => 'Beamrise', + 'F7' => 'BF Browser', 'BB' => 'BlackBerry Browser', 'H1' => 'BrowseHere', 'B8' => 'Browser Hup Pro', @@ -135,6 +138,7 @@ class Browser extends AbstractClientParser 'BN' => 'Borealis Navigator', 'BR' => 'Brave', 'BK' => 'BriskBard', + 'K2' => 'BroKeep Browser', 'B3' => 'Browspeed Browser', 'BX' => 'BrowseX', 'BZ' => 'Browzar', @@ -142,7 +146,7 @@ class Browser extends AbstractClientParser '4B' => 'BrowsBit', 'BY' => 'Biyubi', 'BF' => 'Byffox', - 'B4' => 'BF Browser', + 'B4' => 'BXE Browser', 'CA' => 'Camino', '0C' => 'Cave Browser', 'CL' => 'CCleaner', @@ -194,6 +198,7 @@ class Browser extends AbstractClientParser 'DT' => 'Delta Browser', 'D0' => 'Desi Browser', 'DS' => 'DeskBrowse', + 'D2' => 'DoCoMo', 'DF' => 'Dolphin', 'DZ' => 'Dolphin Zero', 'DO' => 'Dorado', @@ -216,6 +221,7 @@ class Browser extends AbstractClientParser 'EP' => 'GNOME Web', 'G1' => 'G Browser', 'ES' => 'Espial TV Browser', + 'FG' => 'fGet', 'FA' => 'Falkon', 'FX' => 'Faux Browser', 'F4' => 'Fiery Browser', @@ -249,6 +255,7 @@ class Browser extends AbstractClientParser 'GH' => 'Ghostery Privacy Browser', 'GI' => 'GinxDroid Browser', 'GB' => 'Glass Browser', + 'GD' => 'Godzilla Browser', 'GE' => 'Google Earth', 'GP' => 'Google Earth Pro', 'GO' => 'GOG Galaxy', @@ -280,6 +287,7 @@ class Browser extends AbstractClientParser 'IV' => 'Isivioo', 'I8' => 'IVVI Browser', 'IW' => 'Iceweasel', + 'N3' => 'Incognito Browser', 'IN' => 'Inspect Browser', 'I9' => 'Insta Browser', 'IE' => 'Internet Explorer', @@ -371,6 +379,8 @@ class Browser extends AbstractClientParser 'WR' => 'NextWord Browser', 'NT' => 'NTENT Browser', 'NU' => 'Nuanti Meta', + 'NI' => 'Nuviu', + 'O9' => 'Ocean Browser', 'OC' => 'Oculus Browser', 'O6' => 'Odd Browser', 'O1' => 'Opera Mini iOS', @@ -437,6 +447,7 @@ class Browser extends AbstractClientParser 'RW' => 'Reqwireless WebViewer', 'PS' => 'Microsoft Edge', 'QA' => 'Qazweb', + 'Q3' => 'Qmamu', 'Q2' => 'QQ Browser Lite', 'Q1' => 'QQ Browser Mini', 'QQ' => 'QQ Browser', @@ -481,6 +492,7 @@ class Browser extends AbstractClientParser 'S7' => 'SP Browser', '9S' => 'Sony Small Browser', '8S' => 'Secure Private Browser', + 'X2' => 'SecureX', 'T1' => 'Stampy Browser', '7S' => '7Star', 'SQ' => 'Smart Browser', @@ -511,6 +523,7 @@ class Browser extends AbstractClientParser 'TR' => 'T-Browser', 'TO' => 't-online.de Browser', 'TA' => 'Tao Browser', + 'TH' => 'Thor', '1T' => 'Tor Browser', 'TF' => 'TenFourFox', 'TB' => 'Tenta Browser', @@ -620,7 +633,8 @@ class Browser extends AbstractClientParser 'XB', 'W1', 'HT', 'B8', 'F5', 'B9', 'WA', 'T0', 'HC', 'O6', 'P7', 'LJ', 'LC', 'O7', 'N2', 'A8', 'P8', 'RB', '1W', 'EV', 'I9', 'V4', 'H4', '1T', 'M5', '0S', '0C', - 'ZR', 'D6', 'F6', 'RC', 'WD', 'P3', 'FT', + 'ZR', 'D6', 'F6', 'RC', 'WD', 'P3', 'FT', 'A9', 'X2', + 'N3', 'GD', 'O9', 'Q3', 'F7', 'K2', ], 'Firefox' => [ 'AX', 'BI', 'BF', 'BH', 'BN', 'C0', 'CU', 'EI', 'F1', @@ -660,7 +674,8 @@ class Browser extends AbstractClientParser 'DM', '1M', 'A7', 'XN', 'XT', 'XB', 'W1', 'HT', 'B7', 'B9', 'T0', 'I8', 'O6', 'P7', 'O8', '4B', 'A8', 'P8', '1W', 'EV', 'Z0', 'I9', 'V4', 'H4', 'M5', '0S', '0C', - 'ZR', 'D6', 'F6', 'P3', 'FT', + 'ZR', 'D6', 'F6', 'P3', 'FT', 'A9', 'X2', 'NI', 'FG', + 'TH', 'N3', 'GD', 'O9', 'Q3', 'F7', 'K2', ]; /** diff --git a/Parser/OperatingSystem.php b/Parser/OperatingSystem.php index 3123c131bb..f7f6d85b42 100644 --- a/Parser/OperatingSystem.php +++ b/Parser/OperatingSystem.php @@ -54,6 +54,7 @@ class OperatingSystem extends AbstractParser 'QNX' => 'BlackBerry Tablet OS', 'BOS' => 'Bliss OS', 'BMP' => 'Brew', + 'BSN' => 'BrightSignOS', 'CAI' => 'Caixa Mágica', 'CES' => 'CentOS', 'CST' => 'CentOS Stream', @@ -208,7 +209,7 @@ class OperatingSystem extends AbstractParser 'ORD', 'TOS', 'RSO', 'DEE', 'FRE', 'MAG', 'FEN', 'CAI', 'PCL', 'HAS', 'LOS', 'DVK', 'ROK', 'OWR', 'OTV', 'KTV', 'PUR', 'PLA', 'FUC', 'PAR', 'FOR', 'MON', 'KAN', 'ZEN', 'LND', 'LNS', 'CHN', 'AMZ', 'TEN', 'CST', - 'NOV', 'ROU', 'ZOR', 'RED', 'KAL', 'ORA', 'VID', 'TIV', + 'NOV', 'ROU', 'ZOR', 'RED', 'KAL', 'ORA', 'VID', 'TIV', 'BSN', ], 'Mac' => ['MAC'], 'Mobile Gaming Console' => ['PSP', 'NDS', 'XBX'], diff --git a/Tests/Parser/Client/fixtures/browser.yml b/Tests/Parser/Client/fixtures/browser.yml index a6b271c4e1..f161fcea47 100644 --- a/Tests/Parser/Client/fixtures/browser.yml +++ b/Tests/Parser/Client/fixtures/browser.yml @@ -6321,7 +6321,7 @@ user_agent: Mozilla/5.0 (Linux; Android 10; M2007J20CG Build/QKQ1.200512.002; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/91.0.4472.120 Mobile Safari/537.36 client: type: browser - name: BF Browser + name: BXE Browser version: "" engine: Blink engine_version: 91.0.4472.120 @@ -7937,3 +7937,235 @@ family: Chrome headers: Sec-CH-UA: '" Not A;Brand";v="99.0.0.0", "Chromium";v="2024.01"' +- + user_agent: ArcMobile2/1 CFNetwork/1494.0.5 Darwin/23.4.0 + client: + type: browser + name: Arc + version: "" + engine: WebKit + engine_version: "" + family: +- + user_agent: Lilo/66 CFNetwork/978.0.7 Darwin/18.7.0 + client: + type: browser + name: Lilo + version: "" + engine: WebKit + engine_version: "" + family: Chrome +- + user_agent: com.airfind.browser/2.0.5 (Linux; U; Android 8.1.0; es-us; AX751+ Build/O11019; Bmobile) 480X854 Bmobile AX751+ AppleWebKit/537.36 (KHTML, like Gecko) + client: + type: browser + name: Airfind Secure Browser + version: 2.0.5 + engine: Blink + engine_version: "" + family: Chrome + headers: + http-x-requested-with: com.airfind.browser +- + user_agent: 'Dalvik/2.1.0 (Linux; U; Android 10; ZTE 8010 Build/QP1A.190711.020) AppleWebKit [HB/29]' + client: + type: browser + name: SecureX + version: "" + engine: WebKit + engine_version: "" + family: Android Browser + headers: + http-x-requested-with: com.securex.browser +- + user_agent: DoCoMo/2.0 SH901iC(c100;TB;W24H12) + client: + type: browser + name: DoCoMo + version: "2.0" + engine: "" + engine_version: "" + family: +- + user_agent: Mozilla/5.0 (Linux; Android 7.0; LG-M250 Build/NRD90U; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/62.0.3202.84 Mobile Safari/537.36 + client: + type: browser + name: Opera + version: "" + engine: Blink + engine_version: 62.0.3202.84 + family: Opera + headers: + http-x-requested-with: com.opera.browser.beta +- + user_agent: Mozilla/5.0 (Linux; Android 8.1.0; SM-G610M Build/M1AJQ; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/98.0.4758.101 Mobile Safari/537.36 + client: + type: browser + name: Yandex Browser + version: "" + engine: Blink + engine_version: 98.0.4758.101 + family: Chrome + headers: + http-x-requested-with: com.yandex.browser.beta +- + user_agent: Mozilla/5.0 (Linux; Android 11; SM-A715F Build/RP1A.200720.012; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/100.0.4896.88 Mobile Safari/537.36 + client: + type: browser + name: Yandex Browser + version: "" + engine: Blink + engine_version: 100.0.4896.88 + family: Chrome + headers: + http-x-requested-with: com.yandex.browser.alpha +- + user_agent: Mozilla/5.0 (Linux; Android 11; SM-S506DL Build/RP1A.200720.012; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/101.0.4951.61 Mobile Safari/537.36 + client: + type: browser + name: Samsung Browser + version: "" + engine: Blink + engine_version: 101.0.4951.61 + family: Chrome + headers: + http-x-requested-with: com.sec.android.app.sbrowser.beta +- + user_agent: Mozilla/5.0 (Linux; Android 13; f87d8fd4aeea7894 Build/EVW5.118155.252;) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/101.0.0001.320060 Mobile Safari/537.36 + client: + type: browser + name: Via + version: "" + engine: Blink + engine_version: 101.0.0001.320060 + family: Chrome + headers: + http-x-requested-with: mark.via.gq +- + user_agent: Mozilla/5.0 (Linux; Android 11; Pixel 5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/90.0.4430.91 Mobile Safari/537.36 + client: + type: browser + name: Via + version: "" + engine: Blink + engine_version: 90.0.4430.91 + family: Chrome + headers: + http-x-requested-with: mark.via.pm +- + user_agent: Mozilla/5.0 (iPhone; CPU iOS 15.6.1 like Mac OS X) AppleWebKit/613.3.9.0.16 (KHTML, like Gecko) Mobile/15E148 Nuviu/1.0.5 + client: + type: browser + name: Nuviu + version: 1.0.5 + engine: WebKit + engine_version: 613.3.9.0.16 + family: +- + user_agent: fGet/2 CFNetwork/1485 Darwin/23.1.0 + client: + type: browser + name: fGet + version: + engine: + engine_version: + family: +- + user_agent: Mozilla/5.0 (Linux; Android 10) + client: + type: browser + name: Thor + version: "" + engine: WebKit + engine_version: "" + family: Android Browser + headers: + http-x-requested-with: threads.thor +- + user_agent: Mozilla/5.0 (Linux; Android 10) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.5735.130 Mobile Safari/537.36 + client: + type: browser + name: Incognito Browser + version: "" + engine: Blink + engine_version: 114.0.5735.130 + family: Chrome + headers: + http-x-requested-with: com.androidbull.incognito.browser +- + user_agent: Mozilla/5.0 (Linux; Android 10) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/92.0.4515.159 Mobile Safari/537.36 + client: + type: browser + name: Godzilla Browser + version: "" + engine: Blink + engine_version: 92.0.4515.159 + family: Chrome + headers: + http-x-requested-with: com.mosoft.godzilla +- + user_agent: Mozilla/5.0 (Linux; Android 10) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/112.0.5615.101 Mobile DuckDuckGo/105 Safari/537.36 + client: + type: browser + name: Ocean Browser + version: "" + engine: Blink + engine_version: 112.0.5615.101 + family: Chrome + headers: + http-x-requested-with: com.oceanbrowser.mobile.android +- + user_agent: Mozilla/5.0 (Linux; Android 12) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/103.0.5060.129 Mobile DuckDuckGo/1 Safari/537.36 + client: + type: browser + name: Qmamu + version: "" + engine: Blink + engine_version: 103.0.5060.129 + family: Chrome + headers: + http-x-requested-with: com.qmamu.browser +- + user_agent: Mozilla/5.0 (Linux; Android 12) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/111.0.5563.58 Mobile DuckDuckGo/17 Safari/537.36 + client: + type: browser + name: BF Browser + version: "" + engine: Blink + engine_version: 111.0.5563.58 + family: Chrome + headers: + http-x-requested-with: com.techlastudio.bfbrowser +- + user_agent: Mozilla/5.0 (Linux; Android 12) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/111.0.5563.58 Mobile DuckDuckGo/17 Safari/537.36 + client: + type: browser + name: BF Browser + version: "" + engine: Blink + engine_version: 111.0.5563.58 + family: Chrome + headers: + http-x-requested-with: com.befaster.bfbrowser +- + user_agent: Mozilla/5.0 (Linux; Android 7.1.2) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/110.0.5481.153 Mobile DuckDuckGo/8 Safari/537.36 + client: + type: browser + name: BroKeep Browser + version: "" + engine: Blink + engine_version: 110.0.5481.153 + family: Chrome + headers: + http-x-requested-with: app.nextinnovations.brokeep +- + user_agent: Mozilla/5.0 (Linux; Android 8.0.0; SM-G935F Build/R16NW; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/98.0.4758.101 Mobile Safari/537.36 + client: + type: browser + name: Lilo + version: "" + engine: Blink + engine_version: 98.0.4758.101 + family: Chrome + headers: + http-x-requested-with: org.lilo.app diff --git a/Tests/Parser/Client/fixtures/library.yml b/Tests/Parser/Client/fixtures/library.yml index 95e86049c5..9d5201d4b4 100644 --- a/Tests/Parser/Client/fixtures/library.yml +++ b/Tests/Parser/Client/fixtures/library.yml @@ -581,3 +581,33 @@ type: library name: Mandrill PHP version: 1.0.55 +- + user_agent: babashka.http-client/0.4.14 + client: + type: library + name: Babashka HTTP Client + version: 0.4.14 +- + user_agent: http.rb/4.0.0 + client: + type: library + name: http.rb + version: 4.0.0 +- + user_agent: node-superagent/3.8.2 + client: + type: library + name: superagent + version: 3.8.2 +- + user_agent: CakePHP + client: + type: library + name: CakePHP + version: "" +- + user_agent: request.js + client: + type: library + name: request + version: "" diff --git a/Tests/Parser/Client/fixtures/mobile_app.yml b/Tests/Parser/Client/fixtures/mobile_app.yml index 11e1479744..7246cf243b 100644 --- a/Tests/Parser/Client/fixtures/mobile_app.yml +++ b/Tests/Parser/Client/fixtures/mobile_app.yml @@ -1577,3 +1577,57 @@ type: mobile app name: IPTV version: 6.1.11 +- + user_agent: Klarna/1 CFNetwork/1404.0.5 Darwin/22.3.0 + client: + type: mobile app + name: Klarna + version: +- + user_agent: MetaMask/1230 CFNetwork/1492.0.1 Darwin/23.3.0 + client: + type: mobile app + name: MetaMask + version: +- + user_agent: com.zhiliaoapp.musically/2023203040 (Linux; U; Android 13; en_US; SM-G990U; Build/TP1A.220624.014; Cronet/TTNetVersion:e8c372ba 2023-11-06 QuicVersion:0d108adb 2023-07-19) + client: + type: mobile app + name: TikTok + version: +- + user_agent: Mozilla/5.0 (iPad; CPU OS 16_5_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) FSCDCSafe/20.1.291705 Mobile/15E148 Version/16.5.1 Safari/605.1.15 + client: + type: mobile app + name: F-Secure SAFE + version: 20.1.291705 +- + user_agent: +Simple Browser + client: + type: mobile app + name: +Simple + version: +- + user_agent: Mozilla/5.0 (Linux; Android 11; JLN-LX3 Build/HUAWEIJLN-LX3; ) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/88.0.4324.93 Mobile Safari/537.36 Sapphire/23.3.401205602 + client: + type: mobile app + name: Microsoft Bing Search + version: 23.3.401205602 +- + user_agent: Bitwarden_Mobile/2023.1.0 (Android 13; SDK 33; Model SM-F926B) + client: + type: mobile app + name: Bitwarden + version: 2023.1.0 +- + user_agent: MXPlayer/1.73.3/2001002183; Android/33 + client: + type: mobile app + name: MX Player + version: 1.73.3 +- + user_agent: Mozilla/5.0 (Macintosh; PPC Mac OS X; en) AppleWebKit/537.75.14 (KHTML, like Gecko) HistoryHound/1.9 + client: + type: mobile app + name: HistoryHound + version: "1.9" diff --git a/Tests/Parser/fixtures/oss.yml b/Tests/Parser/fixtures/oss.yml index 63a65f3cbd..3432923c4f 100644 --- a/Tests/Parser/fixtures/oss.yml +++ b/Tests/Parser/fixtures/oss.yml @@ -4142,3 +4142,43 @@ version: "14.4" platform: family: Mac +- + user_agent: Aloha/1 Mana/0.0.1 Darwin/21.0.0 + os: + name: iOS + short_name: IOS + version: "15.0" + platform: + family: iOS +- + user_agent: BrightSign/8.3.23 (XT1144) Mozilla/5.0 (X11; Linux aarch64) AppleWebKit/537.36 (KHTML, like Gecko) QtWebEngine/5.12.3 Chrome/69.0.3497.128 Safari/537.36 + os: + name: BrightSignOS + short_name: BSN + version: 8.3.23 + platform: ARM + family: GNU/Linux +- + user_agent: BrightSign/35D6AA000662/6.2.80.1 (HD1023) Mozilla/5.0 (Unknown; Linux arm) AppleWebKit/537.36 (KHTML, like Gecko) QtWebEngine/5.6.0 Chrome/45.0.2454.101 Safari/537.36 + os: + name: BrightSignOS + short_name: BSN + version: 6.2.80.1 + platform: ARM + family: GNU/Linux +- + user_agent: Callpod Keeper for Android 1.0 (10.5.6/270) Dalvik/2.1.0 (Linux; U; Android 13; SM-S908B Build/TP1A.220624.014) + os: + name: Android + short_name: AND + version: "13" + platform: "" + family: Android +- + user_agent: Mozilla/5.0 (iPad; CPU iPad OS 17_1_2 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148 + os: + name: iPadOS + short_name: IPA + version: 17.1.2 + platform: + family: iOS diff --git a/Tests/fixtures/bots.yml b/Tests/fixtures/bots.yml index 60004d01e4..bea68fe905 100644 --- a/Tests/fixtures/bots.yml +++ b/Tests/fixtures/bots.yml @@ -916,8 +916,8 @@ category: Social Media Agent url: https://www.facebook.com/externalhit_uatext.php producer: - name: Facebook - url: http://www.facebook.com + name: Meta Platforms, Inc. + url: https://www.meta.com/ - user_agent: facebookexternalua bot: @@ -925,8 +925,8 @@ category: Social Media Agent url: https://www.facebook.com/externalhit_uatext.php producer: - name: Facebook - url: http://www.facebook.com + name: Meta Platforms, Inc. + url: https://www.meta.com/ - user_agent: facebookplatform/1.0 (+http://developers.facebook.com) bot: @@ -934,8 +934,8 @@ category: Social Media Agent url: https://www.facebook.com/externalhit_uatext.php producer: - name: Facebook - url: http://www.facebook.com + name: Meta Platforms, Inc. + url: https://www.meta.com/ - user_agent: Feed Wrangler/1.0 (3 subscribers; feed-id=248559; http://feedwrangler.net; Allow like Gecko) bot: @@ -1717,7 +1717,8 @@ user_agent: 'ltx71 - (http://ltx71.com/)' bot: name: LTX71 - url: http://ltx71.com/ + category: 'Security Checker' + url: https://ltx71.com/ producer: name: "" url: "" @@ -4544,8 +4545,8 @@ category: Social Media Agent url: https://www.facebook.com/externalhit_uatext.php producer: - name: Facebook - url: http://www.facebook.com + name: Meta Platforms, Inc. + url: https://www.meta.com/ - user_agent: Mozilla/5.0 (compatible; Linespider/1.1; https://lin.ee/4dwXkTH) bot: @@ -6348,3 +6349,162 @@ name: GitHubCopilotChat category: Crawler url: https://github.com/aaamoon/copilot-gpt4-service +- + user_agent: anthropic-ai + bot: + name: Anthropic AI + category: Crawler + url: https://www.anthropic.com/ + producer: + name: Anthropic, PBC + url: https://www.anthropic.com/ +- + user_agent: bingadsbot + bot: + name: BingBot + category: Search bot + url: http://search.msn.com/msnbot.htmn + producer: + name: Microsoft Corporation + url: http://www.microsoft.com +- + user_agent: default_useragent + bot: + name: Generic Bot +- + user_agent: Mozilla/5.0 (compatible; NetpeakCheckerBot/3.7; +https://netpeaksoftware.com/checker) + bot: + name: Netpeak Checker + category: Crawler + url: https://netpeaksoftware.com/checker + producer: + name: Netpeak LTD + url: https://netpeaksoftware.com/ +- + user_agent: mozilla/5.0 (compatible; SandobaCrawler/1.0; +https://www.sandoba.com/en/crawler/) + bot: + name: Sandoba//Crawler + category: Crawler + url: https://www.sandoba.com/en/crawler/ + producer: + name: SANDOBA//EBUSINESS SOLUTIONS + url: https://www.sandoba.com/ +- + user_agent: SirdataBot (+https://semantic-api.docs.sirdata.net/contextual-api/contextual-api/introduction) + bot: + name: Sirdata + category: Crawler + url: https://semantic-api.docs.sirdata.net/contextual-api/contextual-api/introduction + producer: + name: Sirdata SAS + url: https://www.sirdata.com/ +- + user_agent: CheckMarkNetwork/1.0 (+http://www.checkmarknetwork.com/spider.html) + bot: + name: CheckMark Network + category: Crawler + url: https://www.checkmarknetwork.com/spider.html/ + producer: + name: Exipert, Inc. + url: https://www.checkmarknetwork.com/ +- + user_agent: Mozilla/5.0 (compatible; FacebookBot/1.0; +https://developers.facebook.com/docs/sharing/webmasters/facebookbot/) + bot: + name: FacebookBot + category: Crawler + url: https://developers.facebook.com/docs/sharing/bot + producer: + name: Meta Platforms, Inc. + url: https://www.meta.com/ +- + user_agent: cohere-ai + bot: + name: Cohere AI + category: Crawler + url: https://cohere.com/ + producer: + name: Cohere, Inc. + url: https://cohere.com/ +- + user_agent: Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; PerplexityBot/1.0; +https://perplexity.ai/perplexitybot) + bot: + name: PerplexityBot + category: Crawler + url: https://docs.perplexity.ai/docs/perplexitybot + producer: + name: Perplexity AI, Inc. + url: https://www.perplexity.ai/ +- + user_agent: Mozilla/5.0 (compatible; TTD-Content; +https://www.thetradedesk.com/general/ttd-content) + bot: + name: The Trade Desk Content + category: Crawler + url: https://www.thetradedesk.com/us/ttd-content + producer: + name: The Trade Desk, Inc. + url: https://www.thetradedesk.com/ +- + user_agent: montastic-monitor http://www.montastic.com + bot: + name: Montastic Monitor + category: Site Monitor + url: https://www.montastic.com/ + producer: + name: Metadot, Corp. + url: https://www.metadot.com/ +- + user_agent: Ruby, Twurly v1.1 (https://twurly.org) + bot: + name: Twurly + category: Crawler + url: https://twurly.org/ +- + user_agent: Mozilla/5.0 (Mixnode) AppleWebKit/537.36 (KHTML, like Gecko) + bot: + name: Mixnode + category: Crawler + url: https://www.mixnode.com/ + producer: + name: Mixnode Technologies, Inc. + url: https://www.mixnode.com/ +- + user_agent: MixnodeCache/1.8(+https://cache.mixnode.com/) + bot: + name: Mixnode + category: Crawler + url: https://www.mixnode.com/ + producer: + name: Mixnode Technologies, Inc. + url: https://www.mixnode.com/ +- + user_agent: Mixnode/3.0.1 (+https://www.mixnode.com/; Speed analysis; no data is stored) + bot: + name: Mixnode + category: Crawler + url: https://www.mixnode.com/ + producer: + name: Mixnode Technologies, Inc. + url: https://www.mixnode.com/ +- + user_agent: CSSCheck/1.2.2 + bot: + name: CSSCheck + category: Validator +- + user_agent: Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.6167.85 Mobile Safari/537.36 (compatible; AdsBot-Google-Mobile; +http://www.google.com/mobile/adsbot.html) + bot: + name: Googlebot + category: Search bot + url: https://developers.google.com/search/docs/crawling-indexing/overview-google-crawlers + producer: + name: Google Inc. + url: https://www.google.com/ +- + user_agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.2924.87 Safari/537.36 Google-PageRenderer Google (+https://developers.google.com/+/web/snippet/) + bot: + name: Googlebot + category: Search bot + url: https://developers.google.com/search/docs/crawling-indexing/overview-google-crawlers + producer: + name: Google Inc. + url: https://www.google.com/ diff --git a/Tests/fixtures/clienthints-app.yml b/Tests/fixtures/clienthints-app.yml index e078ba2306..3b7b96f55c 100644 --- a/Tests/fixtures/clienthints-app.yml +++ b/Tests/fixtures/clienthints-app.yml @@ -2233,3 +2233,231 @@ model: Pixel 5 os_family: Android browser_family: Unknown +- + user_agent: Mozilla/5.0 (Linux; Android 14; SM-A536B Build/UP1A.231005.007; wv) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.6099.231 Mobile Safari/537.36 + headers: + http-x-requested-with: com.fsecure.ms.dnafi + os: + name: Android + version: "14" + platform: "" + client: + type: mobile app + name: DNA Digiturva + version: "" + device: + type: smartphone + brand: Samsung + model: Galaxy A53 5G + os_family: Android + browser_family: Unknown +- + user_agent: Mozilla/5.0 (Linux; Android 10; STK-LX1 Build/HUAWEISTK-LX1; wv) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.6099.144 Mobile Safari/537.36 + headers: + http-x-requested-with: com.fsecure.ms.teliasweden + os: + name: Android + version: "10" + platform: "" + client: + type: mobile app + name: Telia Trygg + version: "" + device: + type: smartphone + brand: Huawei + model: P smart Z + os_family: Android + browser_family: Unknown +- + user_agent: Mozilla/5.0 (Linux; Android 5.0; SM-N9005 Build/LRX21V; wv) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/95.0.4638.74 Mobile Safari/537.36 + headers: + http-x-requested-with: com.fsecure.ms.upc.ch + os: + name: Android + version: "5.0" + platform: "" + client: + type: mobile app + name: UPC Internet Security + version: "" + device: + type: phablet + brand: Samsung + model: Galaxy Note 3 + os_family: Android + browser_family: Unknown +- + user_agent: Mozilla/5.0 (Linux; Android 12; SM-G975F Build/SP1A.210812.016; wv) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/118.0.0.0 Mobile Safari/537.36 + headers: + http-x-requested-with: com.fsecure.ms.sfr + os: + name: Android + version: "12" + platform: "" + client: + type: mobile app + name: SFR Sécurité + version: "" + device: + type: smartphone + brand: Samsung + model: Galaxy S10+ + os_family: Android + browser_family: Unknown +- + user_agent: Mozilla/5.0 (Linux; Android 11; SM-A202F Build/RP1A.200720.012; wv) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.6099.43 Mobile Safari/537.36 + headers: + http-x-requested-with: com.fsecure.ms.sonera + os: + name: Android + version: "11" + platform: "" + client: + type: mobile app + name: Telia Turvapaketti + version: "" + device: + type: smartphone + brand: Samsung + model: Galaxy A20e + os_family: Android + browser_family: Unknown +- + user_agent: Mozilla/5.0 (Linux; Android 10; SM-G965U Build/QP1A.190711.020; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/99.0.4844.88 Mobile Safari/537.36 DaumApps/6.9.27 DaumDevice/mobile + headers: + http-x-requested-with: net.daum.android.daum + os: + name: Android + version: "10" + platform: "" + client: + type: mobile app + name: Daum + version: 6.9.27 + device: + type: smartphone + brand: Samsung + model: Galaxy S9+ + os_family: Android + browser_family: Unknown +- + user_agent: Mozilla/5.0 (Linux; U; Android 4.4; en-us; Nexus 4 Build/JOP24G) AppleWebKit/534.30 (KHTML, like Gecko) Version/4.0 Mobile Safari/534.30 + headers: + http-x-requested-with: com.ezt.vpn + os: + name: Android + version: "4.4" + platform: "" + client: + type: mobile app + name: EZVPN + version: "" + device: + type: smartphone + brand: Google + model: Nexus 4 + os_family: Android + browser_family: Unknown +- + user_agent: Mozilla/5.0 (Linux; Android 10; 5007G Build/QP1A.190711.020; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/88.0.4324.181 Mobile Safari/537.36 + headers: + http-x-requested-with: com.nocardteam.nocardvpn + os: + name: Android + version: "10" + platform: "" + client: + type: mobile app + name: NoCard VPN + version: "" + device: + type: smartphone + brand: Alcatel + model: 1V (2020) + os_family: Android + browser_family: Unknown +- + user_agent: Mozilla/5.0 (Linux; Android 10; 5007G Build/QP1A.190711.020; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/88.0.4324.181 Mobile Safari/537.36 + headers: + http-x-requested-with: com.nocardteam.nocardvpn.lite + os: + name: Android + version: "10" + platform: "" + client: + type: mobile app + name: NoCard VPN Lite + version: "" + device: + type: smartphone + brand: Alcatel + model: 1V (2020) + os_family: Android + browser_family: Unknown +- + user_agent: +Simple Browser + headers: + http-x-requested-with: com.massimple.nacion.parana.es + os: [ ] + client: + type: mobile app + name: +Simple + version: "" + device: + type: "" + brand: "" + model: "" + os_family: Unknown + browser_family: Unknown +- + user_agent: +Simple Browser + headers: + http-x-requested-with: com.massimple.nacion.gcba.es + os: [ ] + client: + type: mobile app + name: +Simple + version: "" + device: + type: "" + brand: "" + model: "" + os_family: Unknown + browser_family: Unknown +- + user_agent: Mozilla/5.0 (Linux; Android 10; SM-G965U Build/QP1A.190711.020; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/113.0.5672.162 Mobile Safari/537.36 mkBrowser/1.5.1 + headers: + http-x-requested-with: com.antivirus.master.cmsecurity + os: + name: Android + version: "10" + platform: "" + client: + type: mobile app + name: CM Security + version: "" + device: + type: smartphone + brand: Samsung + model: Galaxy S9+ + os_family: Android + browser_family: Unknown +- + user_agent: Mozilla/5.0 (Linux; Android 11; JLN-LX3 Build/HUAWEIJLN-LX3; ) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/88.0.4324.93 Mobile Safari/537.36 Sapphire/23.3.401205602 + headers: + http-x-requested-with: com.microsoft.math + os: + name: Android + version: "11" + platform: "" + client: + type: mobile app + name: Microsoft Math Solver + version: "" + device: + type: smartphone + brand: Huawei + model: Nova 9 SE + os_family: Android + browser_family: Unknown diff --git a/Tests/fixtures/feature_phone.yml b/Tests/fixtures/feature_phone.yml index 968921277a..fd5cd7b7f8 100644 --- a/Tests/fixtures/feature_phone.yml +++ b/Tests/fixtures/feature_phone.yml @@ -1346,7 +1346,12 @@ - user_agent: DoCoMo/2.0 MST_v_SH2101V os: [ ] - client: null + client: + type: browser + name: DoCoMo + version: "2.0" + engine: "" + engine_version: "" device: type: feature phone brand: DoCoMo @@ -1356,7 +1361,12 @@ - user_agent: DoCoMo/1.0/D501i os: [ ] - client: null + client: + type: browser + name: DoCoMo + version: "1.0" + engine: "" + engine_version: "" device: type: feature phone brand: DoCoMo @@ -1366,7 +1376,12 @@ - user_agent: DoCoMo/1.0/F501i os: [ ] - client: null + client: + type: browser + name: DoCoMo + version: "1.0" + engine: "" + engine_version: "" device: type: feature phone brand: DoCoMo @@ -1376,7 +1391,12 @@ - user_agent: DoCoMo/1.0/P501i os: [ ] - client: null + client: + type: browser + name: DoCoMo + version: "1.0" + engine: "" + engine_version: "" device: type: feature phone brand: DoCoMo diff --git a/Tests/fixtures/tv-1.yml b/Tests/fixtures/tv-1.yml index a227a42051..b98ea20271 100644 --- a/Tests/fixtures/tv-1.yml +++ b/Tests/fixtures/tv-1.yml @@ -74,8 +74,8 @@ - user_agent: BrightSign/35D6AA000662/6.2.80.1 (HD1023) Mozilla/5.0 (Unknown; Linux arm) AppleWebKit/537.36 (KHTML, like Gecko) QtWebEngine/5.6.0 Chrome/45.0.2454.101 Safari/537.36 os: - name: GNU/Linux - version: "" + name: BrightSignOS + version: 6.2.80.1 platform: ARM client: type: browser @@ -92,8 +92,8 @@ - user_agent: BrightSign/R3G75N003767/7.0.60 (XD233) Mozilla/5.0 (Unknown; Linux arm) AppleWebKit/537.36 (KHTML, like Gecko) QtWebEngine/5.6.0 Chrome/45.0.2454.101 Safari/537.36 os: - name: GNU/Linux - version: "" + name: BrightSignOS + version: 7.0.60 platform: ARM client: type: browser @@ -110,8 +110,8 @@ - user_agent: BrightSign/X1G389001947/6.1.100 (XD1030) Mozilla/5.0 (Unknown; Linux mips) AppleWebKit/537.36 (KHTML, like Gecko) QtWebEngine/5.4.3 Chrome/37.0.2062.103 Safari/537.36 os: - name: GNU/Linux - version: "" + name: BrightSignOS + version: 6.1.100 platform: MIPS client: type: browser @@ -128,8 +128,8 @@ - user_agent: BrightSign/8.2.35.4 (XT1144-PP) Mozilla/5.0 (X11; Linux aarch64) AppleWebKit/537.36 (KHTML, like Gecko) QtWebEngine/5.12.3 Chrome/69.0.3497.128 Safari/537.36 os: - name: GNU/Linux - version: "" + name: BrightSignOS + version: 8.2.35.4 platform: ARM client: type: browser @@ -146,8 +146,8 @@ - user_agent: BrightSign/7.0.60 (XT1143) Mozilla/5.0 (Unknown; Linux arm) AppleWebKit/537.36 (KHTML, like Gecko) QtWebEngine/5.6.0 Chrome/45.0.2454.101 Safari/537.36 os: - name: GNU/Linux - version: "" + name: BrightSignOS + version: 7.0.60 platform: ARM client: type: browser @@ -164,8 +164,8 @@ - user_agent: BrightSign/X3K4A8000366/8.0.33 (4K1142)Mozilla/5.0 (X11; Linux armv7l) AppleWebKit/537.36 (KHTML, like Gecko) QtWebEngine/5.11.2 Chrome/65.0.3325.230 Safari/537.36 os: - name: GNU/Linux - version: "" + name: BrightSignOS + version: 8.0.33 platform: ARM client: type: browser @@ -182,8 +182,8 @@ - user_agent: BrightSign/4.8.108 (XD230) Mozilla/5.0 (compatible; Linux mips) AppleWebKit/537.4 (KHTML, like Gecko) Chromium/18.0.1025.168 Chrome/18.0.1025.168 Safari/537.4 os: - name: GNU/Linux - version: "" + name: BrightSignOS + version: 4.8.108 platform: MIPS client: type: browser @@ -200,8 +200,8 @@ - user_agent: BrightSign/8.0.48 (LS424-W)Mozilla/5.0 (X11; Linux aarch64) AppleWebKit/537.36 (KHTML, like Gecko) QtWebEngine/5.11.2 Chrome/65.0.3325.230 Safari/537.36 os: - name: GNU/Linux - version: "" + name: BrightSignOS + version: 8.0.48 platform: ARM client: type: browser @@ -4112,8 +4112,8 @@ - user_agent: BrightSign/6.0.13 (XD1132) Mozilla/5.0 (compatible; Linux mips) AppleWebKit/537.4 (KHTML, like Gecko) Chromium/18.0.1025.168 Chrome/18.0.1025.168 Safari/537.4 os: - name: GNU/Linux - version: "" + name: BrightSignOS + version: 6.0.13 platform: MIPS client: type: browser diff --git a/Tests/fixtures/tv-3.yml b/Tests/fixtures/tv-3.yml index 12ad24f26a..5f182ae9eb 100644 --- a/Tests/fixtures/tv-3.yml +++ b/Tests/fixtures/tv-3.yml @@ -2061,3 +2061,21 @@ model: VOL001 os_family: Android browser_family: Chrome +- + user_agent: BrightSign/9.0.109 (LS445) Mozilla/5.0 (X11; Linux aarch64) AppleWebKit/537.36 (KHTML, like Gecko) QtWebEngine/5.15.2 Chrome/87.0.4280.144 Safari/537.36 + os: + name: BrightSignOS + version: 9.0.109 + platform: ARM + client: + type: browser + name: QtWebEngine + version: 5.15.2 + engine: WebKit + engine_version: "537.36" + device: + type: tv + brand: BrightSign + model: LS445 + os_family: GNU/Linux + browser_family: Chrome diff --git a/regexes/bots.yml b/regexes/bots.yml index fca41cfe4f..f101df1589 100644 --- a/regexes/bots.yml +++ b/regexes/bots.yml @@ -220,7 +220,7 @@ name: 'Better Uptime' url: 'https://betteruptime.com/' -- regex: 'MSNBot|msrbot|bingbot|BingPreview|msnbot-(UDiscovery|NewsBlogs)|adidxbot' +- regex: 'MSNBot|msrbot|bingbot|bingadsbot|BingPreview|msnbot-(UDiscovery|NewsBlogs)|adidxbot' name: 'BingBot' category: 'Search bot' url: 'http://search.msn.com/msnbot.htmn' @@ -543,8 +543,16 @@ category: 'Social Media Agent' url: 'https://www.facebook.com/externalhit_uatext.php' producer: - name: 'Facebook' - url: 'http://www.facebook.com' + name: 'Meta Platforms, Inc.' + url: 'https://www.meta.com/' + +- regex: 'FacebookBot/[\d.]+' + name: 'FacebookBot' + category: 'Crawler' + url: 'https://developers.facebook.com/docs/sharing/bot' + producer: + name: 'Meta Platforms, Inc.' + url: 'https://www.meta.com/' - regex: 'Feedbin' name: 'Feedbin' @@ -744,7 +752,7 @@ name: 'Visual Meta' url: 'https://www.shopalike.cz/' -- regex: 'Adwords-(?:DisplayAds|Express|Instant)|Google Web Preview|Google[ -]Publisher[ -]Plugin|Google-(?:Ads-Conversions|Ads-Qualify|Adwords|AMPHTML|Assess|Extended|HotelAdsVerifier|InspectionTool|Read-Aloud|Safety|Shopping-Quality|Site-Verification|speakr|Stale-Content-Probe|Test|Youtube-Links)|(?:AdsBot|APIs|DuplexWeb|Feedfetcher|Mediapartners)-Google|Google(?:AdSenseInfeed|AssociationService|bot|Other|Prober|Producer)|Google.*/\+/web/snippet' +- regex: 'Adwords-(?:DisplayAds|Express|Instant)|Google Web Preview|Google[ -]Publisher[ -]Plugin|Google-(?:Ads-Conversions|Ads-Qualify|Adwords|AMPHTML|Assess|Extended|HotelAdsVerifier|InspectionTool|PageRenderer|Read-Aloud|Safety|Shopping-Quality|Site-Verification|speakr|Stale-Content-Probe|Test|Youtube-Links)|(?:AdsBot|APIs|DuplexWeb|Feedfetcher|Mediapartners)-Google(?:-Mobile)?|Google(?:AdSenseInfeed|AssociationService|bot|Other|Prober|Producer)|Google.*/\+/web/snippet' name: 'Googlebot' category: 'Search bot' url: 'https://developers.google.com/search/docs/crawling-indexing/overview-google-crawlers' @@ -878,7 +886,8 @@ - regex: 'ltx71' name: 'LTX71' - url: 'http://ltx71.com/' + category: 'Security Checker' + url: 'https://ltx71.com/' producer: name: '' url: '' @@ -3702,9 +3711,98 @@ category: 'Crawler' url: 'https://github.com/aaamoon/copilot-gpt4-service' +- regex: 'anthropic-ai' + name: 'Anthropic AI' + category: 'Crawler' + url: 'https://www.anthropic.com/' + producer: + name: 'Anthropic, PBC' + url: 'https://www.anthropic.com/' + +- regex: 'NetpeakCheckerBot/[\d.]+' + name: 'Netpeak Checker' + category: 'Crawler' + url: 'https://netpeaksoftware.com/checker' + producer: + name: 'Netpeak LTD' + url: 'https://netpeaksoftware.com/' + +- regex: 'SandobaCrawler/[\d.]+' + name: 'Sandoba//Crawler' + category: 'Crawler' + url: 'https://www.sandoba.com/en/crawler/' + producer: + name: 'SANDOBA//EBUSINESS SOLUTIONS' + url: 'https://www.sandoba.com/' + +- regex: 'SirdataBot' + name: 'Sirdata' + category: 'Crawler' + url: 'https://semantic-api.docs.sirdata.net/contextual-api/contextual-api/introduction' + producer: + name: 'Sirdata SAS' + url: 'https://www.sirdata.com/' + +- regex: 'CheckMarkNetwork/[\d.]+' + name: 'CheckMark Network' + category: 'Crawler' + url: 'https://www.checkmarknetwork.com/spider.html/' + producer: + name: 'Exipert, Inc.' + url: 'https://www.checkmarknetwork.com/' + +- regex: 'cohere-ai' + name: 'Cohere AI' + category: 'Crawler' + url: 'https://cohere.com/' + producer: + name: 'Cohere, Inc.' + url: 'https://cohere.com/' + +- regex: 'PerplexityBot/[\d.]+' + name: 'PerplexityBot' + category: 'Crawler' + url: 'https://docs.perplexity.ai/docs/perplexitybot' + producer: + name: 'Perplexity AI, Inc.' + url: 'https://www.perplexity.ai/' + +- regex: 'TTD-Content' + name: 'The Trade Desk Content' + category: 'Crawler' + url: 'https://www.thetradedesk.com/us/ttd-content' + producer: + name: 'The Trade Desk, Inc.' + url: 'https://www.thetradedesk.com/' + +- regex: 'montastic-monitor' + name: 'Montastic Monitor' + category: 'Site Monitor' + url: 'https://www.montastic.com/' + producer: + name: 'Metadot, Corp.' + url: 'https://www.metadot.com/' + +- regex: 'Ruby, Twurly v[\d.]+' + name: 'Twurly' + category: 'Crawler' + url: 'https://twurly.org/' + +- regex: 'Mixnode(?:(?:Cache)?/[\d.]+)?' + name: 'Mixnode' + category: 'Crawler' + url: 'https://www.mixnode.com/' + producer: + name: 'Mixnode Technologies, Inc.' + url: 'https://www.mixnode.com/' + +- regex: 'CSSCheck/[\d.]+' + name: 'CSSCheck' + category: 'Validator' + # Generic detections -- regex: '(A6-Indexer|nuhk|TsolCrawler|Yammybot|Openbot|Gulper Web Bot|grub-client|Download Demon|SearchExpress|Microsoft URL Control|borg|altavista|dataminr.com|tweetedtimes.com|TrendsmapResolver|teoma|blitzbot|oegp|furlbot|http%20client|polybot|htdig|mogimogi|larbin|scrubby|searchsight|seekbot|semanticdiscovery|snappy|vortex(?!(?: Build|Plus))|zeal(?!ot)|fast-webcrawler|converacrawler|dataparksearch|findlinks|BrowserMob|HttpMonitor|ThumbShotsBot|URL2PNG|ZooShot|GomezA|Google SketchUp|Read%20Later|RackspaceBot|robots|SeopultContentAnalyzer|7Siters|centuryb.o.t9|InterNaetBoten|EasyBib AutoCite|Bidtellect|tomnomnom/meg|My User Agent|cortex|CF-UC User Agent|Re-re Studio|adreview|AHC/|NameOfAgent|Request-Promise|ALittle Client|Hello,? world|wp_is_mobile|0xAbyssalDoesntExist|Anarchy99|^revolt|nvd0rz|xfa1|Hakai|gbrmss|fuck-your-hp|IDBTE4M CODE87|Antoine|Insomania|Hells-Net|b3astmode|Linux Gnu \(cow\)|custom_user_agent|Test Certificate Info|iplabel|Magellan|CustomUserAgent|TheSafex?Internetx?Search|kirkland-signature|research|project(?!or))' +- regex: '(A6-Indexer|nuhk|TsolCrawler|Yammybot|Openbot|Gulper Web Bot|grub-client|Download Demon|SearchExpress|Microsoft URL Control|borg|altavista|dataminr.com|tweetedtimes.com|TrendsmapResolver|teoma|blitzbot|oegp|furlbot|http%20client|polybot|htdig|mogimogi|larbin|scrubby|searchsight|seekbot|semanticdiscovery|snappy|vortex(?!(?: Build|Plus))|zeal(?!ot)|fast-webcrawler|converacrawler|dataparksearch|findlinks|BrowserMob|HttpMonitor|ThumbShotsBot|URL2PNG|ZooShot|GomezA|Google SketchUp|Read%20Later|RackspaceBot|robots|SeopultContentAnalyzer|7Siters|centuryb.o.t9|InterNaetBoten|EasyBib AutoCite|Bidtellect|tomnomnom/meg|My User Agent|cortex|CF-UC User Agent|Re-re Studio|adreview|AHC/|NameOfAgent|Request-Promise|ALittle Client|Hello,? world|wp_is_mobile|0xAbyssalDoesntExist|Anarchy99|^revolt|nvd0rz|xfa1|Hakai|gbrmss|fuck-your-hp|IDBTE4M CODE87|Antoine|Insomania|Hells-Net|b3astmode|Linux Gnu \(cow\)|custom_user_agent|Test Certificate Info|iplabel|Magellan|CustomUserAgent|TheSafex?Internetx?Search|kirkland-signature|default_useragent)' name: 'Generic Bot' -- regex: '[a-z0-9\-_]*((?