feat(user-agent): expand OS, browser, device, and bot detection

.vale/styles/config/vocabularies/Utopia/accept.txt

@@ -43,6 +43,39 @@ Fux
 Dittmann
 Torsten
 United States
+Wget
+aiohttp
+Axios
+HTTPie
+GPTBot
+ClaudeBot
+PerplexityBot
+OkHttp
+Guzzle
+macOS
+iOS
+iPadOS
+tvOS
+watchOS
+HarmonyOS
+webOS
+Sailfish
+KaiOS
+BlackBerry
+PlayStation
+Nintendo
+Ubuntu
+Debian
+Fedora
+Vivaldi
+Yandex
+DuckDuckGo
+Huawei
+Coc
+Silk
+WebView
+QQ
+UC
 
 # Protocols, formats, acronyms
 OAuth

packages/user-agent/CHANGELOG.md

@@ -1,5 +1,27 @@
 # Changelog
 
+## 0.2.0
+
+- Expanded operating-system coverage: iPadOS, tvOS, watchOS, HarmonyOS,
+  OpenHarmony, Fire OS, webOS, Sailfish, BlackBerry, PlayStation, Nintendo,
+  and popular Linux distributions (Debian, Fedora, Arch Linux, Mint, and
+  more).
+- Expanded browser coverage: Opera Mobile, Brave, Vivaldi, Yandex Browser,
+  UC Browser, DuckDuckGo, QQ Browser, Coc Coc, Whale, Huawei Browser, Amazon
+  Silk, and Firefox Focus, all resolved ahead of generic Chrome and WebView
+  rules.
+- Expanded HTTP-library coverage: Python urllib, aiohttp, Go-http-client,
+  Node Fetch, Axios, HTTPie, Apache HTTP Client, Java, and more.
+- Expanded device-brand coverage: Realme, Asus, Tecno, Infinix, Honor, HTC,
+  Lenovo, ZTE, TCL, Meizu, Nothing, Fairphone, Alcatel, and LG/Sony smart
+  TVs.
+- Expanded crawler coverage: PerplexityBot, Google Extended, GoogleOther,
+  Meta External Agent, CCBot, YouBot, Yahoo Slurp, SeznamBot, Pinterest,
+  and more, with narrowed matching so browsers such as Sogou and the
+  Pinterest app are not misclassified as bots.
+- Short-name codes stay aligned with Matomo DeviceDetector so the added
+  categories match its reference output.
+
 ## 0.1.0
 
 - Added lazy, memoized operating-system, client, device, and bot detection.

packages/user-agent/README.md

@@ -111,16 +111,23 @@ $data = $agent->toArray();
 
 ## Detection coverage
 
-The initial rule set covers common user agents:
-
-- Windows, Windows Phone, macOS, iOS, Android, Chrome OS, Linux, Ubuntu,
-  Tizen, and KaiOS
-- Chrome, Safari, Firefox, Edge, Opera, Samsung Internet, Android WebView,
-  Internet Explorer, and their common mobile variants
-- Common HTTP libraries
+The rule set covers common user agents:
+
+- Windows, Windows Phone, macOS, iOS, iPadOS, tvOS, watchOS, Android,
+  Fire OS, Chrome OS, HarmonyOS, webOS, Sailfish, Tizen, KaiOS, BlackBerry,
+  PlayStation, Nintendo, and popular Linux distributions (Ubuntu, Debian,
+  Fedora, Arch Linux, Mint, and more)
+- Chrome, Safari, Firefox, Edge, Opera, Opera Mobile, Samsung Internet,
+  Brave, Vivaldi, Yandex Browser, UC Browser, DuckDuckGo, QQ Browser,
+  Coc Coc, Whale, Huawei Browser, Amazon Silk, Firefox Focus, Android
+  WebView, Internet Explorer, and their common mobile variants
+- Common HTTP libraries (curl, Wget, OkHttp, Guzzle, Python Requests,
+  aiohttp, Go-http-client, Node Fetch, Axios, Java, HTTPie, and more)
 - Desktop, smartphone, tablet, TV, console, wearable, and portable-media
-  device classes with common mobile brands and models
+  device classes with common device brands and models
 - Search, social-preview, monitoring, automation, SEO, and AI crawlers
+  (including GPTBot, ClaudeBot, PerplexityBot, Google Extended, and other
+  large-language-model agents)
 
 Core result fields are regression-tested against Matomo DeviceDetector 6.4 for
 representative profiles. The runtime does not contain or load Matomo's LGPL

packages/user-agent/benchmark/benchmark.php

@@ -14,7 +14,11 @@
     'Mozilla/5.0 (Linux; Android 13; Pixel 7 Pro Build/TQ3A.230805.001) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Mobile Safari/537.36',
     'Mozilla/5.0 (Linux; Android 12; SM-T970 Build/SP1A.210812.016) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/109.0.0.0 Safari/537.36',
     'Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:47.0) Gecko/20100101 Firefox/47.0',
+    'Mozilla/5.0 (iPad; CPU OS 17_4 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.4 Mobile/15E148 Safari/604.1',
+    'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 YaBrowser/23.7.1.1140 Safari/537.36',
+    'Mozilla/5.0 (X11; Linux x86_64; rv:102.0) Gecko/20100101 Firefox/102.0 Debian/102.0',
     'Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)',
+    'Mozilla/5.0 (compatible; PerplexityBot/1.0; +https://perplexity.ai/perplexitybot)',
     'PostmanRuntime/7.43.0',
     'curl/8.7.1',
     'UNKNOWN',

packages/user-agent/src/UserAgent/Detection/BotDetector.php

@@ -12,34 +12,73 @@ final class BotDetector
     private const array BOTS = [
         'googlebot' => ['Googlebot', 'search crawler'],
         'google-inspectiontool' => ['Google Inspection Tool', 'search crawler'],
+        'googleother' => ['GoogleOther', 'search crawler'],
+        'google-extended' => ['Google Extended', 'ai crawler'],
+        'storebot-google' => ['Google StoreBot', 'search crawler'],
         'adsbot-google' => ['Google AdsBot', 'advertising crawler'],
+        'mediapartners-google' => ['Google AdSense', 'advertising crawler'],
         'bingbot' => ['Bingbot', 'search crawler'],
+        'bingpreview' => ['Bing Preview', 'search crawler'],
         'duckduckbot' => ['DuckDuckBot', 'search crawler'],
+        'duckassistbot' => ['DuckAssistBot', 'ai crawler'],
         'baiduspider' => ['Baiduspider', 'search crawler'],
         'yandexbot' => ['YandexBot', 'search crawler'],
+        'yandeximages' => ['YandexImages', 'search crawler'],
+        'slurp' => ['Yahoo! Slurp', 'search crawler'],
+        'seznambot' => ['SeznamBot', 'search crawler'],
+        'sogou web spider' => ['Sogou Spider', 'search crawler'],
+        'exabot' => ['Exabot', 'search crawler'],
+        'yeti' => ['Naver Bot', 'search crawler'],
+        'yisouspider' => ['YisouSpider', 'search crawler'],
+        'applebot-extended' => ['Applebot Extended', 'ai crawler'],
         'applebot' => ['Applebot', 'search crawler'],
         'petalbot' => ['PetalBot', 'search crawler'],
         'facebookexternalhit' => ['Facebook External Hit', 'social preview'],
+        'facebookbot' => ['Facebook Bot', 'social preview'],
         'facebot' => ['Facebook Bot', 'social preview'],
+        'meta-externalagent' => ['Meta External Agent', 'ai crawler'],
+        'meta-externalfetcher' => ['Meta External Fetcher', 'social preview'],
         'twitterbot' => ['Twitterbot', 'social preview'],
         'linkedinbot' => ['LinkedInBot', 'social preview'],
         'slackbot' => ['Slackbot', 'social preview'],
         'discordbot' => ['Discordbot', 'social preview'],
-        'whatsapp' => ['WhatsApp', 'social preview'],
         'telegrambot' => ['TelegramBot', 'social preview'],
+        'pinterestbot' => ['Pinterestbot', 'social preview'],
+        'pinterest/0.' => ['Pinterest', 'social preview'],
+        'redditbot' => ['Redditbot', 'social preview'],
         'ahrefsbot' => ['AhrefsBot', 'site crawler'],
         'semrushbot' => ['SemrushBot', 'site crawler'],
         'mj12bot' => ['MJ12bot', 'site crawler'],
+        'dotbot' => ['DotBot', 'site crawler'],
+        'dataforseobot' => ['DataForSeoBot', 'site crawler'],
+        'blexbot' => ['BLEXBot', 'site crawler'],
+        'screaming frog' => ['Screaming Frog SEO Spider', 'site crawler'],
         'gptbot' => ['GPTBot', 'ai crawler'],
+        'oai-searchbot' => ['OAI SearchBot', 'ai crawler'],
         'chatgpt-user' => ['ChatGPT User', 'ai assistant'],
         'claudebot' => ['ClaudeBot', 'ai crawler'],
+        'claude-user' => ['Claude User', 'ai assistant'],
+        'claude-searchbot' => ['Claude SearchBot', 'ai crawler'],
         'claude-web' => ['Claude Web', 'ai assistant'],
+        'anthropic-ai' => ['Anthropic AI', 'ai crawler'],
+        'perplexitybot' => ['PerplexityBot', 'ai crawler'],
+        'perplexity-user' => ['Perplexity User', 'ai assistant'],
         'amazonbot' => ['Amazonbot', 'ai crawler'],
         'bytespider' => ['Bytespider', 'ai crawler'],
+        'ccbot' => ['CCBot', 'ai crawler'],
+        'youbot' => ['YouBot', 'ai crawler'],
+        'cohere-ai' => ['Cohere AI', 'ai crawler'],
+        'cohere-training-data-crawler' => ['Cohere', 'ai crawler'],
+        'diffbot' => ['Diffbot', 'ai crawler'],
+        'imagesiftbot' => ['ImageSift Bot', 'ai crawler'],
+        'timpibot' => ['Timpibot', 'ai crawler'],
         'headlesschrome' => ['Headless Chrome', 'automation'],
         'phantomjs' => ['PhantomJS', 'automation'],
         'lighthouse' => ['Lighthouse', 'site monitor'],
         'uptimerobot' => ['UptimeRobot', 'site monitor'],
+        'pingdom' => ['Pingdom', 'site monitor'],
+        'statuscake' => ['StatusCake', 'site monitor'],
+        'gtmetrix' => ['GTmetrix', 'site monitor'],
     ];
 
     public static function detect(string $userAgent): ?Bot

packages/user-agent/src/UserAgent/Detection/ClientDetector.php

@@ -19,6 +19,7 @@ public static function detect(string $userAgent): Client
             ?? self::samsung($userAgent)
             ?? self::chromeIos($userAgent)
             ?? self::firefoxIos($userAgent)
+            ?? self::derivative($userAgent)
             ?? self::androidWebView($userAgent)
             ?? self::chrome($userAgent)
             ?? self::firefox($userAgent)
@@ -29,6 +30,76 @@ public static function detect(string $userAgent): Client
         return $client ?? new Client();
     }
 
+    /**
+     * Chromium-based browsers that ship their own token. These must resolve
+     * before the generic Chrome and Android WebView rules, because their
+     * user-agent strings also carry a `Chrome/` token (and sometimes the
+     * `Version/4.0` WebView marker). The rendering engine is Blink and its
+     * version follows the embedded Chrome token.
+     */
+    private static function derivative(string $userAgent): ?Client
+    {
+        /** @var array<string, array{string, string}> $blink */
+        $blink = [
+            '/coc_coc_browser\/([0-9.]+)/i' => ['CC', 'Coc Coc'],
+            '/Vivaldi\/([0-9.]+)/i' => ['VI', 'Vivaldi'],
+            '/YaBrowser\/([0-9.]+)/i' => ['YA', 'Yandex Browser'],
+            '/Brave\/([0-9.]+)/i' => ['BR', 'Brave'],
+            '/Whale\/([0-9.]+)/i' => ['WH', 'Whale Browser'],
+            '/UCBrowser\/([0-9.]+)/i' => ['UC', 'UC Browser'],
+            '/(?:MQQBrowser|QQBrowser)\/([0-9.]+)/i' => ['QQ', 'QQ Browser'],
+            '/DuckDuckGo\/([0-9.]+)/i' => ['DD', 'DuckDuckGo Privacy Browser'],
+            '/Silk\/([0-9.]+)/i' => ['MS', 'Mobile Silk'],
+        ];
+
+        foreach ($blink as $pattern => [$code, $name]) {
+            if (preg_match($pattern, $userAgent, $matches) === 1) {
+                return self::derivativeClient($userAgent, $code, $name, $matches[1]);
+            }
+        }
+
+        // Firefox Focus reports as Blink only when it embeds a Chrome token.
+        if (preg_match('/Focus\/([0-9.]+)/i', $userAgent, $matches) === 1
+            && self::tokenVersion($userAgent, 'Chrome') !== null) {
+            return self::derivativeClient($userAgent, 'FK', 'Firefox Focus', $matches[1]);
+        }
+
+        // Huawei Browser distinguishes its mobile build by name.
+        if (preg_match('/HuaweiBrowser\/([0-9.]+)/i', $userAgent, $matches) === 1) {
+            $mobile = stripos($userAgent, 'Mobile') !== false;
+
+            return self::derivativeClient(
+                $userAgent,
+                $mobile ? 'HU' : 'HP',
+                $mobile ? 'Huawei Browser Mobile' : 'Huawei Browser',
+                $matches[1],
+            );
+        }
+
+        return null;
+    }
+
+    /**
+     * Build a client for a Chromium-based derivative. When the user-agent
+     * carries a `Chrome/` token the engine is Blink at that version; the older
+     * Amazon Silk builds carry no Chrome token, so their engine is the embedded
+     * WebKit instead. The engine is left unknown when neither token is present.
+     */
+    private static function derivativeClient(string $userAgent, string $code, string $name, string $version): Client
+    {
+        $chrome = self::tokenVersion($userAgent, 'Chrome');
+        if ($chrome !== null) {
+            return new Client('browser', $code, $name, self::displayVersion($version), 'Blink', $chrome);
+        }
+
+        $webKit = self::tokenVersion($userAgent, 'AppleWebKit');
+        if ($webKit !== null) {
+            return new Client('browser', $code, $name, self::displayVersion($version), 'WebKit', $webKit);
+        }
+
+        return new Client('browser', $code, $name, self::displayVersion($version));
+    }
+
     private static function edge(string $userAgent): ?Client
     {
         if (preg_match('/(?:EdgA|EdgiOS|Edg|Edge)\/([0-9.]+)/i', $userAgent, $matches) !== 1) {
@@ -58,8 +129,16 @@ private static function opera(string $userAgent): ?Client
         }
 
         $engineVersion = self::tokenVersion($userAgent, 'Chrome') ?? self::version($matches[1]);
+        $mobile = stripos($userAgent, 'Mobile') !== false || stripos($userAgent, 'Opera Mobi') !== false;
 
-        return new Client('browser', 'OP', 'Opera', self::displayVersion($matches[1]), 'Blink', $engineVersion);
+        return new Client(
+            'browser',
+            $mobile ? 'OM' : 'OP',
+            $mobile ? 'Opera Mobile' : 'Opera',
+            self::displayVersion($matches[1]),
+            'Blink',
+            $engineVersion,
+        );
     }
 
     private static function samsung(string $userAgent): ?Client
@@ -205,6 +284,16 @@ private static function library(string $userAgent): ?Client
             '/Dart\/([0-9.]+)/i' => 'Dart',
             '/GuzzleHttp\/([0-9.]+)/i' => 'Guzzle',
             '/python-requests\/([0-9.]+)/i' => 'Python Requests',
+            '/Python-urllib\/?([0-9.]*)/i' => 'Python urllib',
+            '/aiohttp\/([0-9.]+)/i' => 'aiohttp',
+            '/Go-http-client\/([0-9.]+)/i' => 'Go-http-client',
+            '/node-fetch\/([0-9.]+)/i' => 'Node Fetch',
+            '/axios\/([0-9.]+)/i' => 'Axios',
+            '/HTTPie\/([0-9.]+)/i' => 'HTTPie',
+            '/Apache-HttpClient\/([0-9.]+)/i' => 'Apache HTTP Client',
+            '/Java-http-client\/([0-9.]+)/i' => 'Java HTTP Client',
+            '/Java\/([0-9._]+)/i' => 'Java',
+            '/got\/([0-9.]+)/i' => 'got',
         ];
 
         foreach ($libraries as $pattern => $name) {

packages/user-agent/src/UserAgent/Detection/DeviceDetector.php

@@ -49,13 +49,27 @@ private static function television(string $userAgent): ?Device
             return new Device('tv', 'Apple', 'Apple TV');
         }
 
-        if (preg_match('/(?:Smart-?TV|SMARTTV|HbbTV|GoogleTV|Android TV|BRAVIA|NetCast|Tizen TV|webOS)/i', $userAgent) === 1) {
-            return new Device('tv', self::brand($userAgent));
+        if (preg_match('/(?:Smart-?TV|SMARTTV|HbbTV|GoogleTV|Android TV|BRAVIA|NetCast|Tizen TV|web0S|webOS)/i', $userAgent) === 1) {
+            return new Device('tv', self::televisionBrand($userAgent));
         }
 
         return null;
     }
 
+    private static function televisionBrand(string $userAgent): ?string
+    {
+        // LG smart TVs run webOS/NetCast; Samsung TVs report Tizen.
+        if (preg_match('/(?:web0S|webOS|NetCast|\bLG\b)/i', $userAgent) === 1) {
+            return 'LG';
+        }
+
+        if (preg_match('/(?:Tizen|BRAVIA)/i', $userAgent) === 1) {
+            return preg_match('/BRAVIA/i', $userAgent) === 1 ? 'Sony' : 'Samsung';
+        }
+
+        return self::brand($userAgent);
+    }
+
     private static function apple(string $userAgent): ?Device
     {
         if (stripos($userAgent, 'iPad') !== false) {
@@ -170,14 +184,27 @@ private static function brand(string $userAgent, ?string $model = null): ?string
             'Samsung' => '/(?:\bSM-[A-Z0-9]+|Samsung)/i',
             'Google' => '/(?:\bPixel\b|Nexus)/i',
             'Huawei' => '/(?:Huawei|\bHUAWEI\b|\bANE-|\bELE-|\bVOG-)/i',
-            'Honor' => '/(?:Honor|\bHLK-|\bBKL-)/i',
-            'Xiaomi' => '/(?:Xiaomi|Redmi|\bMi [A-Z0-9])/i',
+            // Match the "HONOR" brand token (uppercase) or "Honor <model>" so the
+            // common word "honor" in app or build tokens is not treated as a brand.
+            'Honor' => '/(?:(?-i:\bHONOR\b)|\bHonor[ _-](?:[0-9]|[XV][0-9]|Play|Magic|View|Note|Pad|Tablet)|\bHLK-|\bBKL-)/i',
+            'Xiaomi' => '/(?:Xiaomi|Redmi|POCO|\bMi [A-Z0-9])/i',
             'OnePlus' => '/(?:OnePlus|\bONEPLUS\b)/i',
             'Oppo' => '/(?:\bOPPO\b|\bCPH[0-9]+)/i',
+            'Realme' => '/(?:realme|\bRMX[0-9]{4}\b)/i',
             'Vivo' => '/(?:\bvivo\b|\bV[0-9]{4})/i',
             'Motorola' => '/(?:Motorola|\bmoto\b|\bXT[0-9]{4})/i',
+            'Asus' => '/(?:\bASUS)/i',
+            'Tecno' => '/(?:\bTECNO\b)/i',
+            'Infinix' => '/(?:Infinix)/i',
             'Nokia' => '/Nokia/i',
             'Sony' => '/(?:Sony|Xperia)/i',
+            'HTC' => '/(?:\bHTC\b)/i',
+            'Lenovo' => '/(?:Lenovo|\bLenovo )/i',
+            'ZTE' => '/(?:\bZTE\b)/i',
+            'TCL' => '/(?:\bTCL\b)/i',
+            'Meizu' => '/(?:Meizu)/i',
+            'Fairphone' => '/(?:Fairphone|\bFP[0-9]\b)/i',
+            'Alcatel' => '/(?:Alcatel)/i',
             'LG' => '/(?:\bLG[- ]|\bLM-[A-Z0-9]+)/i',
             'Amazon' => '/(?:Kindle|Silk\/|\bKF[A-Z0-9]+)/i',
         ];