Use browser client hints for detection (#18843)

* inject client hints in js

* use client hints for detection

* don't use catch, as yui compressor can't parse it

* rebuilt js files

* use new version of device detector

* more code adjustments

* updates expected test files

* improve js

* fix header detection

* improve cache key handling

* fix tests

* use a separate queue to wait for client hints if needed

* try to fix js tests

* also consider X_HTTP_REQUESTED_WITH header as client hints

* updates expected test files

* Extend demo detection with client hints

* code improvements

* use new version of matomo-php-tracker

* Adds test case for client hints set through matomo php tracker

* apply review feedback

* submodule update

* fix test

7 files changed, 62 insertions, 20 deletions

diff --git a/core/Cookie.php b/core/Cookie.php
index 6ce81416e4..e5ec99916f 100644
--- a/core/Cookie.php
+++ b/core/Cookie.php
@@ -456,7 +456,7 @@ class Cookie
             } else {
                 $userAgent = Http::getUserAgent();
                 $ddFactory = StaticContainer::get(\Piwik\DeviceDetector\DeviceDetectorFactory::class);
-                $deviceDetector = $ddFactory->makeInstance($userAgent);
+                $deviceDetector = $ddFactory->makeInstance($userAgent, Http::getClientHintsFromServerVariables());
                 $deviceDetector->parse();
 
                 $browserFamily = \DeviceDetector\Parser\Client\Browser::getBrowserFamily($deviceDetector->getClient('short_name'));
diff --git a/core/DeviceDetector/DeviceDetectorFactory.php b/core/DeviceDetector/DeviceDetectorFactory.php
index bc81a74a57..5902f4fae8 100644
--- a/core/DeviceDetector/DeviceDetectorFactory.php
+++ b/core/DeviceDetector/DeviceDetectorFactory.php
@@ -1,4 +1,5 @@
 <?php
+
 /**
  * Matomo - free/libre analytics platform
  *
@@ -6,50 +7,65 @@
  * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
  *
  */
+
 namespace Piwik\DeviceDetector;
 
+use DeviceDetector\ClientHints;
 use DeviceDetector\DeviceDetector;
 use Piwik\Container\StaticContainer;
 
 class DeviceDetectorFactory
 {
-    protected static $deviceDetectorInstances = array();
+    protected static $deviceDetectorInstances = [];
 
     /**
      * Returns an instance of DeviceDetector for the given user agent. Uses template method pattern
      * and calls getDeviceDetectionInfo() when it doesn't find a matching instance in the cache.
      * @param string $userAgent
-     * @return DeviceDetector|mixed
+     * @param array $clientHints
+     * @return DeviceDetector
      */
-    public function makeInstance($userAgent)
+    public function makeInstance($userAgent, array $clientHints = [])
     {
-        $userAgent = self::getNormalizedUserAgent($userAgent);
+        $cacheKey = self::getNormalizedUserAgent($userAgent, $clientHints);
 
-        if (array_key_exists($userAgent, self::$deviceDetectorInstances)) {
-            return self::$deviceDetectorInstances[$userAgent];
+        if (array_key_exists($cacheKey, self::$deviceDetectorInstances)) {
+            return self::$deviceDetectorInstances[$cacheKey];
         }
 
-        $deviceDetector = $this->getDeviceDetectionInfo($userAgent);
+        $deviceDetector = $this->getDeviceDetectionInfo($userAgent, $clientHints);
 
-        self::$deviceDetectorInstances[$userAgent] = $deviceDetector;
+        self::$deviceDetectorInstances[$cacheKey] = $deviceDetector;
 
         return $deviceDetector;
     }
 
-    public static function getNormalizedUserAgent($userAgent)
+    public static function getNormalizedUserAgent($userAgent, array $clientHints = [])
     {
-        return mb_substr(trim($userAgent), 0, 500);
+        $normalizedClientHints = '';
+        if (is_array($clientHints) && count($clientHints)) {
+            $hints  = ClientHints::factory($clientHints);
+            $brands = $hints->getBrandList();
+            ksort($brands);
+
+            // we only take the (sorted) list of brand, os + version and model name into account, as the other values
+            // are actually not used and should not change the result
+            $normalizedClientHints = md5(json_encode($brands) . $hints->getOperatingSystem() . $hints->getOperatingSystemVersion() . $hints->getModel());
+        }
+
+        return mb_substr($normalizedClientHints . trim($userAgent), 0, 500);
     }
 
     /**
      * Creates a new DeviceDetector for the user agent. Called by makeInstance() when no matching instance
      * was found in the cache.
-     * @param $userAgent
+     * @param string $userAgent
+     * @param array $clientHints
      * @return DeviceDetector
      */
-    protected function getDeviceDetectionInfo($userAgent)
+    protected function getDeviceDetectionInfo($userAgent, array $clientHints = [])
     {
-        $deviceDetector = new DeviceDetector($userAgent);
+        $deviceDetector = new DeviceDetector($userAgent, ClientHints::factory($clientHints));
         $deviceDetector->discardBotInformation();
         $deviceDetector->setCache(StaticContainer::get('DeviceDetector\Cache\Cache'));
         $deviceDetector->parse();
@@ -58,6 +74,6 @@ class DeviceDetectorFactory
 
     public static function clearInstancesCache()
     {
-        self::$deviceDetectorInstances = array();
+        self::$deviceDetectorInstances = [];
     }
-}
\ No newline at end of file
+}
diff --git a/core/Http.php b/core/Http.php
index 6d0117a935..c598b82e63 100644
--- a/core/Http.php
+++ b/core/Http.php
@@ -993,6 +993,24 @@ class Http
             : 'Matomo/' . Version::VERSION;
     }
 
+    public static function getClientHintsFromServerVariables(): array
+    {
+        $clientHints = [];
+
+        foreach ($_SERVER as $key => $value) {
+            if (
+                0 === strpos(strtolower($key), strtolower('HTTP_SEC_CH_UA'))
+                || 'X_HTTP_REQUESTED_WITH' === strtoupper($key)
+            ) {
+                $clientHints[$key] = $value;
+            }
+        }
+
+        ksort($clientHints);
+
+        return $clientHints;
+    }
+
     /**
      * Fetches a file located at `$url` and saves it to `$destinationPath`.
      *
diff --git a/core/SupportedBrowser.php b/core/SupportedBrowser.php
index 3f9f195df0..f2a7cbb133 100644
--- a/core/SupportedBrowser.php
+++ b/core/SupportedBrowser.php
@@ -9,7 +9,6 @@
 
 namespace Piwik;
 
-use Piwik\Piwik;
 use Piwik\Container\StaticContainer;
 use Piwik\DeviceDetector\DeviceDetectorFactory;
 use Piwik\Exception\NotSupportedBrowserException;
@@ -44,7 +43,7 @@ class SupportedBrowser
 
         $ddFactory = StaticContainer::get(DeviceDetectorFactory::class);
         /** @var \DeviceDetector\DeviceDetector */
-        $deviceDetector = $ddFactory->makeInstance($userAgent);
+        $deviceDetector = $ddFactory->makeInstance($userAgent, Http::getClientHintsFromServerVariables());
 
         $deviceDetector->parse();
         $client = $deviceDetector->getClient();
diff --git a/core/Tracker/Request.php b/core/Tracker/Request.php
index 2b1496f451..50a8fa87e4 100644
--- a/core/Tracker/Request.php
+++ b/core/Tracker/Request.php
@@ -14,6 +14,7 @@ use Piwik\Container\StaticContainer;
 use Piwik\Cookie;
 use Piwik\Exception\InvalidRequestParameterException;
 use Piwik\Exception\UnexpectedWebsiteFoundException;
+use Piwik\Http;
 use Piwik\IP;
 use Matomo\Network\IPUtils;
 use Piwik\Piwik;
@@ -634,6 +635,14 @@ class Request
         return Common::getRequestVar('ua', $default, 'string', $this->params);
     }
 
+    public function getClientHints()
+    {
+        // use headers as default if no data was send with the tracking request
+        $default = Http::getClientHintsFromServerVariables();
+
+        return Common::getRequestVar('uadata', $default, 'json', $this->params);
+    }
+
     public function shouldUseThirdPartyCookie()
     {
         return TrackerConfig::getConfigValue('use_third_party_id_cookie', $this->getIdSiteIfExists());
diff --git a/core/Tracker/Settings.php b/core/Tracker/Settings.php
index 74a66d0c65..cf1eba210d 100644
--- a/core/Tracker/Settings.php
+++ b/core/Tracker/Settings.php
@@ -39,7 +39,7 @@ class Settings // TODO: merge w/ visitor recognizer or make it it's own service.
 
         $userAgent = $request->getUserAgent();
 
-        $deviceDetector = StaticContainer::get(DeviceDetectorFactory::class)->makeInstance($userAgent);
+        $deviceDetector = StaticContainer::get(DeviceDetectorFactory::class)->makeInstance($userAgent, $request->getClientHints());
         $aBrowserInfo   = $deviceDetector->getClient();
 
         if (empty($aBrowserInfo['type']) || 'browser' !== $aBrowserInfo['type']) {
diff --git a/core/Tracker/VisitExcluded.php b/core/Tracker/VisitExcluded.php
index 8d412cef26..ca6d64c7c5 100644
--- a/core/Tracker/VisitExcluded.php
+++ b/core/Tracker/VisitExcluded.php
@@ -190,7 +190,7 @@ class VisitExcluded
     {
         $allowBots = $this->request->getParam('bots');
 
-        $deviceDetector = StaticContainer::get(DeviceDetectorFactory::class)->makeInstance($this->userAgent );
+        $deviceDetector = StaticContainer::get(DeviceDetectorFactory::class)->makeInstance($this->userAgent, $this->request->getClientHints());
 
         return !$allowBots
             && ($deviceDetector->isBot() || $this->isIpInRange());