Use browser client hints for detection (#18843)

* inject client hints in js

* use client hints for detection

* don't use catch, as yui compressor can't parse it

* rebuilt js files

* use new version of device detector

* more code adjustments

* updates expected test files

* improve js

* fix header detection

* improve cache key handling

* fix tests

* use a separate queue to wait for client hints if needed

* try to fix js tests

* also consider X_HTTP_REQUESTED_WITH header as client hints

* updates expected test files

* Extend demo detection with client hints

* code improvements

* use new version of matomo-php-tracker

* Adds test case for client hints set through matomo php tracker

* apply review feedback

* submodule update

* fix test

16 files changed, 178 insertions, 30 deletions

diff --git a/plugins/DevicesDetection/Columns/Base.php b/plugins/DevicesDetection/Columns/Base.php
index 5c3a073c91..a28b7de08a 100644
--- a/plugins/DevicesDetection/Columns/Base.php
+++ b/plugins/DevicesDetection/Columns/Base.php
@@ -14,8 +14,8 @@ use Piwik\Plugin\Dimension\VisitDimension;
 
 abstract class Base extends VisitDimension
 {
-    protected function getUAParser($userAgent)
+    protected function getUAParser($userAgent, $clientHints)
     {
-        return StaticContainer::get(DeviceDetectorFactory::class)->makeInstance($userAgent);
+        return StaticContainer::get(DeviceDetectorFactory::class)->makeInstance($userAgent, $clientHints);
     }
 }
diff --git a/plugins/DevicesDetection/Columns/BrowserEngine.php b/plugins/DevicesDetection/Columns/BrowserEngine.php
index d69d70c976..0e5cfd453c 100644
--- a/plugins/DevicesDetection/Columns/BrowserEngine.php
+++ b/plugins/DevicesDetection/Columns/BrowserEngine.php
@@ -37,8 +37,7 @@ class BrowserEngine extends Base
      */
     public function onNewVisit(Request $request, Visitor $visitor, $action)
     {
-        $userAgent = $request->getUserAgent();
-        $parser    = $this->getUAParser($userAgent);
+        $parser    = $this->getUAParser($request->getUserAgent(), $request->getClientHints());
 
         $aBrowserInfo = $parser->getClient();
 
diff --git a/plugins/DevicesDetection/Columns/BrowserName.php b/plugins/DevicesDetection/Columns/BrowserName.php
index 69ff56a235..8acaaed039 100644
--- a/plugins/DevicesDetection/Columns/BrowserName.php
+++ b/plugins/DevicesDetection/Columns/BrowserName.php
@@ -76,8 +76,7 @@ class BrowserName extends Base
      */
     public function onNewVisit(Request $request, Visitor $visitor, $action)
     {
-        $userAgent = $request->getUserAgent();
-        $parser    = $this->getUAParser($userAgent);
+        $parser    = $this->getUAParser($request->getUserAgent(), $request->getClientHints());
 
         $aBrowserInfo = $parser->getClient();
 
diff --git a/plugins/DevicesDetection/Columns/BrowserVersion.php b/plugins/DevicesDetection/Columns/BrowserVersion.php
index 9b2de1de0b..a57bf30593 100644
--- a/plugins/DevicesDetection/Columns/BrowserVersion.php
+++ b/plugins/DevicesDetection/Columns/BrowserVersion.php
@@ -32,8 +32,7 @@ class BrowserVersion extends Base
      */
     public function onNewVisit(Request $request, Visitor $visitor, $action)
     {
-        $userAgent = $request->getUserAgent();
-        $parser    = $this->getUAParser($userAgent);
+        $parser    = $this->getUAParser($request->getUserAgent(), $request->getClientHints());
 
         $aBrowserInfo = $parser->getClient();
 
diff --git a/plugins/DevicesDetection/Columns/ClientType.php b/plugins/DevicesDetection/Columns/ClientType.php
index 489221bb3e..da1066be71 100644
--- a/plugins/DevicesDetection/Columns/ClientType.php
+++ b/plugins/DevicesDetection/Columns/ClientType.php
@@ -48,8 +48,7 @@ class ClientType extends Base
      */
     public function onNewVisit(Request $request, Visitor $visitor, $action)
     {
-        $userAgent = $request->getUserAgent();
-        $parser    = $this->getUAParser($userAgent);
+        $parser    = $this->getUAParser($request->getUserAgent(), $request->getClientHints());
 
         $clientTypes = \Piwik\Plugins\DevicesDetection\getClientTypeMapping();
 
diff --git a/plugins/DevicesDetection/Columns/DeviceBrand.php b/plugins/DevicesDetection/Columns/DeviceBrand.php
index befb4dc75b..683508e431 100644
--- a/plugins/DevicesDetection/Columns/DeviceBrand.php
+++ b/plugins/DevicesDetection/Columns/DeviceBrand.php
@@ -57,8 +57,7 @@ class DeviceBrand extends Base
      */
     public function onNewVisit(Request $request, Visitor $visitor, $action)
     {
-        $userAgent = $request->getUserAgent();
-        $parser    = $this->getUAParser($userAgent);
+        $parser    = $this->getUAParser($request->getUserAgent(), $request->getClientHints());
 
         return $parser->getBrand();
     }
diff --git a/plugins/DevicesDetection/Columns/DeviceModel.php b/plugins/DevicesDetection/Columns/DeviceModel.php
index 98f2c9fb18..6bb3626dca 100644
--- a/plugins/DevicesDetection/Columns/DeviceModel.php
+++ b/plugins/DevicesDetection/Columns/DeviceModel.php
@@ -30,8 +30,7 @@ class DeviceModel extends Base
      */
     public function onNewVisit(Request $request, Visitor $visitor, $action)
     {
-        $userAgent = $request->getUserAgent();
-        $parser    = $this->getUAParser($userAgent);
+        $parser    = $this->getUAParser($request->getUserAgent(), $request->getClientHints());
 
         $model = $parser->getModel();
 
diff --git a/plugins/DevicesDetection/Columns/DeviceType.php b/plugins/DevicesDetection/Columns/DeviceType.php
index a59974368d..a069b485cd 100644
--- a/plugins/DevicesDetection/Columns/DeviceType.php
+++ b/plugins/DevicesDetection/Columns/DeviceType.php
@@ -50,8 +50,7 @@ class DeviceType extends Base
      */
     public function onNewVisit(Request $request, Visitor $visitor, $action)
     {
-        $userAgent = $request->getUserAgent();
-        $parser    = $this->getUAParser($userAgent);
+        $parser    = $this->getUAParser($request->getUserAgent(), $request->getClientHints());
 
         return $parser->getDevice();
     }
diff --git a/plugins/DevicesDetection/Columns/Os.php b/plugins/DevicesDetection/Columns/Os.php
index 63ed9c342e..48c617de09 100644
--- a/plugins/DevicesDetection/Columns/Os.php
+++ b/plugins/DevicesDetection/Columns/Os.php
@@ -78,8 +78,7 @@ class Os extends Base
      */
     public function onNewVisit(Request $request, Visitor $visitor, $action)
     {
-        $userAgent = $request->getUserAgent();
-        $parser    = $this->getUAParser($userAgent);
+        $parser    = $this->getUAParser($request->getUserAgent(), $request->getClientHints());
 
         if ($parser->isBot()) {
             $os = Settings::OS_BOT;
diff --git a/plugins/DevicesDetection/Columns/OsVersion.php b/plugins/DevicesDetection/Columns/OsVersion.php
index 6df88cd832..e9d063cc8f 100644
--- a/plugins/DevicesDetection/Columns/OsVersion.php
+++ b/plugins/DevicesDetection/Columns/OsVersion.php
@@ -30,8 +30,7 @@ class OsVersion extends Base
      */
     public function onNewVisit(Request $request, Visitor $visitor, $action)
     {
-        $userAgent = $request->getUserAgent();
-        $parser    = $this->getUAParser($userAgent);
+        $parser    = $this->getUAParser($request->getUserAgent(), $request->getClientHints());
 
         return $parser->getOs('version');
     }
diff --git a/plugins/DevicesDetection/Controller.php b/plugins/DevicesDetection/Controller.php
index d74ba9d527..dff2b25d67 100644
--- a/plugins/DevicesDetection/Controller.php
+++ b/plugins/DevicesDetection/Controller.php
@@ -8,6 +8,7 @@
  */
 namespace Piwik\Plugins\DevicesDetection;
 
+use DeviceDetector\ClientHints;
 use DeviceDetector\DeviceDetector;
 use Piwik\Common;
 use Piwik\Piwik;
@@ -25,11 +26,13 @@ class Controller extends \Piwik\Plugin\Controller
         ControllerAdmin::setBasicVariablesAdminView($view);
 
         $userAgent = Common::getRequestVar('ua', $_SERVER['HTTP_USER_AGENT'], 'string');
+        $clientHints = Common::getRequestVar('clienthints', '', 'json');
 
-        $uaParser = new DeviceDetector($userAgent);
+        $uaParser = new DeviceDetector($userAgent, is_array($clientHints) ? ClientHints::factory($clientHints) : null);
         $uaParser->parse();
 
         $view->userAgent           = $userAgent;
+        $view->clientHints         = $clientHints;
         $view->bot_info            = $uaParser->getBot();
         $view->browser_name        = $uaParser->getClient('name');
         $view->browser_short_name  = $uaParser->getClient('short_name');
diff --git a/plugins/DevicesDetection/lang/en.json b/plugins/DevicesDetection/lang/en.json
index acec477774..b1a18c7a83 100644
--- a/plugins/DevicesDetection/lang/en.json
+++ b/plugins/DevicesDetection/lang/en.json
@@ -22,6 +22,9 @@
         "dataTableLabelTypes": "Type",
         "ClientType": "Client type",
         "ClientTypes": "Client types",
+        "ClientHints": "Client Hints",
+        "ClientHintsNotSupported": "Your Browser does not support client hints.",
+        "ConsiderClientHints": "Consider Client Hints",
         "Device": "Device",
         "DeviceBrand": "Device brand",
         "DeviceBrands": "Device brands",
diff --git a/plugins/DevicesDetection/templates/detection.twig b/plugins/DevicesDetection/templates/detection.twig
index 399e6434c5..c78dcf1284 100644
--- a/plugins/DevicesDetection/templates/detection.twig
+++ b/plugins/DevicesDetection/templates/detection.twig
@@ -6,6 +6,34 @@
 
     <script type="text/javascript">
 
+        $(document).ready(function() {
+            if (!navigator.userAgentData || typeof navigator.userAgentData.getHighEntropyValues !== 'function') {
+                $('#noclienthints').css({display: 'inline-block'});
+                $('[name=clienthints],.usech').hide();
+            } else {
+                // Initialize with low entropy values that are always available
+                var clientHints = {
+                    brands: navigator.userAgentData.brands,
+                    platform: navigator.userAgentData.platform
+                };
+
+                // try to gather high entropy values
+                // currently this methods simply returns the requested values through a Promise
+                // In later versions it might require a user permission
+                navigator.userAgentData.getHighEntropyValues(
+                    ['brands', 'model', 'platform', 'platformVersion', 'uaFullVersion', 'fullVersionList']
+                ).then(function (ua) {
+                    if (ua.fullVersionList) {
+                        // if fullVersionList is available, brands and uaFullVersion isn't needed
+                        delete ua.brands;
+                        delete ua.uaFullVersion;
+                    }
+
+                    clientHints = ua;
+                });
+            }
+        });
+
         function showList(type) {
             var ajaxHandler = new ajaxHelper();
             ajaxHandler.addParams({
@@ -22,6 +50,15 @@
             ajaxHandler.send();
         }
 
+        function toggleClientHints() {
+            $('[name=clienthints]').toggle();
+            if ($('[name=clienthints]:visible').length) {
+                $('[name=clienthints]').text().length || $('[name=clienthints]').text(JSON.stringify(clientHints));
+            } else {
+                $('[name=clienthints]').text('');
+            }
+        }
+
     </script>
 
     <style type="text/css">
@@ -42,11 +79,26 @@
     </style>
 
     <div piwik-content-block content-title="{{ title|e('html_attr') }}">
-        <h3>{{ 'DevicesDetection_UserAgent'|translate|e('html_attr') }}</h3>
-
         <form action="{{ linkTo({}) }}" method="POST">
+
+            <h3>{{ 'DevicesDetection_UserAgent'|translate }}</h3>
+
             <textarea name="ua">{{ userAgent }}</textarea>
-            <br />
+
+            <h3>{{ 'DevicesDetection_ClientHints'|translate }}</h3>
+
+            <span class="checkbox-container usech">
+                <label>
+                    <input type="checkbox" id="usech" {% if clientHints %}checked{% endif %} onchange="toggleClientHints()"/>
+                    <span>{{ 'DevicesDetection_ConsiderClientHints'|translate }}</span>
+                </label>
+            </span>
+
+            <textarea name="clienthints" style="margin-top: 2em; {% if not clientHints %}display: none{% endif %}">{% if clientHints %}{{ clientHints|json_encode }}{% endif %}</textarea>
+
+            <span id="noclienthints" class="alert alert-warning" style="display: none">{{ 'DevicesDetection_ClientHintsNotSupported'|translate }}</span>
+
+            <br /><br />
             <input type="submit" value="{{ 'General_Refresh'|translate }}" class="btn" />
         </form>
 
diff --git a/plugins/DevicesDetection/tests/Fixtures/MultiDeviceGoalConversions.php b/plugins/DevicesDetection/tests/Fixtures/MultiDeviceGoalConversions.php
index dca57f9897..b4d38249e2 100644
--- a/plugins/DevicesDetection/tests/Fixtures/MultiDeviceGoalConversions.php
+++ b/plugins/DevicesDetection/tests/Fixtures/MultiDeviceGoalConversions.php
@@ -159,6 +159,8 @@ class MultiDeviceGoalConversions extends Fixture
         $t = self::getTracker($this->idSite, $this->getAdjustedDateTime(1.6), $defaultInit = true);
 
         $t->setUserAgent('Mozilla/5.0 (Windows NT 6.1; WOW64; Trident/7.0; Banca Caboto s.p.a.; rv:11.0) like Gecko');
+        // The client hints below should change the OS to Windows 11 and browser to Edge 95.5.2
+        $t->setClientHints('', 'Windows', '14.0.0', '" Not A;Brand";v="99", "Chromium";v="95", "Microsoft Edge";v="95"', '95.5.2');
 
         $t->setUrl('http://example.org/index.htm');
         self::checkResponse($t->doTrackPageView('0'));
diff --git a/plugins/DevicesDetection/tests/System/GoalReportForDevicesTest.php b/plugins/DevicesDetection/tests/System/GoalReportForDevicesTest.php
index 475158055c..8b9e862309 100644
--- a/plugins/DevicesDetection/tests/System/GoalReportForDevicesTest.php
+++ b/plugins/DevicesDetection/tests/System/GoalReportForDevicesTest.php
@@ -1,10 +1,12 @@
 <?php
+
 /**
  * Matomo - free/libre analytics platform
  *
  * @link https://matomo.org
  * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
  */
+
 namespace Piwik\Plugins\DevicesDetection\tests\System;
 
 use Piwik\Plugins\DevicesDetection\tests\Fixtures\MultiDeviceGoalConversions;
@@ -34,11 +36,12 @@ class GoalReportForDevicesTest extends SystemTestCase
         $idSite   = self::$fixture->idSite;
         $dateTime = self::$fixture->dateTime;
 
-        return array(
-            array('DevicesDetection.getType', array('idSite'  => $idSite, 'date' => $dateTime)),
-            array('DevicesDetection.getBrand', array('idSite' => $idSite, 'date' => $dateTime)),
-            array('DevicesDetection.getModel', array('idSite' => $idSite, 'date' => $dateTime)),
-        );
+        return [
+            ['DevicesDetection.getType', ['idSite'  => $idSite, 'date' => $dateTime]],
+            ['DevicesDetection.getOsVersions', ['idSite'  => $idSite, 'date' => $dateTime]],
+            ['DevicesDetection.getBrand', ['idSite' => $idSite, 'date' => $dateTime]],
+            ['DevicesDetection.getModel', ['idSite' => $idSite, 'date' => $dateTime]],
+        ];
     }
 
     /**
@@ -50,4 +53,4 @@ class GoalReportForDevicesTest extends SystemTestCase
     }
 }
 
-GoalReportForDevicesTest::$fixture = new MultiDeviceGoalConversions();
\ No newline at end of file
+GoalReportForDevicesTest::$fixture = new MultiDeviceGoalConversions();
diff --git a/plugins/DevicesDetection/tests/System/expected/test___DevicesDetection.getOsVersions_day.xml b/plugins/DevicesDetection/tests/System/expected/test___DevicesDetection.getOsVersions_day.xml
new file mode 100644
index 0000000000..b64d346b97
--- /dev/null
+++ b/plugins/DevicesDetection/tests/System/expected/test___DevicesDetection.getOsVersions_day.xml
@@ -0,0 +1,94 @@
+<?xml version="1.0" encoding="utf-8" ?>
+<result>
+	<row>
+		<label>Unknown</label>
+		<nb_uniq_visitors>2</nb_uniq_visitors>
+		<nb_visits>3</nb_visits>
+		<nb_actions>2</nb_actions>
+		<nb_users>0</nb_users>
+		<max_actions>1</max_actions>
+		<sum_visit_length>3</sum_visit_length>
+		<bounce_count>3</bounce_count>
+		<nb_visits_converted>1</nb_visits_converted>
+		<segment>operatingSystemCode==UNK;operatingSystemVersion==UNK</segment>
+		<logo>plugins/Morpheus/icons/dist/os/UNK.png</logo>
+	</row>
+	<row>
+		<label>Android 4.2</label>
+		<nb_uniq_visitors>2</nb_uniq_visitors>
+		<nb_visits>2</nb_visits>
+		<nb_actions>3</nb_actions>
+		<nb_users>0</nb_users>
+		<max_actions>2</max_actions>
+		<sum_visit_length>721</sum_visit_length>
+		<bounce_count>1</bounce_count>
+		<nb_visits_converted>1</nb_visits_converted>
+		<segment>operatingSystemCode==AND;operatingSystemVersion==4.2</segment>
+		<logo>plugins/Morpheus/icons/dist/os/AND.png</logo>
+	</row>
+	<row>
+		<label>iOS 6.0</label>
+		<nb_uniq_visitors>1</nb_uniq_visitors>
+		<nb_visits>2</nb_visits>
+		<nb_actions>1</nb_actions>
+		<nb_users>0</nb_users>
+		<max_actions>1</max_actions>
+		<sum_visit_length>3</sum_visit_length>
+		<bounce_count>2</bounce_count>
+		<nb_visits_converted>1</nb_visits_converted>
+		<segment>operatingSystemCode==IOS;operatingSystemVersion==6.0</segment>
+		<logo>plugins/Morpheus/icons/dist/os/IOS.png</logo>
+	</row>
+	<row>
+		<label>iOS 7.1</label>
+		<nb_uniq_visitors>1</nb_uniq_visitors>
+		<nb_visits>2</nb_visits>
+		<nb_actions>1</nb_actions>
+		<nb_users>0</nb_users>
+		<max_actions>1</max_actions>
+		<sum_visit_length>3</sum_visit_length>
+		<bounce_count>2</bounce_count>
+		<nb_visits_converted>1</nb_visits_converted>
+		<segment>operatingSystemCode==IOS;operatingSystemVersion==7.1</segment>
+		<logo>plugins/Morpheus/icons/dist/os/IOS.png</logo>
+	</row>
+	<row>
+		<label>Android 2.3</label>
+		<nb_uniq_visitors>1</nb_uniq_visitors>
+		<nb_visits>1</nb_visits>
+		<nb_actions>1</nb_actions>
+		<nb_users>0</nb_users>
+		<max_actions>1</max_actions>
+		<sum_visit_length>1084</sum_visit_length>
+		<bounce_count>1</bounce_count>
+		<nb_visits_converted>1</nb_visits_converted>
+		<segment>operatingSystemCode==AND;operatingSystemVersion==2.3</segment>
+		<logo>plugins/Morpheus/icons/dist/os/AND.png</logo>
+	</row>
+	<row>
+		<label>Java ME</label>
+		<nb_uniq_visitors>1</nb_uniq_visitors>
+		<nb_visits>1</nb_visits>
+		<nb_actions>1</nb_actions>
+		<nb_users>0</nb_users>
+		<max_actions>1</max_actions>
+		<sum_visit_length>724</sum_visit_length>
+		<bounce_count>1</bounce_count>
+		<nb_visits_converted>1</nb_visits_converted>
+		<segment>operatingSystemCode==JME;operatingSystemVersion==</segment>
+		<logo>plugins/Morpheus/icons/dist/os/UNK.png</logo>
+	</row>
+	<row>
+		<label>Windows 11</label>
+		<nb_uniq_visitors>1</nb_uniq_visitors>
+		<nb_visits>1</nb_visits>
+		<nb_actions>1</nb_actions>
+		<nb_users>0</nb_users>
+		<max_actions>1</max_actions>
+		<sum_visit_length>1084</sum_visit_length>
+		<bounce_count>1</bounce_count>
+		<nb_visits_converted>1</nb_visits_converted>
+		<segment>operatingSystemCode==WIN;operatingSystemVersion==11</segment>
+		<logo>plugins/Morpheus/icons/dist/os/WIN.png</logo>
+	</row>
+</result>
\ No newline at end of file