Fix for HTML entity decoding of single quotes in campaigns and csv/xml exports (#18170)

* Tweaked html entity decoding to include single quotes for visitor detail campaign and csv/xml exports

* Decode HTML entities for outlinks on the visits log

* Updated UI tests for the live visitor log with quote special characters

* Update UI test screenshots

* Update UI test screenshots

* update submodule

Co-authored-by: sgiehl <stefan@matomo.org>

20 files changed, 73 insertions, 32 deletions

diff --git a/core/DataTable/Renderer.php b/core/DataTable/Renderer.php
index 4331374fa3..ef7f017b4a 100644
--- a/core/DataTable/Renderer.php
+++ b/core/DataTable/Renderer.php
@@ -184,7 +184,7 @@ abstract class Renderer extends BaseFactory
         if (is_string($value)
             && !is_numeric($value)
         ) {
-            $value = html_entity_decode($value, ENT_COMPAT, 'UTF-8');
+            $value = html_entity_decode($value, ENT_QUOTES, 'UTF-8');
             // make sure non-UTF-8 chars don't cause htmlspecialchars to choke
             if (function_exists('mb_convert_encoding')) {
                 $value = @mb_convert_encoding($value, 'UTF-8', 'UTF-8');
diff --git a/core/DataTable/Renderer/Csv.php b/core/DataTable/Renderer/Csv.php
index aae748c167..bf864238e5 100644
--- a/core/DataTable/Renderer/Csv.php
+++ b/core/DataTable/Renderer/Csv.php
@@ -241,7 +241,7 @@ class Csv extends Renderer
         if (is_string($value)
             && !is_numeric($value)
         ) {
-            $value = html_entity_decode($value, ENT_COMPAT, 'UTF-8');
+            $value = html_entity_decode($value, ENT_QUOTES, 'UTF-8');
         } elseif ($value === false) {
             $value = 0;
         }
diff --git a/plugins/Live/VisitorDetails.php b/plugins/Live/VisitorDetails.php
index ae1b6ebc64..97f78d255d 100644
--- a/plugins/Live/VisitorDetails.php
+++ b/plugins/Live/VisitorDetails.php
@@ -95,6 +95,10 @@ class VisitorDetails extends VisitorDetailsAbstract
 
         $sitesModel = new \Piwik\Plugins\SitesManager\Model();
 
+        if (isset($action['type']) && $action['type'] == 'outlink' && isset($action['url'])) {
+            $action['url'] = html_entity_decode($action['url'], ENT_QUOTES, "UTF-8");
+        }
+
         $view                 = new View($template);
         $view->sendHeadersWhenRendering = false;
         $view->mainUrl        = trim(Site::getMainUrlFor($this->getIdSite()));
diff --git a/plugins/Live/tests/Fixtures/VisitsWithAllActionsAndDevices.php b/plugins/Live/tests/Fixtures/VisitsWithAllActionsAndDevices.php
index cf3ee64fb7..4acf32e1bb 100644
--- a/plugins/Live/tests/Fixtures/VisitsWithAllActionsAndDevices.php
+++ b/plugins/Live/tests/Fixtures/VisitsWithAllActionsAndDevices.php
@@ -19,6 +19,7 @@ class VisitsWithAllActionsAndDevices extends Fixture
 {
     public $dateTime = '2010-02-01 11:22:33';
     public $idSite = 1;
+    private $specialChars = "sc_'_".'"'.'_%27_%22';
 
     public function setUp(): void
     {
@@ -31,14 +32,21 @@ class VisitsWithAllActionsAndDevices extends Fixture
         CustomDimensionsApi::getInstance()->configureNewCustomDimension(1, 'age', 'visit', 1);
         CustomDimensionsApi::getInstance()->configureNewCustomDimension(1, 'currency', 'action', 1);
 
+        // Campaign - needs a separate user otherwise it overrides the referrer
         $t = self::getTracker($this->idSite, $this->dateTime, $defaultInit = true);
         $t->setTokenAuth(self::getTokenAuth());
+        $t->setUserId('Z4F66G776HGI');
+        $t->setVisitorId(substr(sha1('Z4F66G776HGI'), 0, 16));
+        $this->trackCampaignVisit($t, Date::factory($this->dateTime)->addHour(0)->getDatetime());
 
+        // Main user for all other visits
+        $t = self::getTracker($this->idSite, $this->dateTime, $defaultInit = true);
+        $t->setTokenAuth(self::getTokenAuth());
         $t->setUserId('X4F66G776HGI');
         $t->setVisitorId(substr(sha1('X4F66G776HGI'), 0, 16));
 
         // smart display
-        $this->trackDeviceVisit($t, Date::factory($this->dateTime)->addHour(0)->getDatetime(), 'Mozilla/5.0 (Linux; U; Android 4.0.4; de-de; VSD220 Build/IMM76D.UI23ED12_VSC) AppleWebKit/534.30 (KHTML, like Gecko) Version/4.0 Safari/534.30');
+        $this->trackDeviceVisit($t, Date::factory($this->dateTime)->addHour(1)->getDatetime(), 'Mozilla/5.0 (Linux; U; Android 4.0.4; de-de; VSD220 Build/IMM76D.UI23ED12_VSC) AppleWebKit/534.30 (KHTML, like Gecko) Version/4.0 Safari/534.30');
 
         // media player
         $this->trackVisitMediaPlayer($t, Date::factory($this->dateTime)->addHour(6.33)->getDatetime());
@@ -144,7 +152,7 @@ class VisitsWithAllActionsAndDevices extends Fixture
         self::checkResponse($t->doTrackPageView('cart'));
 
         $t->setForceVisitDateTime(Date::factory($dateTime)->addHour(0.5)->getDatetime());
-        self::checkResponse($t->doTrackAction('http://vendor.site', 'link'));
+        self::checkResponse($t->doTrackAction('http://vendor.site/'.$this->specialChars.'.html', 'link'));
     }
 
     private function trackVisitTablet(\MatomoTracker $t, $dateTime)
@@ -152,13 +160,14 @@ class VisitsWithAllActionsAndDevices extends Fixture
         $t->setForceVisitDateTime($dateTime);
         $t->setUserAgent('Mozilla/5.0 (Linux; U; en-us; KFAPWI Build/JDQ39) AppleWebKit/535.19 (KHTML, like Gecko) Silk/3.8 Safari/535.19 Silk-Accelerated=true');
 
-        $t->setUrlReferrer('http://www.google.com/search?q=product%2042');
-        $t->setUrl('http://example.org/product42');
+        $t->setUrlReferrer('http://www.google.com/search?q=product%2042'.$this->specialChars);
+        $t->setUrl('http://example.org/product42&name='.$this->specialChars);
         $t->setPerformanceTimings(55, 348, 256, 299, 165, 144);
         $t->setDebugStringAppend('bw_bytes=6851');
         $t->setCustomVariable(1, 'custom', 'variable', 'page');
         $t->setCustomDimension('1', '42');
         $t->setCustomDimension('2', '€');
+
         self::checkResponse($t->doTrackPageView('product 42'));
 
         $t->setForceVisitDateTime(Date::factory($dateTime)->addHour(0.09)->getDatetime());
@@ -228,4 +237,13 @@ class VisitsWithAllActionsAndDevices extends Fixture
         $t->doTrackContentImpression('product slider', 'product_18.jpg', 'http://example.org/product18');
         $t->doTrackContentImpression('product zoom', 'product_18.jpg', 'http://example.org/product18');
     }
+
+    private function trackCampaignVisit(\MatomoTracker $t, $dateTime)
+    {
+        $t->setForceVisitDateTime($dateTime);
+        $t->setUserAgent('Mozilla/5.0 (iPod; U; CPU iPhone OS 4_2_1 like Mac OS X; ja-jp) AppleWebKit/533.17.9 (KHTML, like Gecko) Mobile/8C148');
+
+        $t->setUrl('http://example.org?pk_campaign='.$this->specialChars);
+        self::checkResponse($t->doTrackPageView('home'));
+    }
 }
\ No newline at end of file
diff --git a/plugins/Live/tests/UI/Live_spec.js b/plugins/Live/tests/UI/Live_spec.js
index d31d470c6c..f2e49a4a2d 100644
--- a/plugins/Live/tests/UI/Live_spec.js
+++ b/plugins/Live/tests/UI/Live_spec.js
@@ -142,16 +142,32 @@ describe("Live", function () {
         expect(await dialog.screenshot()).to.matchImage('visitor_profile_limited');
     });
 
+    it('should show visitor log next page', async function() {
+        await page.goto("?module=CoreHome&action=index&idSite=1&period=year&date=2010-01-03#?idSite=1&period=year&date=2010-01-03&category=General_Visitors&subcategory=Live_VisitorLog");
+
+        await page.waitForNetworkIdle();
+        await page.waitForSelector('.dataTableVizVisitorLog');
+
+        const link = await page.jQuery('.dataTableNext');
+        await link.click();
+        await page.waitForNetworkIdle();
+
+        var report = await page.$('.reporting-page');
+        expect(await report.screenshot()).to.matchImage('visitor_log_page_next');
+    });
+
     it('should show visitor log purge message when purged and no data', async function() {
         testEnvironment.overrideConfig('Deletelogs', 'delete_logs_enable', 1);
         testEnvironment.overrideConfig('Deletelogs', 'delete_logs_older_than', 4000);
         testEnvironment.save();
 
         await page.goto("?module=CoreHome&action=index&idSite=1&period=year&date=2005-01-03#?idSite=1&period=year&date=2005-01-03&category=General_Visitors&subcategory=Live_VisitorLog");
-
         await page.waitForNetworkIdle();
 
         var report = await page.$('.reporting-page');
         expect(await report.screenshot()).to.matchImage('visitor_log_purged');
     });
+
+
+
 });
diff --git a/plugins/Live/tests/UI/expected-screenshots/Live_visitor_log.png b/plugins/Live/tests/UI/expected-screenshots/Live_visitor_log.png
index 50c5d91ecd..a537679801 100644
--- a/plugins/Live/tests/UI/expected-screenshots/Live_visitor_log.png
+++ b/plugins/Live/tests/UI/expected-screenshots/Live_visitor_log.png
@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:5a606dc50663830984e0e30edb727860db7de042c9241e625009923210b89196
-size 386185
+oid sha256:96d72c04beddc6d08cd7c1573e7c3c91f0fec4c65e38127a27b7dbfea0e62150
+size 390945
diff --git a/plugins/Live/tests/UI/expected-screenshots/Live_visitor_log_expand_actions.png b/plugins/Live/tests/UI/expected-screenshots/Live_visitor_log_expand_actions.png
index 8264bbfaa6..694d42dac0 100644
--- a/plugins/Live/tests/UI/expected-screenshots/Live_visitor_log_expand_actions.png
+++ b/plugins/Live/tests/UI/expected-screenshots/Live_visitor_log_expand_actions.png
@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:f0086120e8f9548298357d45ffec30ceb507701687009fc69b9f64917388b970
-size 51598
+oid sha256:5ceb9f98b14d9311758bc4dcdde566352850767c6d119d44f90f5e9387280659
+size 54148
diff --git a/plugins/Live/tests/UI/expected-screenshots/Live_visitor_log_expand_pageview_actions.png b/plugins/Live/tests/UI/expected-screenshots/Live_visitor_log_expand_pageview_actions.png
index bb3c6f5196..c795530e4c 100644
--- a/plugins/Live/tests/UI/expected-screenshots/Live_visitor_log_expand_pageview_actions.png
+++ b/plugins/Live/tests/UI/expected-screenshots/Live_visitor_log_expand_pageview_actions.png
@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a1b8136cf7a0607857f929eddafd7eaa2719284e94d99434e08ca4b5bde6034f
-size 102065
+oid sha256:26b8c4f0871962b4341a30312dbbc4dcdb1a76c6fbf1b6c9099150a081a2158d
+size 104093
diff --git a/plugins/Live/tests/UI/expected-screenshots/Live_visitor_log_page_next.png b/plugins/Live/tests/UI/expected-screenshots/Live_visitor_log_page_next.png
new file mode 100644
index 0000000000..5b537c0bc2
--- /dev/null
+++ b/plugins/Live/tests/UI/expected-screenshots/Live_visitor_log_page_next.png
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:26b1c6de6eedeaee8757faca38b54a0af4d879091522807ee911c645c0fb0f11
+size 47896
diff --git a/plugins/Live/tests/UI/expected-screenshots/Live_visitor_profile.png b/plugins/Live/tests/UI/expected-screenshots/Live_visitor_profile.png
index 96bf7578de..0a87e45e13 100644
--- a/plugins/Live/tests/UI/expected-screenshots/Live_visitor_profile.png
+++ b/plugins/Live/tests/UI/expected-screenshots/Live_visitor_profile.png
@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:87dfa7cbb02b66ff0ae21baf441e80cc612a45f3abb45ee533deef3839636132
-size 423221
+oid sha256:f2bf8381a4dc8b02d9a90182c4780021d0dfd50bf123805a029311f5d327b3d6
+size 427940
diff --git a/plugins/Live/tests/UI/expected-screenshots/Live_visitor_profile_action_details.png b/plugins/Live/tests/UI/expected-screenshots/Live_visitor_profile_action_details.png
index 12e45b210b..bbdb962a97 100644
--- a/plugins/Live/tests/UI/expected-screenshots/Live_visitor_profile_action_details.png
+++ b/plugins/Live/tests/UI/expected-screenshots/Live_visitor_profile_action_details.png
@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:8f78a93a96e768fbe3f583d95a3ec9f83b7c52b77b08bd3c29a3e2a883487a91
-size 297878
+oid sha256:76cb9e7bd015dd398addb99db083a58d16276f608f04f2ef5d6a5a2760b9fd0a
+size 301114
diff --git a/plugins/Live/tests/UI/expected-screenshots/Live_visitor_profile_action_tooltip.png b/plugins/Live/tests/UI/expected-screenshots/Live_visitor_profile_action_tooltip.png
index c0bab52fc4..2fd86f4d87 100644
--- a/plugins/Live/tests/UI/expected-screenshots/Live_visitor_profile_action_tooltip.png
+++ b/plugins/Live/tests/UI/expected-screenshots/Live_visitor_profile_action_tooltip.png
@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:95e7fc93175b8223514f90fefd271bafcbee14fcad2ec389a675c85b875b189c
-size 11695
+oid sha256:28d07596ae1d1e34e7cecae8ca17214ad928ece003dd31d224ee7104e6f9f624
+size 13386
diff --git a/plugins/Live/tests/UI/expected-screenshots/Live_visitor_profile_actions_hidden.png b/plugins/Live/tests/UI/expected-screenshots/Live_visitor_profile_actions_hidden.png
index 47b72ddeea..4bc4a0b924 100644
--- a/plugins/Live/tests/UI/expected-screenshots/Live_visitor_profile_actions_hidden.png
+++ b/plugins/Live/tests/UI/expected-screenshots/Live_visitor_profile_actions_hidden.png
@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:825ac1227ff83003aaf61a5fa3d0cf0291a13da28f3417bb85d11eeef687241b
-size 263180
+oid sha256:0d80b43d87d3b6a8b2fcdcb2150ee4c24715c3d893f1df4e410cec58fddfcdda
+size 263689
diff --git a/plugins/Live/tests/UI/expected-screenshots/Live_visitor_profile_limited.png b/plugins/Live/tests/UI/expected-screenshots/Live_visitor_profile_limited.png
index 69407fc7f8..b7d4a7e7e7 100644
--- a/plugins/Live/tests/UI/expected-screenshots/Live_visitor_profile_limited.png
+++ b/plugins/Live/tests/UI/expected-screenshots/Live_visitor_profile_limited.png
@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:1e8980055f55066eed4f95bfb2e40e293a953a2478ca18ac15c2a7db3192dd7c
-size 311397
+oid sha256:823ed4c0d4dbc06e83a92f053fa51a743fa1ca3731cd0e7ffff157fc856e75d9
+size 316178
diff --git a/plugins/Live/tests/UI/expected-screenshots/Live_visitor_profile_more_visits.png b/plugins/Live/tests/UI/expected-screenshots/Live_visitor_profile_more_visits.png
index 1545c63998..989e3979dd 100644
--- a/plugins/Live/tests/UI/expected-screenshots/Live_visitor_profile_more_visits.png
+++ b/plugins/Live/tests/UI/expected-screenshots/Live_visitor_profile_more_visits.png
@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:69241c8e7ab81203abf8faffcb241616151ba000afa093a4efc052b04aba9408
-size 449462
+oid sha256:0e5605052b14493fbbe028ac35bcd9077892e533ecdf94e2fd9311da671582f4
+size 454424
diff --git a/plugins/Live/tests/UI/expected-screenshots/Live_visitor_profile_visit_details.png b/plugins/Live/tests/UI/expected-screenshots/Live_visitor_profile_visit_details.png
index 5f32e56a8b..af0f28ddda 100644
--- a/plugins/Live/tests/UI/expected-screenshots/Live_visitor_profile_visit_details.png
+++ b/plugins/Live/tests/UI/expected-screenshots/Live_visitor_profile_visit_details.png
@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:5d2295f79916b22ace47155c4526827ac00c69c83447f0581448a6bb4497f04b
-size 273722
+oid sha256:66aa460a5ae683667f61d9bb3933042b29c0137915e556754eb3a50092c99f3e
+size 274883
diff --git a/plugins/MarketingCampaignsReporting b/plugins/MarketingCampaignsReporting
-Subproject 1eb724fd191b706144dd6f0d8859d1efcbd6b1f
+Subproject 2146eac5028499d8ebde517128a3155f23298ea
diff --git a/plugins/Referrers/VisitorDetails.php b/plugins/Referrers/VisitorDetails.php
index ece8dc40fe..8bb4bd93b5 100644
--- a/plugins/Referrers/VisitorDetails.php
+++ b/plugins/Referrers/VisitorDetails.php
@@ -96,7 +96,7 @@ class VisitorDetails extends VisitorDetailsAbstract
 
     protected function getReferrerName(): string
     {
-        return urldecode($this->details['referer_name'] ?? '');
+         return html_entity_decode(($this->details['referer_name'] ?? ''), ENT_QUOTES, "UTF-8");
     }
 
     protected function getSearchEngineUrl()
diff --git a/tests/UI/expected-screenshots/UIIntegrationTest_actions_outlinks_vlog.png b/tests/UI/expected-screenshots/UIIntegrationTest_actions_outlinks_vlog.png
index 9a8b129fad..192d6d800a 100644
--- a/tests/UI/expected-screenshots/UIIntegrationTest_actions_outlinks_vlog.png
+++ b/tests/UI/expected-screenshots/UIIntegrationTest_actions_outlinks_vlog.png
@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:538571f7e610df012df69361973ed9aa801f9e66d8c6c59f540b149e30ee7357
-size 76840
+oid sha256:84cf856bedceefed3a488acefd70037265a575c69487d4b3a704d631852e44b1
+size 76264
diff --git a/tests/UI/expected-screenshots/UIIntegrationTest_dashboard3.png b/tests/UI/expected-screenshots/UIIntegrationTest_dashboard3.png
index c96c3926b6..eac9cacdae 100644
--- a/tests/UI/expected-screenshots/UIIntegrationTest_dashboard3.png
+++ b/tests/UI/expected-screenshots/UIIntegrationTest_dashboard3.png
@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:0bc0c865e5b6a4ac9bccfa786047ef57a0cb871b24b6923d32b5497877c9d6b7
-size 712601
+oid sha256:4141352d07768f4b0c71170180b051d9c58b2b5d3c63e6cae6e354cc85eda399
+size 712615