diff options
author | Ben Burgess <88810029+bx80@users.noreply.github.com> | 2021-11-05 17:45:18 +0300 |
---|---|---|
committer | GitHub <noreply@github.com> | 2021-11-05 17:45:18 +0300 |
commit | e31d6205fbc363558d63d6f6a25bf7afe5d6dd74 (patch) | |
tree | ae8dcdeb6ef2344a979ecc32692b0cfb4c06d774 | |
parent | c743b61947a8729c773c713d897e517babfeadab (diff) |
Fix for HTML entity decoding of single quotes in campaigns and csv/xml exports (#18170)
* Tweaked html entity decoding to include single quotes for visitor detail campaign and csv/xml exports
* Decode HTML entities for outlinks on the visits log
* Updated UI tests for the live visitor log with quote special characters
* Update UI test screenshots
* Update UI test screenshots
* update submodule
Co-authored-by: sgiehl <stefan@matomo.org>
20 files changed, 73 insertions, 32 deletions
diff --git a/core/DataTable/Renderer.php b/core/DataTable/Renderer.php index 4331374fa3..ef7f017b4a 100644 --- a/core/DataTable/Renderer.php +++ b/core/DataTable/Renderer.php @@ -184,7 +184,7 @@ abstract class Renderer extends BaseFactory if (is_string($value) && !is_numeric($value) ) { - $value = html_entity_decode($value, ENT_COMPAT, 'UTF-8'); + $value = html_entity_decode($value, ENT_QUOTES, 'UTF-8'); // make sure non-UTF-8 chars don't cause htmlspecialchars to choke if (function_exists('mb_convert_encoding')) { $value = @mb_convert_encoding($value, 'UTF-8', 'UTF-8'); diff --git a/core/DataTable/Renderer/Csv.php b/core/DataTable/Renderer/Csv.php index aae748c167..bf864238e5 100644 --- a/core/DataTable/Renderer/Csv.php +++ b/core/DataTable/Renderer/Csv.php @@ -241,7 +241,7 @@ class Csv extends Renderer if (is_string($value) && !is_numeric($value) ) { - $value = html_entity_decode($value, ENT_COMPAT, 'UTF-8'); + $value = html_entity_decode($value, ENT_QUOTES, 'UTF-8'); } elseif ($value === false) { $value = 0; } diff --git a/plugins/Live/VisitorDetails.php b/plugins/Live/VisitorDetails.php index ae1b6ebc64..97f78d255d 100644 --- a/plugins/Live/VisitorDetails.php +++ b/plugins/Live/VisitorDetails.php @@ -95,6 +95,10 @@ class VisitorDetails extends VisitorDetailsAbstract $sitesModel = new \Piwik\Plugins\SitesManager\Model(); + if (isset($action['type']) && $action['type'] == 'outlink' && isset($action['url'])) { + $action['url'] = html_entity_decode($action['url'], ENT_QUOTES, "UTF-8"); + } + $view = new View($template); $view->sendHeadersWhenRendering = false; $view->mainUrl = trim(Site::getMainUrlFor($this->getIdSite())); diff --git a/plugins/Live/tests/Fixtures/VisitsWithAllActionsAndDevices.php b/plugins/Live/tests/Fixtures/VisitsWithAllActionsAndDevices.php index cf3ee64fb7..4acf32e1bb 100644 --- a/plugins/Live/tests/Fixtures/VisitsWithAllActionsAndDevices.php +++ b/plugins/Live/tests/Fixtures/VisitsWithAllActionsAndDevices.php @@ -19,6 +19,7 @@ class VisitsWithAllActionsAndDevices extends Fixture { public $dateTime = '2010-02-01 11:22:33'; public $idSite = 1; + private $specialChars = "sc_'_".'"'.'_%27_%22'; public function setUp(): void { @@ -31,14 +32,21 @@ class VisitsWithAllActionsAndDevices extends Fixture CustomDimensionsApi::getInstance()->configureNewCustomDimension(1, 'age', 'visit', 1); CustomDimensionsApi::getInstance()->configureNewCustomDimension(1, 'currency', 'action', 1); + // Campaign - needs a separate user otherwise it overrides the referrer $t = self::getTracker($this->idSite, $this->dateTime, $defaultInit = true); $t->setTokenAuth(self::getTokenAuth()); + $t->setUserId('Z4F66G776HGI'); + $t->setVisitorId(substr(sha1('Z4F66G776HGI'), 0, 16)); + $this->trackCampaignVisit($t, Date::factory($this->dateTime)->addHour(0)->getDatetime()); + // Main user for all other visits + $t = self::getTracker($this->idSite, $this->dateTime, $defaultInit = true); + $t->setTokenAuth(self::getTokenAuth()); $t->setUserId('X4F66G776HGI'); $t->setVisitorId(substr(sha1('X4F66G776HGI'), 0, 16)); // smart display - $this->trackDeviceVisit($t, Date::factory($this->dateTime)->addHour(0)->getDatetime(), 'Mozilla/5.0 (Linux; U; Android 4.0.4; de-de; VSD220 Build/IMM76D.UI23ED12_VSC) AppleWebKit/534.30 (KHTML, like Gecko) Version/4.0 Safari/534.30'); + $this->trackDeviceVisit($t, Date::factory($this->dateTime)->addHour(1)->getDatetime(), 'Mozilla/5.0 (Linux; U; Android 4.0.4; de-de; VSD220 Build/IMM76D.UI23ED12_VSC) AppleWebKit/534.30 (KHTML, like Gecko) Version/4.0 Safari/534.30'); // media player $this->trackVisitMediaPlayer($t, Date::factory($this->dateTime)->addHour(6.33)->getDatetime()); @@ -144,7 +152,7 @@ class VisitsWithAllActionsAndDevices extends Fixture self::checkResponse($t->doTrackPageView('cart')); $t->setForceVisitDateTime(Date::factory($dateTime)->addHour(0.5)->getDatetime()); - self::checkResponse($t->doTrackAction('http://vendor.site', 'link')); + self::checkResponse($t->doTrackAction('http://vendor.site/'.$this->specialChars.'.html', 'link')); } private function trackVisitTablet(\MatomoTracker $t, $dateTime) @@ -152,13 +160,14 @@ class VisitsWithAllActionsAndDevices extends Fixture $t->setForceVisitDateTime($dateTime); $t->setUserAgent('Mozilla/5.0 (Linux; U; en-us; KFAPWI Build/JDQ39) AppleWebKit/535.19 (KHTML, like Gecko) Silk/3.8 Safari/535.19 Silk-Accelerated=true'); - $t->setUrlReferrer('http://www.google.com/search?q=product%2042'); - $t->setUrl('http://example.org/product42'); + $t->setUrlReferrer('http://www.google.com/search?q=product%2042'.$this->specialChars); + $t->setUrl('http://example.org/product42&name='.$this->specialChars); $t->setPerformanceTimings(55, 348, 256, 299, 165, 144); $t->setDebugStringAppend('bw_bytes=6851'); $t->setCustomVariable(1, 'custom', 'variable', 'page'); $t->setCustomDimension('1', '42'); $t->setCustomDimension('2', '€'); + self::checkResponse($t->doTrackPageView('product 42')); $t->setForceVisitDateTime(Date::factory($dateTime)->addHour(0.09)->getDatetime()); @@ -228,4 +237,13 @@ class VisitsWithAllActionsAndDevices extends Fixture $t->doTrackContentImpression('product slider', 'product_18.jpg', 'http://example.org/product18'); $t->doTrackContentImpression('product zoom', 'product_18.jpg', 'http://example.org/product18'); } + + private function trackCampaignVisit(\MatomoTracker $t, $dateTime) + { + $t->setForceVisitDateTime($dateTime); + $t->setUserAgent('Mozilla/5.0 (iPod; U; CPU iPhone OS 4_2_1 like Mac OS X; ja-jp) AppleWebKit/533.17.9 (KHTML, like Gecko) Mobile/8C148'); + + $t->setUrl('http://example.org?pk_campaign='.$this->specialChars); + self::checkResponse($t->doTrackPageView('home')); + } }
\ No newline at end of file diff --git a/plugins/Live/tests/UI/Live_spec.js b/plugins/Live/tests/UI/Live_spec.js index d31d470c6c..f2e49a4a2d 100644 --- a/plugins/Live/tests/UI/Live_spec.js +++ b/plugins/Live/tests/UI/Live_spec.js @@ -142,16 +142,32 @@ describe("Live", function () { expect(await dialog.screenshot()).to.matchImage('visitor_profile_limited'); }); + it('should show visitor log next page', async function() { + await page.goto("?module=CoreHome&action=index&idSite=1&period=year&date=2010-01-03#?idSite=1&period=year&date=2010-01-03&category=General_Visitors&subcategory=Live_VisitorLog"); + + await page.waitForNetworkIdle(); + await page.waitForSelector('.dataTableVizVisitorLog'); + + const link = await page.jQuery('.dataTableNext'); + await link.click(); + await page.waitForNetworkIdle(); + + var report = await page.$('.reporting-page'); + expect(await report.screenshot()).to.matchImage('visitor_log_page_next'); + }); + it('should show visitor log purge message when purged and no data', async function() { testEnvironment.overrideConfig('Deletelogs', 'delete_logs_enable', 1); testEnvironment.overrideConfig('Deletelogs', 'delete_logs_older_than', 4000); testEnvironment.save(); await page.goto("?module=CoreHome&action=index&idSite=1&period=year&date=2005-01-03#?idSite=1&period=year&date=2005-01-03&category=General_Visitors&subcategory=Live_VisitorLog"); - await page.waitForNetworkIdle(); var report = await page.$('.reporting-page'); expect(await report.screenshot()).to.matchImage('visitor_log_purged'); }); + + + }); diff --git a/plugins/Live/tests/UI/expected-screenshots/Live_visitor_log.png b/plugins/Live/tests/UI/expected-screenshots/Live_visitor_log.png index 50c5d91ecd..a537679801 100644 --- a/plugins/Live/tests/UI/expected-screenshots/Live_visitor_log.png +++ b/plugins/Live/tests/UI/expected-screenshots/Live_visitor_log.png @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:5a606dc50663830984e0e30edb727860db7de042c9241e625009923210b89196 -size 386185 +oid sha256:96d72c04beddc6d08cd7c1573e7c3c91f0fec4c65e38127a27b7dbfea0e62150 +size 390945 diff --git a/plugins/Live/tests/UI/expected-screenshots/Live_visitor_log_expand_actions.png b/plugins/Live/tests/UI/expected-screenshots/Live_visitor_log_expand_actions.png index 8264bbfaa6..694d42dac0 100644 --- a/plugins/Live/tests/UI/expected-screenshots/Live_visitor_log_expand_actions.png +++ b/plugins/Live/tests/UI/expected-screenshots/Live_visitor_log_expand_actions.png @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:f0086120e8f9548298357d45ffec30ceb507701687009fc69b9f64917388b970 -size 51598 +oid sha256:5ceb9f98b14d9311758bc4dcdde566352850767c6d119d44f90f5e9387280659 +size 54148 diff --git a/plugins/Live/tests/UI/expected-screenshots/Live_visitor_log_expand_pageview_actions.png b/plugins/Live/tests/UI/expected-screenshots/Live_visitor_log_expand_pageview_actions.png index bb3c6f5196..c795530e4c 100644 --- a/plugins/Live/tests/UI/expected-screenshots/Live_visitor_log_expand_pageview_actions.png +++ b/plugins/Live/tests/UI/expected-screenshots/Live_visitor_log_expand_pageview_actions.png @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:a1b8136cf7a0607857f929eddafd7eaa2719284e94d99434e08ca4b5bde6034f -size 102065 +oid sha256:26b8c4f0871962b4341a30312dbbc4dcdb1a76c6fbf1b6c9099150a081a2158d +size 104093 diff --git a/plugins/Live/tests/UI/expected-screenshots/Live_visitor_log_page_next.png b/plugins/Live/tests/UI/expected-screenshots/Live_visitor_log_page_next.png new file mode 100644 index 0000000000..5b537c0bc2 --- /dev/null +++ b/plugins/Live/tests/UI/expected-screenshots/Live_visitor_log_page_next.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:26b1c6de6eedeaee8757faca38b54a0af4d879091522807ee911c645c0fb0f11 +size 47896 diff --git a/plugins/Live/tests/UI/expected-screenshots/Live_visitor_profile.png b/plugins/Live/tests/UI/expected-screenshots/Live_visitor_profile.png index 96bf7578de..0a87e45e13 100644 --- a/plugins/Live/tests/UI/expected-screenshots/Live_visitor_profile.png +++ b/plugins/Live/tests/UI/expected-screenshots/Live_visitor_profile.png @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:87dfa7cbb02b66ff0ae21baf441e80cc612a45f3abb45ee533deef3839636132 -size 423221 +oid sha256:f2bf8381a4dc8b02d9a90182c4780021d0dfd50bf123805a029311f5d327b3d6 +size 427940 diff --git a/plugins/Live/tests/UI/expected-screenshots/Live_visitor_profile_action_details.png b/plugins/Live/tests/UI/expected-screenshots/Live_visitor_profile_action_details.png index 12e45b210b..bbdb962a97 100644 --- a/plugins/Live/tests/UI/expected-screenshots/Live_visitor_profile_action_details.png +++ b/plugins/Live/tests/UI/expected-screenshots/Live_visitor_profile_action_details.png @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:8f78a93a96e768fbe3f583d95a3ec9f83b7c52b77b08bd3c29a3e2a883487a91 -size 297878 +oid sha256:76cb9e7bd015dd398addb99db083a58d16276f608f04f2ef5d6a5a2760b9fd0a +size 301114 diff --git a/plugins/Live/tests/UI/expected-screenshots/Live_visitor_profile_action_tooltip.png b/plugins/Live/tests/UI/expected-screenshots/Live_visitor_profile_action_tooltip.png index c0bab52fc4..2fd86f4d87 100644 --- a/plugins/Live/tests/UI/expected-screenshots/Live_visitor_profile_action_tooltip.png +++ b/plugins/Live/tests/UI/expected-screenshots/Live_visitor_profile_action_tooltip.png @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:95e7fc93175b8223514f90fefd271bafcbee14fcad2ec389a675c85b875b189c -size 11695 +oid sha256:28d07596ae1d1e34e7cecae8ca17214ad928ece003dd31d224ee7104e6f9f624 +size 13386 diff --git a/plugins/Live/tests/UI/expected-screenshots/Live_visitor_profile_actions_hidden.png b/plugins/Live/tests/UI/expected-screenshots/Live_visitor_profile_actions_hidden.png index 47b72ddeea..4bc4a0b924 100644 --- a/plugins/Live/tests/UI/expected-screenshots/Live_visitor_profile_actions_hidden.png +++ b/plugins/Live/tests/UI/expected-screenshots/Live_visitor_profile_actions_hidden.png @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:825ac1227ff83003aaf61a5fa3d0cf0291a13da28f3417bb85d11eeef687241b -size 263180 +oid sha256:0d80b43d87d3b6a8b2fcdcb2150ee4c24715c3d893f1df4e410cec58fddfcdda +size 263689 diff --git a/plugins/Live/tests/UI/expected-screenshots/Live_visitor_profile_limited.png b/plugins/Live/tests/UI/expected-screenshots/Live_visitor_profile_limited.png index 69407fc7f8..b7d4a7e7e7 100644 --- a/plugins/Live/tests/UI/expected-screenshots/Live_visitor_profile_limited.png +++ b/plugins/Live/tests/UI/expected-screenshots/Live_visitor_profile_limited.png @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:1e8980055f55066eed4f95bfb2e40e293a953a2478ca18ac15c2a7db3192dd7c -size 311397 +oid sha256:823ed4c0d4dbc06e83a92f053fa51a743fa1ca3731cd0e7ffff157fc856e75d9 +size 316178 diff --git a/plugins/Live/tests/UI/expected-screenshots/Live_visitor_profile_more_visits.png b/plugins/Live/tests/UI/expected-screenshots/Live_visitor_profile_more_visits.png index 1545c63998..989e3979dd 100644 --- a/plugins/Live/tests/UI/expected-screenshots/Live_visitor_profile_more_visits.png +++ b/plugins/Live/tests/UI/expected-screenshots/Live_visitor_profile_more_visits.png @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:69241c8e7ab81203abf8faffcb241616151ba000afa093a4efc052b04aba9408 -size 449462 +oid sha256:0e5605052b14493fbbe028ac35bcd9077892e533ecdf94e2fd9311da671582f4 +size 454424 diff --git a/plugins/Live/tests/UI/expected-screenshots/Live_visitor_profile_visit_details.png b/plugins/Live/tests/UI/expected-screenshots/Live_visitor_profile_visit_details.png index 5f32e56a8b..af0f28ddda 100644 --- a/plugins/Live/tests/UI/expected-screenshots/Live_visitor_profile_visit_details.png +++ b/plugins/Live/tests/UI/expected-screenshots/Live_visitor_profile_visit_details.png @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:5d2295f79916b22ace47155c4526827ac00c69c83447f0581448a6bb4497f04b -size 273722 +oid sha256:66aa460a5ae683667f61d9bb3933042b29c0137915e556754eb3a50092c99f3e +size 274883 diff --git a/plugins/MarketingCampaignsReporting b/plugins/MarketingCampaignsReporting -Subproject 1eb724fd191b706144dd6f0d8859d1efcbd6b1f +Subproject 2146eac5028499d8ebde517128a3155f23298ea diff --git a/plugins/Referrers/VisitorDetails.php b/plugins/Referrers/VisitorDetails.php index ece8dc40fe..8bb4bd93b5 100644 --- a/plugins/Referrers/VisitorDetails.php +++ b/plugins/Referrers/VisitorDetails.php @@ -96,7 +96,7 @@ class VisitorDetails extends VisitorDetailsAbstract protected function getReferrerName(): string { - return urldecode($this->details['referer_name'] ?? ''); + return html_entity_decode(($this->details['referer_name'] ?? ''), ENT_QUOTES, "UTF-8"); } protected function getSearchEngineUrl() diff --git a/tests/UI/expected-screenshots/UIIntegrationTest_actions_outlinks_vlog.png b/tests/UI/expected-screenshots/UIIntegrationTest_actions_outlinks_vlog.png index 9a8b129fad..192d6d800a 100644 --- a/tests/UI/expected-screenshots/UIIntegrationTest_actions_outlinks_vlog.png +++ b/tests/UI/expected-screenshots/UIIntegrationTest_actions_outlinks_vlog.png @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:538571f7e610df012df69361973ed9aa801f9e66d8c6c59f540b149e30ee7357 -size 76840 +oid sha256:84cf856bedceefed3a488acefd70037265a575c69487d4b3a704d631852e44b1 +size 76264 diff --git a/tests/UI/expected-screenshots/UIIntegrationTest_dashboard3.png b/tests/UI/expected-screenshots/UIIntegrationTest_dashboard3.png index c96c3926b6..eac9cacdae 100644 --- a/tests/UI/expected-screenshots/UIIntegrationTest_dashboard3.png +++ b/tests/UI/expected-screenshots/UIIntegrationTest_dashboard3.png @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:0bc0c865e5b6a4ac9bccfa786047ef57a0cb871b24b6923d32b5497877c9d6b7 -size 712601 +oid sha256:4141352d07768f4b0c71170180b051d9c58b2b5d3c63e6cae6e354cc85eda399 +size 712615 |