store segment hash in DB (#17408)

* get segment hash

* convert tab indentations to spaces

* Create 4.3.0-b2.php

* update tests and update file

* bump version

* Update 4.3.0-b3.php

* Update ApiTest.php

* fixing urlencode bugs

* cache segment hashes

* update segment caching

* update segment caching

* add segment cache test

* add testdox to phpunit.xml

* revert phunit.xml

* test investigation

* Update phpunit.xml.dist

* update blobreportlimitingt test

* Revert "update blobreportlimitingt test"

This reverts commit 90fe7355cd1d227193f0b73ddf1d715c8b016616.

* Update phpunit.xml.dist

* Update BlobReportLimitingTest.php

* Update BlobReportLimitingTest.php

* Update SystemTestCase.php

* Update BlobReportLimitingTest.php

* Update SystemTestCase.php

* Update phpunit.xml.dist

* modify mem limit for travis

* Update .travis.yml

* revert travis.yml

* Update SystemTestCase.php

* try test without cache

* test witch cache and gc_disabled

* Revert "test witch cache and gc_disabled"

This reverts commit 7e1d37093c7be648a84b57335edf397c89913758.

* test witch cache and gc_disabled

* use other model method

* refactor test

* Workaround error in Overlay when site has no URLs (#17457)

* Set setting value even if set to NULL so it will still be validated.

* Make sure when creating a site that the urls options is set.

* workaround in Overlay for instances that have an invalid site URL set for some reason

* Add integration tests for changes to SitesManager API.

* revert non-overlay changes

* Add warning if site has no URLs when viewing Overlay.

* add more tests for segment caches

* get segment hash

* convert tab indentations to spaces

* Create 4.3.0-b2.php

* update tests and update file

* bump version

* Update 4.3.0-b3.php

* Update ApiTest.php

* fixing urlencode bugs

* cache segment hashes

* update segment caching

* update segment caching

* add segment cache test

* add testdox to phpunit.xml

* revert phunit.xml

* test investigation

* Update phpunit.xml.dist

* update blobreportlimitingt test

* Revert "update blobreportlimitingt test"

This reverts commit 90fe7355cd1d227193f0b73ddf1d715c8b016616.

* Update phpunit.xml.dist

* Update BlobReportLimitingTest.php

* Update BlobReportLimitingTest.php

* Update SystemTestCase.php

* Update BlobReportLimitingTest.php

* Update SystemTestCase.php

* Update phpunit.xml.dist

* modify mem limit for travis

* Update .travis.yml

* revert travis.yml

* Update SystemTestCase.php

* try test without cache

* test witch cache and gc_disabled

* Revert "test witch cache and gc_disabled"

This reverts commit 7e1d37093c7be648a84b57335edf397c89913758.

* test witch cache and gc_disabled

* use other model method

* refactor test

* add more tests for segment caches

* revert phpunit.xml

* revert phpunit.xml

* add more tests

Co-authored-by: dizzy <diosmosis@users.noreply.github.com>

5 files changed, 104 insertions, 8 deletions

diff --git a/core/Archive.php b/core/Archive.php
index 0c0651a029..8338d19ec3 100644
--- a/core/Archive.php
+++ b/core/Archive.php
@@ -756,7 +756,7 @@ class Archive implements ArchiveQuery
     {
         $periods = $this->params->getPeriods();
         $periodLabel = reset($periods)->getLabel();
-        
+
         if (Rules::shouldProcessReportsAllPlugins($this->params->getIdSites(), $this->params->getSegment(), $periodLabel)) {
             return self::ARCHIVE_ALL_PLUGINS_FLAG;
         }
@@ -822,7 +822,7 @@ class Archive implements ArchiveQuery
             $this->initializeArchiveIdCache($doneFlag);
 
             $prepareResult = $coreAdminHomeApi->archiveReports(
-                $site->getId(), $period->getLabel(), $periodDateStr, $this->params->getSegment()->getString(),
+                $site->getId(), $period->getLabel(), $periodDateStr, urlencode($this->params->getSegment()->getString()),
                 $plugin, $requestedReport);
 
             if (!empty($prepareResult)
diff --git a/core/CronArchive/QueueConsumer.php b/core/CronArchive/QueueConsumer.php
index ff751abd0d..966e9ea6ca 100644
--- a/core/CronArchive/QueueConsumer.php
+++ b/core/CronArchive/QueueConsumer.php
@@ -575,7 +575,7 @@ class QueueConsumer
         $dateStr = $periodLabel == 'range' ? ($invalidatedArchive['date1'] . ',' . $invalidatedArchive['date2']) : $invalidatedArchive['date1'];
         $period = PeriodFactory::build($periodLabel, $dateStr);
 
-        $segment = new Segment($invalidatedArchive['segment'], [$invalidatedArchive['idsite']]);
+        $segment = new Segment(urlencode($invalidatedArchive['segment']), [$invalidatedArchive['idsite']]);
 
         $params = new Parameters($site, $period, $segment);
 
@@ -604,4 +604,4 @@ class QueueConsumer
     {
         return $this->idSite;
     }
-}
\ No newline at end of file
+}
diff --git a/core/Segment.php b/core/Segment.php
index 20761ffa17..2d06cb5199 100644
--- a/core/Segment.php
+++ b/core/Segment.php
@@ -15,6 +15,8 @@ use Piwik\Container\StaticContainer;
 use Piwik\DataAccess\LogQueryBuilder;
 use Piwik\Plugins\SegmentEditor\SegmentEditor;
 use Piwik\Segment\SegmentExpression;
+use Piwik\Plugins\SegmentEditor\Model as SegmentEditorModel;
+use Piwik\Cache;
 
 /**
  * Limits the set of visits Piwik uses when aggregating analytics data.
@@ -100,6 +102,9 @@ class Segment
      */
     const SEGMENT_TRUNCATE_LIMIT = 8192;
 
+    const CACHE_KEY = 'segmenthashes';
+    const SEGMENT_HAS_BUILT_CACHE_KEY ='segmenthashbuilt';
+
     /**
      * Constructor.
      *
@@ -460,8 +465,46 @@ class Segment
 
     public static function getSegmentHash($definition)
     {
-        // urldecode to normalize the string, as browsers may send slightly different payloads for the same archive
-        return md5(urldecode($definition));
+        $cache = Cache::getEagerCache();
+        $cacheKey = self::CACHE_KEY . md5($definition);
+
+        if ($cache->contains($cacheKey)) {
+            return $cache->fetch($cacheKey);
+        }
+
+        $defaultHash = md5(urldecode($definition));
+
+        // if the cache for segments already built, but this segment was not found,
+        // we return the default segment, this can be a segment from url or
+        // something like "visitorType==new"
+        if ($cache->contains(self::SEGMENT_HAS_BUILT_CACHE_KEY)) {
+            return $defaultHash;
+        }
+
+        // the segment hash is not built yet, let's do it
+        $model = new SegmentEditorModel();
+        $segments = $model->getAllSegmentsAndIgnoreVisibility();
+
+        foreach ($segments as $segment) {
+            $cacheKeyTemp = self::CACHE_KEY . md5($segment['definition']);
+            $cache->save($cacheKeyTemp, $segment['hash']);
+
+            $cacheKeyTemp = self::CACHE_KEY . md5(urldecode($segment['definition']));
+            $cache->save($cacheKeyTemp, $segment['hash']);
+
+            $cacheKeyTemp = self::CACHE_KEY . md5(urlencode($segment['definition']));
+            $cache->save($cacheKeyTemp, $segment['hash']);
+        }
+
+        $cache->save(self::SEGMENT_HAS_BUILT_CACHE_KEY, true);
+
+        // if we found the segment, return it's hash, but maybe this
+        // segment is not stored in the db, return the default
+        if ($cache->contains($cacheKey)) {
+            return $cache->fetch($cacheKey);
+        }
+
+        return $defaultHash;
     }
 
     /**
@@ -477,7 +520,7 @@ class Segment
      * @param int $limit Limit number of result to $limit
      * @param int $offset Specified the offset of the first row to return
      * @param bool $forceGroupBy Force the group by and not using a subquery. Note: This may make the query slower see https://github.com/matomo-org/matomo/issues/9200#issuecomment-183641293
-     *                           A $groupBy value needs to be set for this to work. 
+     *                           A $groupBy value needs to be set for this to work.
      * @param int If set to value >= 1 then the Select query (and All inner queries) will be LIMIT'ed by this value.
      *              Use only when you're not aggregating or it will sample the data.
      * @return string The entire select query.
diff --git a/core/Updates/4.3.0-b3.php b/core/Updates/4.3.0-b3.php
new file mode 100644
index 0000000000..92237e25ca
--- /dev/null
+++ b/core/Updates/4.3.0-b3.php
@@ -0,0 +1,53 @@
+<?php
+/**
+ * Matomo - free/libre analytics platform
+ *
+ * @link https://matomo.org
+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
+ *
+ */
+
+namespace Piwik\Updates;
+
+use Piwik\Updater;
+use Piwik\Updates as PiwikUpdates;
+use Piwik\Updater\Migration;
+use Piwik\Updater\Migration\Factory as MigrationFactory;
+use Piwik\Db;
+use Piwik\Common;
+
+/**
+ * Update for version 4.3.0-b3.
+ */
+class Updates_4_3_0_b3 extends PiwikUpdates
+{
+    /**
+     * @var MigrationFactory
+     */
+    private $migration;
+
+    public function __construct(MigrationFactory $factory)
+    {
+        $this->migration = $factory;
+    }
+
+    public function getMigrations(Updater $updater)
+    {
+        $migrations = [];
+
+        $migrations[] = $this->migration->db->addColumn('segment', 'hash', 'CHAR(32) NULL', 'definition');
+
+        $segmentTable = Common::prefixTable('segment');
+        $segments = Db::fetchAll("SELECT idsegment, definition FROM $segmentTable WHERE deleted = ?", [0]);
+        foreach ($segments as $segment) {
+            $hash = md5(urldecode($segment['definition']));
+            $migrations[] = $this->migration->db->sql("UPDATE `$segmentTable` SET `hash` = '$hash' WHERE `idsegment` = '{$segment['idsegment']}'");
+        }
+        return $migrations;
+    }
+
+    public function doUpdate(Updater $updater)
+    {
+        $updater->executeMigrations(__FILE__, $this->getMigrations($updater));
+    }
+}
diff --git a/core/Version.php b/core/Version.php
index dd5d8de953..e38899c779 100644
--- a/core/Version.php
+++ b/core/Version.php
@@ -20,7 +20,7 @@ final class Version
      * The current Matomo version.
      * @var string
      */
-    const VERSION = '4.3.0-b2';
+    const VERSION = '4.3.0-b3';
     const MAJOR_VERSION = 4;
 
     public function isStableVersion($version)