Add ability for Archivers to initiate archiving for other plugins & use in Goals (#13105)

* Add required segments for Goals.get to known segments to archive list.

* Add test to ArchiveCronTest.

* Allow archiving dependent archives in Goals Archiver.

* Clean up last commit & use Segment::combine in more places.

* skip dependent processing if same plugin/segment

* Move ArchiveCronTest to CoreConsole plugin since it is rather long running.

* Fixing tests.

* Remove use statements.

* Fix tests dependent on archive tables.

* check w/ urlencoded/decoded segment/condition in Segment::combine().

* Another test fix

* final test fix

4 files changed, 121 insertions, 1 deletions

diff --git a/core/ArchiveProcessor.php b/core/ArchiveProcessor.php
index 65e8cbe86e..2b5547b13b 100644
--- a/core/ArchiveProcessor.php
+++ b/core/ArchiveProcessor.php
@@ -17,6 +17,8 @@ use Piwik\DataAccess\LogAggregator;
 use Piwik\DataTable\Manager;
 use Piwik\DataTable\Map;
 use Piwik\DataTable\Row;
+use Piwik\Segment\SegmentExpression;
+
 /**
  * Used by {@link Piwik\Plugin\Archiver} instances to insert and aggregate archive data.
  *
@@ -584,4 +586,44 @@ class ArchiveProcessor
 
         return $metrics;
     }
+
+    /**
+     * Initiate archiving for a plugin during an ongoing archiving. The plugin can be another
+     * plugin or the same plugin.
+     *
+     * This method should be called during archiving when one plugin uses the report of another
+     * plugin with a segment. It will ensure reports for that segment & plugin will be archived
+     * without initiating archiving for every plugin with that segment (which would be a performance
+     * killer).
+     *
+     * @param string $plugin
+     * @param string $segment
+     */
+    public function processDependentArchive($plugin, $segment)
+    {
+        $params = $this->getParams();
+        if (!$params->isRootArchiveRequest()) { // prevent all recursion
+            return;
+        }
+
+        $idSites = [$params->getSite()->getId()];
+
+        $newSegment = Segment::combine($params->getSegment()->getString(), SegmentExpression::AND_DELIMITER, $segment);
+        if ($newSegment === $segment && $params->getRequestedPlugin() === $plugin) { // being processed now
+            return;
+        }
+
+        $newSegment = new Segment($newSegment, $idSites);
+        if (ArchiveProcessor\Rules::isSegmentPreProcessed($idSites, $newSegment)) {
+            // will be processed anyway
+            return;
+        }
+
+        $parameters = new ArchiveProcessor\Parameters($params->getSite(), $params->getPeriod(), $newSegment);
+        $parameters->onlyArchiveRequestedPlugin();
+        $parameters->setIsRootArchiveRequest(false);
+
+        $archiveLoader = new ArchiveProcessor\Loader($parameters);
+        $archiveLoader->prepareArchive($plugin);
+    }
 }
diff --git a/core/ArchiveProcessor/Parameters.php b/core/ArchiveProcessor/Parameters.php
index fdbfe3e4f0..ce62199a53 100644
--- a/core/ArchiveProcessor/Parameters.php
+++ b/core/ArchiveProcessor/Parameters.php
@@ -46,6 +46,11 @@ class Parameters
     private $onlyArchiveRequestedPlugin = false;
 
     /**
+     * @var bool
+     */
+    private $isRootArchiveRequest = true;
+
+    /**
      * Constructor.
      *
      * @ignore
@@ -233,4 +238,27 @@ class Parameters
             $this->getDateEnd()->getDateEndUTC()
         );
     }
+
+    /**
+     * Returns `true` if these parameters are part of an initial archiving request.
+     * Returns `false` if these parameters are for an archiving request that was initiated
+     * during archiving.
+     *
+     * @return bool
+     */
+    public function isRootArchiveRequest()
+    {
+        return $this->isRootArchiveRequest;
+    }
+
+    /**
+     * Sets whether these parameters are part of the initial archiving request or if they are
+     * for a request that was initiated during archiving.
+     *
+     * @param $isRootArchiveRequest
+     */
+    public function setIsRootArchiveRequest($isRootArchiveRequest)
+    {
+        $this->isRootArchiveRequest = $isRootArchiveRequest;
+    }
 }
diff --git a/core/Segment.php b/core/Segment.php
index 8e3e23cd9a..8fcef9c932 100644
--- a/core/Segment.php
+++ b/core/Segment.php
@@ -350,4 +350,55 @@ class Segment
     {
         return (string) $this->getString();
     }
+
+    /**
+     * Combines this segment with another segment condition, if the segment condition is not already
+     * in the segment.
+     *
+     * The combination is naive in that it does not take order of operations into account.
+     *
+     * @param string $segment
+     * @param string $operator The operator to use. Should be either SegmentExpression::AND_DELIMITER
+     *                         or SegmentExpression::OR_DELIMITER.
+     * @param string $segmentCondition The segment condition to add.
+     * @return string
+     * @throws Exception
+     */
+    public static function combine($segment, $operator, $segmentCondition)
+    {
+        if (empty($segment)) {
+            return $segmentCondition;
+        }
+
+        if (empty($segmentCondition)
+            || self::containsCondition($segment, $operator, $segmentCondition)
+        ) {
+            return $segment;
+        }
+
+        return $segment . $operator . $segmentCondition;
+    }
+
+    private static function containsCondition($segment, $operator, $segmentCondition)
+    {
+        // check when segment/condition are of same encoding
+        return strpos($segment, $operator . $segmentCondition) !== false
+            || strpos($segment, $segmentCondition . $operator) !== false
+
+            // check when both operator & condition are urlencoded in $segment
+            || strpos($segment, urlencode($operator . $segmentCondition)) !== false
+            || strpos($segment, urlencode($segmentCondition . $operator)) !== false
+
+            // check when operator is not urlencoded, but condition is in $segment
+            || strpos($segment, $operator . urlencode($segmentCondition)) !== false
+            || strpos($segment, urlencode($segmentCondition) . $operator) !== false
+
+            // check when segment condition is urlencoded & $segment isn't
+            || strpos($segment, $operator . urldecode($segmentCondition)) !== false
+            || strpos($segment, urldecode($segmentCondition) . $operator) !== false
+
+            || $segment === $segmentCondition
+            || $segment === urlencode($segmentCondition)
+            || $segment === urldecode($segmentCondition);
+    }
 }
diff --git a/core/Segment/SegmentExpression.php b/core/Segment/SegmentExpression.php
index ab037c1a6e..40eea9581f 100644
--- a/core/Segment/SegmentExpression.php
+++ b/core/Segment/SegmentExpression.php
@@ -71,7 +71,6 @@ class SegmentExpression
 
     protected $joins = array();
     protected $valuesBind = array();
-    protected $parsedTree = array();
     protected $tree = array();
     protected $parsedSubExpressions = array();