Merge pull request #7468 from piwik/faster_archiving

Faster archiving by calculating the recursive count only if needed

17 files changed, 161 insertions, 34 deletions

diff --git a/core/Archive/DataTableFactory.php b/core/Archive/DataTableFactory.php
index 59af8a4e0b..cfc1937813 100644
--- a/core/Archive/DataTableFactory.php
+++ b/core/Archive/DataTableFactory.php
@@ -326,7 +326,7 @@ class DataTableFactory
             && $treeLevel >= $this->maxSubtableDepth
         ) {
             // unset the subtables so DataTableManager doesn't throw
-            foreach ($dataTable->getRows() as $row) {
+            foreach ($dataTable->getRowsWithoutSummaryRow() as $row) {
                 $row->removeSubtable();
             }
 
@@ -335,7 +335,7 @@ class DataTableFactory
 
         $dataName = reset($this->dataNames);
 
-        foreach ($dataTable->getRows() as $row) {
+        foreach ($dataTable->getRowsWithoutSummaryRow() as $row) {
             $sid = $row->getIdSubDataTable();
             if ($sid === null) {
                 continue;
diff --git a/core/ArchiveProcessor.php b/core/ArchiveProcessor.php
index 81ac213344..31f79b61a8 100644
--- a/core/ArchiveProcessor.php
+++ b/core/ArchiveProcessor.php
@@ -187,6 +187,9 @@ class ArchiveProcessor
      * @param array $columnsToRenameAfterAggregation Columns mapped to new names for columns that must change names
      *                                               when summed because they cannot be summed, eg,
      *                                               `array('nb_uniq_visitors' => 'sum_daily_nb_uniq_visitors')`.
+     * @param bool|array $countRowsRecursive if set to true, will calculate the recursive rows count for all record names
+     *                                       which makes it slower. If you only need it for some records pass an array of
+     *                                       recordNames that defines for which ones you need a recursive row count.
      * @return array Returns the row counts of each aggregated report before truncation, eg,
      *
      *                   array(
@@ -203,7 +206,8 @@ class ArchiveProcessor
                                               $maximumRowsInSubDataTable = null,
                                               $columnToSortByBeforeTruncation = null,
                                               &$columnsAggregationOperation = null,
-                                              $columnsToRenameAfterAggregation = null)
+                                              $columnsToRenameAfterAggregation = null,
+                                              $countRowsRecursive = true)
     {
         if (!is_array($recordNames)) {
             $recordNames = array($recordNames);
@@ -216,8 +220,9 @@ class ArchiveProcessor
             $table = $this->aggregateDataTableRecord($recordName, $columnsAggregationOperation, $columnsToRenameAfterAggregation);
 
             $nameToCount[$recordName]['level0'] = $table->getRowsCount();
-
-            $nameToCount[$recordName]['recursive'] = $table->getRowsCountRecursive();
+            if ($countRowsRecursive === true || (is_array($countRowsRecursive) && in_array($recordName, $countRowsRecursive))) {
+                $nameToCount[$recordName]['recursive'] = $table->getRowsCountRecursive();
+            }
 
             $blob = $table->getSerialized($maximumRowsInDataTableLevelZero, $maximumRowsInSubDataTable, $columnToSortByBeforeTruncation);
             Common::destroy($table);
diff --git a/core/DataTable/Row.php b/core/DataTable/Row.php
index 71dde68031..047e19b230 100644
--- a/core/DataTable/Row.php
+++ b/core/DataTable/Row.php
@@ -229,11 +229,6 @@ class Row implements \ArrayAccess, \IteratorAggregate
         return $this->c[self::METADATA][$name];
     }
 
-    private function getColumnsRaw()
-    {
-        return $this->c[self::COLUMNS];
-    }
-
     /**
      * Returns true if a column having the given name is already registered. The value will not be evaluated, it will
      * just check whether a column exists independent of its value.
@@ -475,7 +470,7 @@ class Row implements \ArrayAccess, \IteratorAggregate
      */
     public function sumRow(Row $rowToSum, $enableCopyMetadata = true, $aggregationOperations = false)
     {
-        foreach ($rowToSum->getColumnsRaw() as $columnToSumName => $columnToSumValue) {
+        foreach ($rowToSum->getColumns() as $columnToSumName => $columnToSumValue) {
             if (!$this->isSummableColumn($columnToSumName)) {
                 continue;
             }
diff --git a/plugins/Actions/Archiver.php b/plugins/Actions/Archiver.php
index 17fc4f058a..5110130b84 100644
--- a/plugins/Actions/Archiver.php
+++ b/plugins/Actions/Archiver.php
@@ -497,7 +497,8 @@ class Archiver extends \Piwik\Plugin\Archiver
             ArchivingHelper::$maximumRowsInSubDataTable,
             ArchivingHelper::$columnToSortByBeforeTruncation,
             Metrics::$columnsAggregationOperation,
-            Metrics::$columnsToRenameAfterAggregation
+            Metrics::$columnsToRenameAfterAggregation,
+            $countRowsRecursive = array()
         );
 
         $dataTableToSum = array(
@@ -511,7 +512,8 @@ class Archiver extends \Piwik\Plugin\Archiver
             ArchivingHelper::$maximumRowsInSubDataTable,
             ArchivingHelper::$columnToSortByBeforeTruncation,
             $aggregation,
-            Metrics::$columnsToRenameAfterAggregation
+            Metrics::$columnsToRenameAfterAggregation,
+            $countRowsRecursive = array()
         );
 
         $this->getProcessor()->aggregateNumericMetrics($this->getMetricNames());
diff --git a/plugins/Contents/Archiver.php b/plugins/Contents/Archiver.php
index 6d3a2e05ce..ba8df12d79 100644
--- a/plugins/Contents/Archiver.php
+++ b/plugins/Contents/Archiver.php
@@ -48,7 +48,15 @@ class Archiver extends \Piwik\Plugin\Archiver
     public function aggregateMultipleReports()
     {
         $dataTableToSum = $this->getRecordNames();
-        $this->getProcessor()->aggregateDataTableRecords($dataTableToSum, $this->maximumRowsInDataTable, $this->maximumRowsInSubDataTable, $this->columnToSortByBeforeTruncation);
+        $columnsAggregationOperation = null;
+        $this->getProcessor()->aggregateDataTableRecords(
+            $dataTableToSum,
+            $this->maximumRowsInDataTable,
+            $this->maximumRowsInSubDataTable,
+            $this->columnToSortByBeforeTruncation,
+            $columnsAggregationOperation,
+            $columnsToRenameAfterAggregation = null,
+            $countRowsRecursive = array());
     }
 
     private function getRecordNames()
diff --git a/plugins/CustomVariables/Archiver.php b/plugins/CustomVariables/Archiver.php
index 704038b0a1..b613f65d02 100644
--- a/plugins/CustomVariables/Archiver.php
+++ b/plugins/CustomVariables/Archiver.php
@@ -50,9 +50,16 @@ class Archiver extends \Piwik\Plugin\Archiver
 
     public function aggregateMultipleReports()
     {
+        $columnsAggregationOperation = null;
+
         $this->getProcessor()->aggregateDataTableRecords(
-            self::CUSTOM_VARIABLE_RECORD_NAME, $this->maximumRowsInDataTableLevelZero, $this->maximumRowsInSubDataTable,
-            $columnToSort = Metrics::INDEX_NB_VISITS);
+            self::CUSTOM_VARIABLE_RECORD_NAME,
+            $this->maximumRowsInDataTableLevelZero,
+            $this->maximumRowsInSubDataTable,
+            $columnToSort = Metrics::INDEX_NB_VISITS,
+            $columnsAggregationOperation,
+            $columnsToRenameAfterAggregation = null,
+            $countRowsRecursive = array());
     }
 
     public function aggregateDayReport()
diff --git a/plugins/DevicePlugins/Archiver.php b/plugins/DevicePlugins/Archiver.php
index 1b4b92213e..acec8c0cde 100644
--- a/plugins/DevicePlugins/Archiver.php
+++ b/plugins/DevicePlugins/Archiver.php
@@ -40,7 +40,16 @@ class Archiver extends \Piwik\Plugin\Archiver
         $dataTableRecords = array(
             self::PLUGIN_RECORD_NAME,
         );
-        $this->getProcessor()->aggregateDataTableRecords($dataTableRecords, $this->maximumRows);
+        $columnsAggregationOperation = null;
+        $this->getProcessor()->aggregateDataTableRecords(
+            $dataTableRecords,
+            $this->maximumRows,
+            $maximumRowsInSubDataTable = null,
+            $columnToSortByBeforeTruncation = null,
+            $columnsAggregationOperation,
+            $columnsToRenameAfterAggregation = null,
+            $countRowsRecursive = array()
+        );
     }
 
     protected function aggregateByPlugin()
diff --git a/plugins/DevicesDetection/Archiver.php b/plugins/DevicesDetection/Archiver.php
index 3ecb310028..21e4b4cf01 100644
--- a/plugins/DevicesDetection/Archiver.php
+++ b/plugins/DevicesDetection/Archiver.php
@@ -56,9 +56,18 @@ class Archiver extends \Piwik\Plugin\Archiver
             self::BROWSER_ENGINE_RECORD_NAME,
             self::BROWSER_VERSION_RECORD_NAME
         );
+
+        $columnsAggregationOperation = null;
+
         foreach ($dataTablesToSum as $dt) {
             $this->getProcessor()->aggregateDataTableRecords(
-                $dt, $this->maximumRows, $this->maximumRows, $columnToSort = "nb_visits");
+                $dt,
+                $this->maximumRows,
+                $this->maximumRows,
+                $columnToSort = 'nb_visits',
+                $columnsAggregationOperation,
+                $columnsToRenameAfterAggregation = null,
+                $countRowsRecursive = array());
         }
     }
 
diff --git a/plugins/Events/Archiver.php b/plugins/Events/Archiver.php
index 5a5160bdb2..c928ec318e 100644
--- a/plugins/Events/Archiver.php
+++ b/plugins/Events/Archiver.php
@@ -98,7 +98,16 @@ class Archiver extends \Piwik\Plugin\Archiver
     public function aggregateMultipleReports()
     {
         $dataTableToSum = $this->getRecordNames();
-        $this->getProcessor()->aggregateDataTableRecords($dataTableToSum, $this->maximumRowsInDataTable, $this->maximumRowsInSubDataTable, $this->columnToSortByBeforeTruncation);
+        $columnsAggregationOperation = null;
+
+        $this->getProcessor()->aggregateDataTableRecords(
+            $dataTableToSum,
+            $this->maximumRowsInDataTable,
+            $this->maximumRowsInSubDataTable,
+            $this->columnToSortByBeforeTruncation,
+            $columnsAggregationOperation,
+            $columnsToRenameAfterAggregation = null,
+            $countRowsRecursive = array());
     }
 
     protected function getRecordNames()
diff --git a/plugins/Goals/Archiver.php b/plugins/Goals/Archiver.php
index 379752a6e2..b96d9b94b7 100644
--- a/plugins/Goals/Archiver.php
+++ b/plugins/Goals/Archiver.php
@@ -361,7 +361,15 @@ class Archiver extends \Piwik\Plugin\Archiver
         foreach ($this->dimensionRecord as $recordName) {
             $dataTableToSum[] = self::getItemRecordNameAbandonedCart($recordName);
         }
-        $this->getProcessor()->aggregateDataTableRecords($dataTableToSum);
+        $columnsAggregationOperation = null;
+
+        $this->getProcessor()->aggregateDataTableRecords($dataTableToSum,
+            $maximumRowsInDataTableLevelZero = null,
+            $maximumRowsInSubDataTable = null,
+            $columnToSortByBeforeTruncation = null,
+            $columnsAggregationOperation,
+            $columnsToRenameAfterAggregation = null,
+            $countRowsRecursive = array());
 
         /*
          *  Archive General Goal metrics
@@ -383,16 +391,31 @@ class Archiver extends \Piwik\Plugin\Archiver
         }
         $this->getProcessor()->aggregateNumericMetrics($fieldsToSum);
 
+        $columnsAggregationOperation = null;
+
         foreach ($goalIdsToSum as $goalId) {
             // sum up the visits to conversion data table & the days to conversion data table
-            $this->getProcessor()->aggregateDataTableRecords(array(
-                                                                  self::getRecordName(self::VISITS_UNTIL_RECORD_NAME, $goalId),
-                                                                  self::getRecordName(self::DAYS_UNTIL_CONV_RECORD_NAME, $goalId)));
+            $this->getProcessor()->aggregateDataTableRecords(
+                array(self::getRecordName(self::VISITS_UNTIL_RECORD_NAME, $goalId),
+                      self::getRecordName(self::DAYS_UNTIL_CONV_RECORD_NAME, $goalId)),
+                $maximumRowsInDataTableLevelZero = null,
+                $maximumRowsInSubDataTable = null,
+                $columnToSortByBeforeTruncation = null,
+                $columnsAggregationOperation,
+                $columnsToRenameAfterAggregation = null,
+                $countRowsRecursive = array());
         }
 
+        $columnsAggregationOperation = null;
         // sum up goal overview reports
-        $this->getProcessor()->aggregateDataTableRecords(array(
-                                                              self::getRecordName(self::VISITS_UNTIL_RECORD_NAME),
-                                                              self::getRecordName(self::DAYS_UNTIL_CONV_RECORD_NAME)));
+        $this->getProcessor()->aggregateDataTableRecords(
+                array(self::getRecordName(self::VISITS_UNTIL_RECORD_NAME),
+                      self::getRecordName(self::DAYS_UNTIL_CONV_RECORD_NAME)),
+                $maximumRowsInDataTableLevelZero = null,
+                $maximumRowsInSubDataTable = null,
+                $columnToSortByBeforeTruncation = null,
+                $columnsAggregationOperation,
+                $columnsToRenameAfterAggregation = null,
+                $countRowsRecursive = array());
     }
 }
diff --git a/plugins/Provider/Archiver.php b/plugins/Provider/Archiver.php
index a8a22c9527..ed140b05e7 100644
--- a/plugins/Provider/Archiver.php
+++ b/plugins/Provider/Archiver.php
@@ -24,6 +24,16 @@ class Archiver extends \Piwik\Plugin\Archiver
 
     public function aggregateMultipleReports()
     {
-        $this->getProcessor()->aggregateDataTableRecords(array(self::PROVIDER_RECORD_NAME), $this->maximumRows);
+        $columnsAggregationOperation = null;
+
+        $this->getProcessor()->aggregateDataTableRecords(
+            array(self::PROVIDER_RECORD_NAME),
+            $this->maximumRows,
+            $maximumRowsInSubDataTable = null,
+            $columnToSortByBeforeTruncation = null,
+            $columnsAggregationOperation,
+            $columnsToRenameAfterAggregation = null,
+            $countRowsRecursive = array()
+        );
     }
 }
diff --git a/plugins/Referrers/Archiver.php b/plugins/Referrers/Archiver.php
index 3093449a6c..ea4939f138 100644
--- a/plugins/Referrers/Archiver.php
+++ b/plugins/Referrers/Archiver.php
@@ -217,7 +217,16 @@ class Archiver extends \Piwik\Plugin\Archiver
     public function aggregateMultipleReports()
     {
         $dataTableToSum = $this->getRecordNames();
-        $nameToCount = $this->getProcessor()->aggregateDataTableRecords($dataTableToSum, $this->maximumRowsInDataTableLevelZero, $this->maximumRowsInSubDataTable, $this->columnToSortByBeforeTruncation);
+        $columnsAggregationOperation = null;
+        $nameToCount = $this->getProcessor()->aggregateDataTableRecords(
+            $dataTableToSum,
+            $this->maximumRowsInDataTableLevelZero,
+            $this->maximumRowsInSubDataTable,
+            $this->columnToSortByBeforeTruncation,
+            $columnsAggregationOperation,
+            $columnsToRenameAfterAggregation = null,
+            $countRowsRecursive = array(self::WEBSITES_RECORD_NAME)
+        );
 
         $mappingFromArchiveName = array(
             self::METRIC_DISTINCT_SEARCH_ENGINE_RECORD_NAME =>
diff --git a/plugins/Resolution/Archiver.php b/plugins/Resolution/Archiver.php
index f44d744c53..06937c24f3 100644
--- a/plugins/Resolution/Archiver.php
+++ b/plugins/Resolution/Archiver.php
@@ -39,7 +39,15 @@ class Archiver extends \Piwik\Plugin\Archiver
             self::RESOLUTION_RECORD_NAME,
             self::CONFIGURATION_RECORD_NAME,
         );
-        $this->getProcessor()->aggregateDataTableRecords($dataTableRecords, $this->maximumRows);
+        $columnsAggregationOperation = null;
+        $this->getProcessor()->aggregateDataTableRecords(
+            $dataTableRecords,
+            $this->maximumRows,
+            $maximumRowsInSubDataTable = null,
+            $columnToSortByBeforeTruncation = null,
+            $columnsAggregationOperation,
+            $columnsToRenameAfterAggregation = null,
+            $countRowsRecursive = array());
     }
 
     protected function aggregateByConfiguration()
diff --git a/plugins/UserCountry/Archiver.php b/plugins/UserCountry/Archiver.php
index d2203e2479..03a789b14c 100644
--- a/plugins/UserCountry/Archiver.php
+++ b/plugins/UserCountry/Archiver.php
@@ -61,8 +61,17 @@ class Archiver extends \Piwik\Plugin\Archiver
             self::REGION_RECORD_NAME,
             self::CITY_RECORD_NAME,
         );
-
-        $nameToCount = $this->getProcessor()->aggregateDataTableRecords($dataTableToSum);
+        $columnsAggregationOperation = null;
+
+        $nameToCount = $this->getProcessor()->aggregateDataTableRecords(
+            $dataTableToSum,
+            $maximumRowsInDataTableLevelZero = null,
+            $maximumRowsInSubDataTable = null,
+            $columnToSortByBeforeTruncation = null,
+            $columnsAggregationOperation,
+            $columnsToRenameAfterAggregation = null,
+            $countRowsRecursive = array()
+        );
         $this->getProcessor()->insertNumericRecord(self::DISTINCT_COUNTRIES_METRIC,
             $nameToCount[self::COUNTRY_RECORD_NAME]['level0']);
     }
diff --git a/plugins/UserLanguage/Archiver.php b/plugins/UserLanguage/Archiver.php
index 14fd369eec..161630b7f2 100644
--- a/plugins/UserLanguage/Archiver.php
+++ b/plugins/UserLanguage/Archiver.php
@@ -46,7 +46,15 @@ class Archiver extends \Piwik\Plugin\Archiver
         $dataTableRecords = array(
             self::LANGUAGE_RECORD_NAME,
         );
-        $this->getProcessor()->aggregateDataTableRecords($dataTableRecords, $this->maximumRows);
+        $columnsAggregationOperation = null;
+        $this->getProcessor()->aggregateDataTableRecords(
+            $dataTableRecords,
+            $this->maximumRows,
+            $maximumRowsInSubDataTable = null,
+            $columnToSortByBeforeTruncation = null,
+            $columnsAggregationOperation,
+            $columnsToRenameAfterAggregation = null,
+            $countRowsRecursive = array());
     }
 
     protected function aggregateByLanguage()
diff --git a/plugins/VisitTime/Archiver.php b/plugins/VisitTime/Archiver.php
index e7fd9c7a19..30bb3dd869 100644
--- a/plugins/VisitTime/Archiver.php
+++ b/plugins/VisitTime/Archiver.php
@@ -30,7 +30,15 @@ class Archiver extends \Piwik\Plugin\Archiver
             self::LOCAL_TIME_RECORD_NAME,
             self::SERVER_TIME_RECORD_NAME,
         );
-        $this->getProcessor()->aggregateDataTableRecords($dataTableRecords);
+        $columnsAggregationOperation = null;
+        $this->getProcessor()->aggregateDataTableRecords(
+            $dataTableRecords,
+            $maximumRowsInDataTableLevelZero = null,
+            $maximumRowsInSubDataTable = null,
+            $columnToSortByBeforeTruncation = null,
+            $columnsAggregationOperation,
+            $columnsToRenameAfterAggregation = null,
+            $countRowsRecursive = array());
     }
 
     protected function aggregateByServerTime()
diff --git a/plugins/VisitorInterest/Archiver.php b/plugins/VisitorInterest/Archiver.php
index cba45c6934..8d432a70b2 100644
--- a/plugins/VisitorInterest/Archiver.php
+++ b/plugins/VisitorInterest/Archiver.php
@@ -128,7 +128,15 @@ class Archiver extends \Piwik\Plugin\Archiver
             self::VISITS_COUNT_RECORD_NAME,
             self::DAYS_SINCE_LAST_RECORD_NAME
         );
-        $this->getProcessor()->aggregateDataTableRecords($dataTableRecords);
+        $columnsAggregationOperation = null;
+        $this->getProcessor()->aggregateDataTableRecords(
+            $dataTableRecords,
+            $maximumRowsInDataTableLevelZero = null,
+            $maximumRowsInSubDataTable = null,
+            $columnToSortByBeforeTruncation = null,
+            $columnsAggregationOperation,
+            $columnsToRenameAfterAggregation = null,
+            $countRowsRecursive = array());
     }
 
     /**