faster archive by calculating the recursive count only if needed

17 files changed, 136 insertions, 33 deletions

diff --git a/core/Archive/DataTableFactory.php b/core/Archive/DataTableFactory.php
index 59af8a4e0b..cfc1937813 100644
--- a/core/Archive/DataTableFactory.php
+++ b/core/Archive/DataTableFactory.php
@@ -326,7 +326,7 @@ class DataTableFactory
             && $treeLevel >= $this->maxSubtableDepth
         ) {
             // unset the subtables so DataTableManager doesn't throw
-            foreach ($dataTable->getRows() as $row) {
+            foreach ($dataTable->getRowsWithoutSummaryRow() as $row) {
                 $row->removeSubtable();
             }
 
@@ -335,7 +335,7 @@ class DataTableFactory
 
         $dataName = reset($this->dataNames);
 
-        foreach ($dataTable->getRows() as $row) {
+        foreach ($dataTable->getRowsWithoutSummaryRow() as $row) {
             $sid = $row->getIdSubDataTable();
             if ($sid === null) {
                 continue;
diff --git a/core/ArchiveProcessor.php b/core/ArchiveProcessor.php
index 81ac213344..8f7b979915 100644
--- a/core/ArchiveProcessor.php
+++ b/core/ArchiveProcessor.php
@@ -187,6 +187,8 @@ class ArchiveProcessor
      * @param array $columnsToRenameAfterAggregation Columns mapped to new names for columns that must change names
      *                                               when summed because they cannot be summed, eg,
      *                                               `array('nb_uniq_visitors' => 'sum_daily_nb_uniq_visitors')`.
+     * @param bool $countRowsRecursive if set to false, will not calculate the recursive rows count which results in
+     *                                 faster aggregation
      * @return array Returns the row counts of each aggregated report before truncation, eg,
      *
      *                   array(
@@ -203,7 +205,8 @@ class ArchiveProcessor
                                               $maximumRowsInSubDataTable = null,
                                               $columnToSortByBeforeTruncation = null,
                                               &$columnsAggregationOperation = null,
-                                              $columnsToRenameAfterAggregation = null)
+                                              $columnsToRenameAfterAggregation = null,
+                                              $countRowsRecursive = true)
     {
         if (!is_array($recordNames)) {
             $recordNames = array($recordNames);
@@ -216,8 +219,9 @@ class ArchiveProcessor
             $table = $this->aggregateDataTableRecord($recordName, $columnsAggregationOperation, $columnsToRenameAfterAggregation);
 
             $nameToCount[$recordName]['level0'] = $table->getRowsCount();
-
-            $nameToCount[$recordName]['recursive'] = $table->getRowsCountRecursive();
+            if ($countRowsRecursive) {
+                $nameToCount[$recordName]['recursive'] = $table->getRowsCountRecursive();
+            }
 
             $blob = $table->getSerialized($maximumRowsInDataTableLevelZero, $maximumRowsInSubDataTable, $columnToSortByBeforeTruncation);
             Common::destroy($table);
diff --git a/core/DataTable/Row.php b/core/DataTable/Row.php
index 71dde68031..629f968e9a 100644
--- a/core/DataTable/Row.php
+++ b/core/DataTable/Row.php
@@ -229,11 +229,6 @@ class Row implements \ArrayAccess, \IteratorAggregate
         return $this->c[self::METADATA][$name];
     }
 
-    private function getColumnsRaw()
-    {
-        return $this->c[self::COLUMNS];
-    }
-
     /**
      * Returns true if a column having the given name is already registered. The value will not be evaluated, it will
      * just check whether a column exists independent of its value.
@@ -475,12 +470,12 @@ class Row implements \ArrayAccess, \IteratorAggregate
      */
     public function sumRow(Row $rowToSum, $enableCopyMetadata = true, $aggregationOperations = false)
     {
-        foreach ($rowToSum->getColumnsRaw() as $columnToSumName => $columnToSumValue) {
+        foreach ($rowToSum->getColumns() as $columnToSumName => $columnToSumValue) {
             if (!$this->isSummableColumn($columnToSumName)) {
                 continue;
             }
 
-            $thisColumnValue = $this->getColumn($columnToSumName);
+            $thisColumnValue = &$this->getColumn($columnToSumName);
 
             $operation = 'sum';
             if (is_array($aggregationOperations) && isset($aggregationOperations[$columnToSumName])) {
diff --git a/plugins/Actions/Archiver.php b/plugins/Actions/Archiver.php
index 17fc4f058a..9995b3e2dc 100644
--- a/plugins/Actions/Archiver.php
+++ b/plugins/Actions/Archiver.php
@@ -497,7 +497,8 @@ class Archiver extends \Piwik\Plugin\Archiver
             ArchivingHelper::$maximumRowsInSubDataTable,
             ArchivingHelper::$columnToSortByBeforeTruncation,
             Metrics::$columnsAggregationOperation,
-            Metrics::$columnsToRenameAfterAggregation
+            Metrics::$columnsToRenameAfterAggregation,
+            $countRowsRecursive = false
         );
 
         $dataTableToSum = array(
diff --git a/plugins/Contents/Archiver.php b/plugins/Contents/Archiver.php
index 6d3a2e05ce..9cd6fbfc49 100644
--- a/plugins/Contents/Archiver.php
+++ b/plugins/Contents/Archiver.php
@@ -48,7 +48,14 @@ class Archiver extends \Piwik\Plugin\Archiver
     public function aggregateMultipleReports()
     {
         $dataTableToSum = $this->getRecordNames();
-        $this->getProcessor()->aggregateDataTableRecords($dataTableToSum, $this->maximumRowsInDataTable, $this->maximumRowsInSubDataTable, $this->columnToSortByBeforeTruncation);
+        $this->getProcessor()->aggregateDataTableRecords(
+            $dataTableToSum,
+            $this->maximumRowsInDataTable,
+            $this->maximumRowsInSubDataTable,
+            $this->columnToSortByBeforeTruncation,
+            $columnsAggregationOperation = null,
+            $columnsToRenameAfterAggregation = null,
+            $countRowsRecursive = false);
     }
 
     private function getRecordNames()
diff --git a/plugins/CustomVariables/Archiver.php b/plugins/CustomVariables/Archiver.php
index 704038b0a1..f7b9c9ddff 100644
--- a/plugins/CustomVariables/Archiver.php
+++ b/plugins/CustomVariables/Archiver.php
@@ -51,8 +51,13 @@ class Archiver extends \Piwik\Plugin\Archiver
     public function aggregateMultipleReports()
     {
         $this->getProcessor()->aggregateDataTableRecords(
-            self::CUSTOM_VARIABLE_RECORD_NAME, $this->maximumRowsInDataTableLevelZero, $this->maximumRowsInSubDataTable,
-            $columnToSort = Metrics::INDEX_NB_VISITS);
+            self::CUSTOM_VARIABLE_RECORD_NAME,
+            $this->maximumRowsInDataTableLevelZero,
+            $this->maximumRowsInSubDataTable,
+            $columnToSort = Metrics::INDEX_NB_VISITS,
+            $columnsAggregationOperation = null,
+            $columnsToRenameAfterAggregation = null,
+            $countRowsRecursive = false);
     }
 
     public function aggregateDayReport()
diff --git a/plugins/DevicePlugins/Archiver.php b/plugins/DevicePlugins/Archiver.php
index 1b4b92213e..798efec2e0 100644
--- a/plugins/DevicePlugins/Archiver.php
+++ b/plugins/DevicePlugins/Archiver.php
@@ -40,7 +40,15 @@ class Archiver extends \Piwik\Plugin\Archiver
         $dataTableRecords = array(
             self::PLUGIN_RECORD_NAME,
         );
-        $this->getProcessor()->aggregateDataTableRecords($dataTableRecords, $this->maximumRows);
+        $this->getProcessor()->aggregateDataTableRecords(
+            $dataTableRecords,
+            $this->maximumRows,
+            $maximumRowsInSubDataTable = null,
+            $columnToSortByBeforeTruncation = null,
+            $columnsAggregationOperation = null,
+            $columnsToRenameAfterAggregation = null,
+            $countRowsRecursive = false
+        );
     }
 
     protected function aggregateByPlugin()
diff --git a/plugins/DevicesDetection/Archiver.php b/plugins/DevicesDetection/Archiver.php
index 3ecb310028..e19a064172 100644
--- a/plugins/DevicesDetection/Archiver.php
+++ b/plugins/DevicesDetection/Archiver.php
@@ -58,7 +58,13 @@ class Archiver extends \Piwik\Plugin\Archiver
         );
         foreach ($dataTablesToSum as $dt) {
             $this->getProcessor()->aggregateDataTableRecords(
-                $dt, $this->maximumRows, $this->maximumRows, $columnToSort = "nb_visits");
+                $dt,
+                $this->maximumRows,
+                $this->maximumRows,
+                $columnToSort = 'nb_visits',
+                $columnsAggregationOperation = null,
+                $columnsToRenameAfterAggregation = null,
+                $countRowsRecursive = false);
         }
     }
 
diff --git a/plugins/Events/Archiver.php b/plugins/Events/Archiver.php
index 5a5160bdb2..256c49491e 100644
--- a/plugins/Events/Archiver.php
+++ b/plugins/Events/Archiver.php
@@ -98,7 +98,14 @@ class Archiver extends \Piwik\Plugin\Archiver
     public function aggregateMultipleReports()
     {
         $dataTableToSum = $this->getRecordNames();
-        $this->getProcessor()->aggregateDataTableRecords($dataTableToSum, $this->maximumRowsInDataTable, $this->maximumRowsInSubDataTable, $this->columnToSortByBeforeTruncation);
+        $this->getProcessor()->aggregateDataTableRecords(
+            $dataTableToSum,
+            $this->maximumRowsInDataTable,
+            $this->maximumRowsInSubDataTable,
+            $this->columnToSortByBeforeTruncation,
+            $columnsAggregationOperation = null,
+            $columnsToRenameAfterAggregation = null,
+            $countRowsRecursive = false);
     }
 
     protected function getRecordNames()
diff --git a/plugins/Goals/Archiver.php b/plugins/Goals/Archiver.php
index 379752a6e2..447ae5a267 100644
--- a/plugins/Goals/Archiver.php
+++ b/plugins/Goals/Archiver.php
@@ -361,7 +361,13 @@ class Archiver extends \Piwik\Plugin\Archiver
         foreach ($this->dimensionRecord as $recordName) {
             $dataTableToSum[] = self::getItemRecordNameAbandonedCart($recordName);
         }
-        $this->getProcessor()->aggregateDataTableRecords($dataTableToSum);
+        $this->getProcessor()->aggregateDataTableRecords($dataTableToSum,
+            $maximumRowsInDataTableLevelZero = null,
+            $maximumRowsInSubDataTable = null,
+            $columnToSortByBeforeTruncation = null,
+            $columnsAggregationOperation = null,
+            $columnsToRenameAfterAggregation = null,
+            $countRowsRecursive = false);
 
         /*
          *  Archive General Goal metrics
@@ -385,14 +391,26 @@ class Archiver extends \Piwik\Plugin\Archiver
 
         foreach ($goalIdsToSum as $goalId) {
             // sum up the visits to conversion data table & the days to conversion data table
-            $this->getProcessor()->aggregateDataTableRecords(array(
-                                                                  self::getRecordName(self::VISITS_UNTIL_RECORD_NAME, $goalId),
-                                                                  self::getRecordName(self::DAYS_UNTIL_CONV_RECORD_NAME, $goalId)));
+            $this->getProcessor()->aggregateDataTableRecords(
+                array(self::getRecordName(self::VISITS_UNTIL_RECORD_NAME, $goalId),
+                      self::getRecordName(self::DAYS_UNTIL_CONV_RECORD_NAME, $goalId)),
+                $maximumRowsInDataTableLevelZero = null,
+                $maximumRowsInSubDataTable = null,
+                $columnToSortByBeforeTruncation = null,
+                $columnsAggregationOperation = null,
+                $columnsToRenameAfterAggregation = null,
+                $countRowsRecursive = false);
         }
 
         // sum up goal overview reports
-        $this->getProcessor()->aggregateDataTableRecords(array(
-                                                              self::getRecordName(self::VISITS_UNTIL_RECORD_NAME),
-                                                              self::getRecordName(self::DAYS_UNTIL_CONV_RECORD_NAME)));
+        $this->getProcessor()->aggregateDataTableRecords(
+                array(self::getRecordName(self::VISITS_UNTIL_RECORD_NAME),
+                      self::getRecordName(self::DAYS_UNTIL_CONV_RECORD_NAME)),
+                $maximumRowsInDataTableLevelZero = null,
+                $maximumRowsInSubDataTable = null,
+                $columnToSortByBeforeTruncation = null,
+                $columnsAggregationOperation = null,
+                $columnsToRenameAfterAggregation = null,
+                $countRowsRecursive = false);
     }
 }
diff --git a/plugins/Provider/Archiver.php b/plugins/Provider/Archiver.php
index a8a22c9527..c152807385 100644
--- a/plugins/Provider/Archiver.php
+++ b/plugins/Provider/Archiver.php
@@ -24,6 +24,14 @@ class Archiver extends \Piwik\Plugin\Archiver
 
     public function aggregateMultipleReports()
     {
-        $this->getProcessor()->aggregateDataTableRecords(array(self::PROVIDER_RECORD_NAME), $this->maximumRows);
+        $this->getProcessor()->aggregateDataTableRecords(
+            array(self::PROVIDER_RECORD_NAME),
+            $this->maximumRows,
+            $maximumRowsInSubDataTable = null,
+            $columnToSortByBeforeTruncation = null,
+            $columnsAggregationOperation = null,
+            $columnsToRenameAfterAggregation = null,
+            $countRowsRecursive = false
+        );
     }
 }
diff --git a/plugins/Referrers/Archiver.php b/plugins/Referrers/Archiver.php
index 3093449a6c..f520aed00e 100644
--- a/plugins/Referrers/Archiver.php
+++ b/plugins/Referrers/Archiver.php
@@ -217,7 +217,15 @@ class Archiver extends \Piwik\Plugin\Archiver
     public function aggregateMultipleReports()
     {
         $dataTableToSum = $this->getRecordNames();
-        $nameToCount = $this->getProcessor()->aggregateDataTableRecords($dataTableToSum, $this->maximumRowsInDataTableLevelZero, $this->maximumRowsInSubDataTable, $this->columnToSortByBeforeTruncation);
+        $nameToCount = $this->getProcessor()->aggregateDataTableRecords(
+            $dataTableToSum,
+            $this->maximumRowsInDataTableLevelZero,
+            $this->maximumRowsInSubDataTable,
+            $this->columnToSortByBeforeTruncation,
+            $columnsAggregationOperation = null,
+            $columnsToRenameAfterAggregation = null,
+            $countRowsRecursive = false
+        );
 
         $mappingFromArchiveName = array(
             self::METRIC_DISTINCT_SEARCH_ENGINE_RECORD_NAME =>
diff --git a/plugins/Resolution/Archiver.php b/plugins/Resolution/Archiver.php
index f44d744c53..cc1b82e056 100644
--- a/plugins/Resolution/Archiver.php
+++ b/plugins/Resolution/Archiver.php
@@ -39,7 +39,14 @@ class Archiver extends \Piwik\Plugin\Archiver
             self::RESOLUTION_RECORD_NAME,
             self::CONFIGURATION_RECORD_NAME,
         );
-        $this->getProcessor()->aggregateDataTableRecords($dataTableRecords, $this->maximumRows);
+        $this->getProcessor()->aggregateDataTableRecords(
+            $dataTableRecords,
+            $this->maximumRows,
+            $maximumRowsInSubDataTable = null,
+            $columnToSortByBeforeTruncation = null,
+            $columnsAggregationOperation = null,
+            $columnsToRenameAfterAggregation = null,
+            $countRowsRecursive = false);
     }
 
     protected function aggregateByConfiguration()
diff --git a/plugins/UserCountry/Archiver.php b/plugins/UserCountry/Archiver.php
index d2203e2479..dcb96e2c66 100644
--- a/plugins/UserCountry/Archiver.php
+++ b/plugins/UserCountry/Archiver.php
@@ -62,7 +62,15 @@ class Archiver extends \Piwik\Plugin\Archiver
             self::CITY_RECORD_NAME,
         );
 
-        $nameToCount = $this->getProcessor()->aggregateDataTableRecords($dataTableToSum);
+        $nameToCount = $this->getProcessor()->aggregateDataTableRecords(
+            $dataTableToSum,
+            $maximumRowsInDataTableLevelZero = null,
+            $maximumRowsInSubDataTable = null,
+            $columnToSortByBeforeTruncation = null,
+            $columnsAggregationOperation = null,
+            $columnsToRenameAfterAggregation = null,
+            $countRowsRecursive = false
+        );
         $this->getProcessor()->insertNumericRecord(self::DISTINCT_COUNTRIES_METRIC,
             $nameToCount[self::COUNTRY_RECORD_NAME]['level0']);
     }
diff --git a/plugins/UserLanguage/Archiver.php b/plugins/UserLanguage/Archiver.php
index 14fd369eec..edfbfd8608 100644
--- a/plugins/UserLanguage/Archiver.php
+++ b/plugins/UserLanguage/Archiver.php
@@ -46,7 +46,14 @@ class Archiver extends \Piwik\Plugin\Archiver
         $dataTableRecords = array(
             self::LANGUAGE_RECORD_NAME,
         );
-        $this->getProcessor()->aggregateDataTableRecords($dataTableRecords, $this->maximumRows);
+        $this->getProcessor()->aggregateDataTableRecords(
+            $dataTableRecords,
+            $this->maximumRows,
+            $maximumRowsInSubDataTable = null,
+            $columnToSortByBeforeTruncation = null,
+            $columnsAggregationOperation = null,
+            $columnsToRenameAfterAggregation = null,
+            $countRowsRecursive = false);
     }
 
     protected function aggregateByLanguage()
diff --git a/plugins/VisitTime/Archiver.php b/plugins/VisitTime/Archiver.php
index e7fd9c7a19..63be1dbecb 100644
--- a/plugins/VisitTime/Archiver.php
+++ b/plugins/VisitTime/Archiver.php
@@ -30,7 +30,14 @@ class Archiver extends \Piwik\Plugin\Archiver
             self::LOCAL_TIME_RECORD_NAME,
             self::SERVER_TIME_RECORD_NAME,
         );
-        $this->getProcessor()->aggregateDataTableRecords($dataTableRecords);
+        $this->getProcessor()->aggregateDataTableRecords(
+            $dataTableRecords,
+            $maximumRowsInDataTableLevelZero = null,
+            $maximumRowsInSubDataTable = null,
+            $columnToSortByBeforeTruncation = null,
+            $columnsAggregationOperation = null,
+            $columnsToRenameAfterAggregation = null,
+            $countRowsRecursive = false);
     }
 
     protected function aggregateByServerTime()
diff --git a/plugins/VisitorInterest/Archiver.php b/plugins/VisitorInterest/Archiver.php
index cba45c6934..67a963ffcf 100644
--- a/plugins/VisitorInterest/Archiver.php
+++ b/plugins/VisitorInterest/Archiver.php
@@ -128,7 +128,14 @@ class Archiver extends \Piwik\Plugin\Archiver
             self::VISITS_COUNT_RECORD_NAME,
             self::DAYS_SINCE_LAST_RECORD_NAME
         );
-        $this->getProcessor()->aggregateDataTableRecords($dataTableRecords);
+        $this->getProcessor()->aggregateDataTableRecords(
+            $dataTableRecords,
+            $maximumRowsInDataTableLevelZero = null,
+            $maximumRowsInSubDataTable = null,
+            $columnToSortByBeforeTruncation = null,
+            $columnsAggregationOperation = null,
+            $columnsToRenameAfterAggregation = null,
+            $countRowsRecursive = false);
     }
 
     /**