diff options
author | Thomas Steur <thomas.steur@gmail.com> | 2015-03-18 06:40:20 +0300 |
---|---|---|
committer | Thomas Steur <thomas.steur@gmail.com> | 2015-03-18 06:40:20 +0300 |
commit | 6e8ec4575c1b19af63fe4910dd0dd7c5dc8a6003 (patch) | |
tree | 043835e20d15cc352de0ea6cfbc4696aaabc4648 | |
parent | bf56078002db6c9d1bd5653443486b9148af76b0 (diff) |
faster archive by calculating the recursive count only if needed
-rw-r--r-- | core/Archive/DataTableFactory.php | 4 | ||||
-rw-r--r-- | core/ArchiveProcessor.php | 10 | ||||
-rw-r--r-- | core/DataTable/Row.php | 9 | ||||
-rw-r--r-- | plugins/Actions/Archiver.php | 3 | ||||
-rw-r--r-- | plugins/Contents/Archiver.php | 9 | ||||
-rw-r--r-- | plugins/CustomVariables/Archiver.php | 9 | ||||
-rw-r--r-- | plugins/DevicePlugins/Archiver.php | 10 | ||||
-rw-r--r-- | plugins/DevicesDetection/Archiver.php | 8 | ||||
-rw-r--r-- | plugins/Events/Archiver.php | 9 | ||||
-rw-r--r-- | plugins/Goals/Archiver.php | 32 | ||||
-rw-r--r-- | plugins/Provider/Archiver.php | 10 | ||||
-rw-r--r-- | plugins/Referrers/Archiver.php | 10 | ||||
-rw-r--r-- | plugins/Resolution/Archiver.php | 9 | ||||
-rw-r--r-- | plugins/UserCountry/Archiver.php | 10 | ||||
-rw-r--r-- | plugins/UserLanguage/Archiver.php | 9 | ||||
-rw-r--r-- | plugins/VisitTime/Archiver.php | 9 | ||||
-rw-r--r-- | plugins/VisitorInterest/Archiver.php | 9 |
17 files changed, 136 insertions, 33 deletions
diff --git a/core/Archive/DataTableFactory.php b/core/Archive/DataTableFactory.php index 59af8a4e0b..cfc1937813 100644 --- a/core/Archive/DataTableFactory.php +++ b/core/Archive/DataTableFactory.php @@ -326,7 +326,7 @@ class DataTableFactory && $treeLevel >= $this->maxSubtableDepth ) { // unset the subtables so DataTableManager doesn't throw - foreach ($dataTable->getRows() as $row) { + foreach ($dataTable->getRowsWithoutSummaryRow() as $row) { $row->removeSubtable(); } @@ -335,7 +335,7 @@ class DataTableFactory $dataName = reset($this->dataNames); - foreach ($dataTable->getRows() as $row) { + foreach ($dataTable->getRowsWithoutSummaryRow() as $row) { $sid = $row->getIdSubDataTable(); if ($sid === null) { continue; diff --git a/core/ArchiveProcessor.php b/core/ArchiveProcessor.php index 81ac213344..8f7b979915 100644 --- a/core/ArchiveProcessor.php +++ b/core/ArchiveProcessor.php @@ -187,6 +187,8 @@ class ArchiveProcessor * @param array $columnsToRenameAfterAggregation Columns mapped to new names for columns that must change names * when summed because they cannot be summed, eg, * `array('nb_uniq_visitors' => 'sum_daily_nb_uniq_visitors')`. + * @param bool $countRowsRecursive if set to false, will not calculate the recursive rows count which results in + * faster aggregation * @return array Returns the row counts of each aggregated report before truncation, eg, * * array( @@ -203,7 +205,8 @@ class ArchiveProcessor $maximumRowsInSubDataTable = null, $columnToSortByBeforeTruncation = null, &$columnsAggregationOperation = null, - $columnsToRenameAfterAggregation = null) + $columnsToRenameAfterAggregation = null, + $countRowsRecursive = true) { if (!is_array($recordNames)) { $recordNames = array($recordNames); @@ -216,8 +219,9 @@ class ArchiveProcessor $table = $this->aggregateDataTableRecord($recordName, $columnsAggregationOperation, $columnsToRenameAfterAggregation); $nameToCount[$recordName]['level0'] = $table->getRowsCount(); - - $nameToCount[$recordName]['recursive'] = $table->getRowsCountRecursive(); + if ($countRowsRecursive) { + $nameToCount[$recordName]['recursive'] = $table->getRowsCountRecursive(); + } $blob = $table->getSerialized($maximumRowsInDataTableLevelZero, $maximumRowsInSubDataTable, $columnToSortByBeforeTruncation); Common::destroy($table); diff --git a/core/DataTable/Row.php b/core/DataTable/Row.php index 71dde68031..629f968e9a 100644 --- a/core/DataTable/Row.php +++ b/core/DataTable/Row.php @@ -229,11 +229,6 @@ class Row implements \ArrayAccess, \IteratorAggregate return $this->c[self::METADATA][$name]; } - private function getColumnsRaw() - { - return $this->c[self::COLUMNS]; - } - /** * Returns true if a column having the given name is already registered. The value will not be evaluated, it will * just check whether a column exists independent of its value. @@ -475,12 +470,12 @@ class Row implements \ArrayAccess, \IteratorAggregate */ public function sumRow(Row $rowToSum, $enableCopyMetadata = true, $aggregationOperations = false) { - foreach ($rowToSum->getColumnsRaw() as $columnToSumName => $columnToSumValue) { + foreach ($rowToSum->getColumns() as $columnToSumName => $columnToSumValue) { if (!$this->isSummableColumn($columnToSumName)) { continue; } - $thisColumnValue = $this->getColumn($columnToSumName); + $thisColumnValue = &$this->getColumn($columnToSumName); $operation = 'sum'; if (is_array($aggregationOperations) && isset($aggregationOperations[$columnToSumName])) { diff --git a/plugins/Actions/Archiver.php b/plugins/Actions/Archiver.php index 17fc4f058a..9995b3e2dc 100644 --- a/plugins/Actions/Archiver.php +++ b/plugins/Actions/Archiver.php @@ -497,7 +497,8 @@ class Archiver extends \Piwik\Plugin\Archiver ArchivingHelper::$maximumRowsInSubDataTable, ArchivingHelper::$columnToSortByBeforeTruncation, Metrics::$columnsAggregationOperation, - Metrics::$columnsToRenameAfterAggregation + Metrics::$columnsToRenameAfterAggregation, + $countRowsRecursive = false ); $dataTableToSum = array( diff --git a/plugins/Contents/Archiver.php b/plugins/Contents/Archiver.php index 6d3a2e05ce..9cd6fbfc49 100644 --- a/plugins/Contents/Archiver.php +++ b/plugins/Contents/Archiver.php @@ -48,7 +48,14 @@ class Archiver extends \Piwik\Plugin\Archiver public function aggregateMultipleReports() { $dataTableToSum = $this->getRecordNames(); - $this->getProcessor()->aggregateDataTableRecords($dataTableToSum, $this->maximumRowsInDataTable, $this->maximumRowsInSubDataTable, $this->columnToSortByBeforeTruncation); + $this->getProcessor()->aggregateDataTableRecords( + $dataTableToSum, + $this->maximumRowsInDataTable, + $this->maximumRowsInSubDataTable, + $this->columnToSortByBeforeTruncation, + $columnsAggregationOperation = null, + $columnsToRenameAfterAggregation = null, + $countRowsRecursive = false); } private function getRecordNames() diff --git a/plugins/CustomVariables/Archiver.php b/plugins/CustomVariables/Archiver.php index 704038b0a1..f7b9c9ddff 100644 --- a/plugins/CustomVariables/Archiver.php +++ b/plugins/CustomVariables/Archiver.php @@ -51,8 +51,13 @@ class Archiver extends \Piwik\Plugin\Archiver public function aggregateMultipleReports() { $this->getProcessor()->aggregateDataTableRecords( - self::CUSTOM_VARIABLE_RECORD_NAME, $this->maximumRowsInDataTableLevelZero, $this->maximumRowsInSubDataTable, - $columnToSort = Metrics::INDEX_NB_VISITS); + self::CUSTOM_VARIABLE_RECORD_NAME, + $this->maximumRowsInDataTableLevelZero, + $this->maximumRowsInSubDataTable, + $columnToSort = Metrics::INDEX_NB_VISITS, + $columnsAggregationOperation = null, + $columnsToRenameAfterAggregation = null, + $countRowsRecursive = false); } public function aggregateDayReport() diff --git a/plugins/DevicePlugins/Archiver.php b/plugins/DevicePlugins/Archiver.php index 1b4b92213e..798efec2e0 100644 --- a/plugins/DevicePlugins/Archiver.php +++ b/plugins/DevicePlugins/Archiver.php @@ -40,7 +40,15 @@ class Archiver extends \Piwik\Plugin\Archiver $dataTableRecords = array( self::PLUGIN_RECORD_NAME, ); - $this->getProcessor()->aggregateDataTableRecords($dataTableRecords, $this->maximumRows); + $this->getProcessor()->aggregateDataTableRecords( + $dataTableRecords, + $this->maximumRows, + $maximumRowsInSubDataTable = null, + $columnToSortByBeforeTruncation = null, + $columnsAggregationOperation = null, + $columnsToRenameAfterAggregation = null, + $countRowsRecursive = false + ); } protected function aggregateByPlugin() diff --git a/plugins/DevicesDetection/Archiver.php b/plugins/DevicesDetection/Archiver.php index 3ecb310028..e19a064172 100644 --- a/plugins/DevicesDetection/Archiver.php +++ b/plugins/DevicesDetection/Archiver.php @@ -58,7 +58,13 @@ class Archiver extends \Piwik\Plugin\Archiver ); foreach ($dataTablesToSum as $dt) { $this->getProcessor()->aggregateDataTableRecords( - $dt, $this->maximumRows, $this->maximumRows, $columnToSort = "nb_visits"); + $dt, + $this->maximumRows, + $this->maximumRows, + $columnToSort = 'nb_visits', + $columnsAggregationOperation = null, + $columnsToRenameAfterAggregation = null, + $countRowsRecursive = false); } } diff --git a/plugins/Events/Archiver.php b/plugins/Events/Archiver.php index 5a5160bdb2..256c49491e 100644 --- a/plugins/Events/Archiver.php +++ b/plugins/Events/Archiver.php @@ -98,7 +98,14 @@ class Archiver extends \Piwik\Plugin\Archiver public function aggregateMultipleReports() { $dataTableToSum = $this->getRecordNames(); - $this->getProcessor()->aggregateDataTableRecords($dataTableToSum, $this->maximumRowsInDataTable, $this->maximumRowsInSubDataTable, $this->columnToSortByBeforeTruncation); + $this->getProcessor()->aggregateDataTableRecords( + $dataTableToSum, + $this->maximumRowsInDataTable, + $this->maximumRowsInSubDataTable, + $this->columnToSortByBeforeTruncation, + $columnsAggregationOperation = null, + $columnsToRenameAfterAggregation = null, + $countRowsRecursive = false); } protected function getRecordNames() diff --git a/plugins/Goals/Archiver.php b/plugins/Goals/Archiver.php index 379752a6e2..447ae5a267 100644 --- a/plugins/Goals/Archiver.php +++ b/plugins/Goals/Archiver.php @@ -361,7 +361,13 @@ class Archiver extends \Piwik\Plugin\Archiver foreach ($this->dimensionRecord as $recordName) { $dataTableToSum[] = self::getItemRecordNameAbandonedCart($recordName); } - $this->getProcessor()->aggregateDataTableRecords($dataTableToSum); + $this->getProcessor()->aggregateDataTableRecords($dataTableToSum, + $maximumRowsInDataTableLevelZero = null, + $maximumRowsInSubDataTable = null, + $columnToSortByBeforeTruncation = null, + $columnsAggregationOperation = null, + $columnsToRenameAfterAggregation = null, + $countRowsRecursive = false); /* * Archive General Goal metrics @@ -385,14 +391,26 @@ class Archiver extends \Piwik\Plugin\Archiver foreach ($goalIdsToSum as $goalId) { // sum up the visits to conversion data table & the days to conversion data table - $this->getProcessor()->aggregateDataTableRecords(array( - self::getRecordName(self::VISITS_UNTIL_RECORD_NAME, $goalId), - self::getRecordName(self::DAYS_UNTIL_CONV_RECORD_NAME, $goalId))); + $this->getProcessor()->aggregateDataTableRecords( + array(self::getRecordName(self::VISITS_UNTIL_RECORD_NAME, $goalId), + self::getRecordName(self::DAYS_UNTIL_CONV_RECORD_NAME, $goalId)), + $maximumRowsInDataTableLevelZero = null, + $maximumRowsInSubDataTable = null, + $columnToSortByBeforeTruncation = null, + $columnsAggregationOperation = null, + $columnsToRenameAfterAggregation = null, + $countRowsRecursive = false); } // sum up goal overview reports - $this->getProcessor()->aggregateDataTableRecords(array( - self::getRecordName(self::VISITS_UNTIL_RECORD_NAME), - self::getRecordName(self::DAYS_UNTIL_CONV_RECORD_NAME))); + $this->getProcessor()->aggregateDataTableRecords( + array(self::getRecordName(self::VISITS_UNTIL_RECORD_NAME), + self::getRecordName(self::DAYS_UNTIL_CONV_RECORD_NAME)), + $maximumRowsInDataTableLevelZero = null, + $maximumRowsInSubDataTable = null, + $columnToSortByBeforeTruncation = null, + $columnsAggregationOperation = null, + $columnsToRenameAfterAggregation = null, + $countRowsRecursive = false); } } diff --git a/plugins/Provider/Archiver.php b/plugins/Provider/Archiver.php index a8a22c9527..c152807385 100644 --- a/plugins/Provider/Archiver.php +++ b/plugins/Provider/Archiver.php @@ -24,6 +24,14 @@ class Archiver extends \Piwik\Plugin\Archiver public function aggregateMultipleReports() { - $this->getProcessor()->aggregateDataTableRecords(array(self::PROVIDER_RECORD_NAME), $this->maximumRows); + $this->getProcessor()->aggregateDataTableRecords( + array(self::PROVIDER_RECORD_NAME), + $this->maximumRows, + $maximumRowsInSubDataTable = null, + $columnToSortByBeforeTruncation = null, + $columnsAggregationOperation = null, + $columnsToRenameAfterAggregation = null, + $countRowsRecursive = false + ); } } diff --git a/plugins/Referrers/Archiver.php b/plugins/Referrers/Archiver.php index 3093449a6c..f520aed00e 100644 --- a/plugins/Referrers/Archiver.php +++ b/plugins/Referrers/Archiver.php @@ -217,7 +217,15 @@ class Archiver extends \Piwik\Plugin\Archiver public function aggregateMultipleReports() { $dataTableToSum = $this->getRecordNames(); - $nameToCount = $this->getProcessor()->aggregateDataTableRecords($dataTableToSum, $this->maximumRowsInDataTableLevelZero, $this->maximumRowsInSubDataTable, $this->columnToSortByBeforeTruncation); + $nameToCount = $this->getProcessor()->aggregateDataTableRecords( + $dataTableToSum, + $this->maximumRowsInDataTableLevelZero, + $this->maximumRowsInSubDataTable, + $this->columnToSortByBeforeTruncation, + $columnsAggregationOperation = null, + $columnsToRenameAfterAggregation = null, + $countRowsRecursive = false + ); $mappingFromArchiveName = array( self::METRIC_DISTINCT_SEARCH_ENGINE_RECORD_NAME => diff --git a/plugins/Resolution/Archiver.php b/plugins/Resolution/Archiver.php index f44d744c53..cc1b82e056 100644 --- a/plugins/Resolution/Archiver.php +++ b/plugins/Resolution/Archiver.php @@ -39,7 +39,14 @@ class Archiver extends \Piwik\Plugin\Archiver self::RESOLUTION_RECORD_NAME, self::CONFIGURATION_RECORD_NAME, ); - $this->getProcessor()->aggregateDataTableRecords($dataTableRecords, $this->maximumRows); + $this->getProcessor()->aggregateDataTableRecords( + $dataTableRecords, + $this->maximumRows, + $maximumRowsInSubDataTable = null, + $columnToSortByBeforeTruncation = null, + $columnsAggregationOperation = null, + $columnsToRenameAfterAggregation = null, + $countRowsRecursive = false); } protected function aggregateByConfiguration() diff --git a/plugins/UserCountry/Archiver.php b/plugins/UserCountry/Archiver.php index d2203e2479..dcb96e2c66 100644 --- a/plugins/UserCountry/Archiver.php +++ b/plugins/UserCountry/Archiver.php @@ -62,7 +62,15 @@ class Archiver extends \Piwik\Plugin\Archiver self::CITY_RECORD_NAME, ); - $nameToCount = $this->getProcessor()->aggregateDataTableRecords($dataTableToSum); + $nameToCount = $this->getProcessor()->aggregateDataTableRecords( + $dataTableToSum, + $maximumRowsInDataTableLevelZero = null, + $maximumRowsInSubDataTable = null, + $columnToSortByBeforeTruncation = null, + $columnsAggregationOperation = null, + $columnsToRenameAfterAggregation = null, + $countRowsRecursive = false + ); $this->getProcessor()->insertNumericRecord(self::DISTINCT_COUNTRIES_METRIC, $nameToCount[self::COUNTRY_RECORD_NAME]['level0']); } diff --git a/plugins/UserLanguage/Archiver.php b/plugins/UserLanguage/Archiver.php index 14fd369eec..edfbfd8608 100644 --- a/plugins/UserLanguage/Archiver.php +++ b/plugins/UserLanguage/Archiver.php @@ -46,7 +46,14 @@ class Archiver extends \Piwik\Plugin\Archiver $dataTableRecords = array( self::LANGUAGE_RECORD_NAME, ); - $this->getProcessor()->aggregateDataTableRecords($dataTableRecords, $this->maximumRows); + $this->getProcessor()->aggregateDataTableRecords( + $dataTableRecords, + $this->maximumRows, + $maximumRowsInSubDataTable = null, + $columnToSortByBeforeTruncation = null, + $columnsAggregationOperation = null, + $columnsToRenameAfterAggregation = null, + $countRowsRecursive = false); } protected function aggregateByLanguage() diff --git a/plugins/VisitTime/Archiver.php b/plugins/VisitTime/Archiver.php index e7fd9c7a19..63be1dbecb 100644 --- a/plugins/VisitTime/Archiver.php +++ b/plugins/VisitTime/Archiver.php @@ -30,7 +30,14 @@ class Archiver extends \Piwik\Plugin\Archiver self::LOCAL_TIME_RECORD_NAME, self::SERVER_TIME_RECORD_NAME, ); - $this->getProcessor()->aggregateDataTableRecords($dataTableRecords); + $this->getProcessor()->aggregateDataTableRecords( + $dataTableRecords, + $maximumRowsInDataTableLevelZero = null, + $maximumRowsInSubDataTable = null, + $columnToSortByBeforeTruncation = null, + $columnsAggregationOperation = null, + $columnsToRenameAfterAggregation = null, + $countRowsRecursive = false); } protected function aggregateByServerTime() diff --git a/plugins/VisitorInterest/Archiver.php b/plugins/VisitorInterest/Archiver.php index cba45c6934..67a963ffcf 100644 --- a/plugins/VisitorInterest/Archiver.php +++ b/plugins/VisitorInterest/Archiver.php @@ -128,7 +128,14 @@ class Archiver extends \Piwik\Plugin\Archiver self::VISITS_COUNT_RECORD_NAME, self::DAYS_SINCE_LAST_RECORD_NAME ); - $this->getProcessor()->aggregateDataTableRecords($dataTableRecords); + $this->getProcessor()->aggregateDataTableRecords( + $dataTableRecords, + $maximumRowsInDataTableLevelZero = null, + $maximumRowsInSubDataTable = null, + $columnToSortByBeforeTruncation = null, + $columnsAggregationOperation = null, + $columnsToRenameAfterAggregation = null, + $countRowsRecursive = false); } /** |