diff options
author | Matthieu Aubry <matt@piwik.org> | 2015-03-18 07:58:22 +0300 |
---|---|---|
committer | Matthieu Aubry <matt@piwik.org> | 2015-03-18 07:58:22 +0300 |
commit | 8dc8976f8185c47b8ef5be7da6dba0c9d3873fb9 (patch) | |
tree | b3bf18044bb44073eca227ac1b7dd4852d1dd7c3 | |
parent | 9cc114308dd5310f48e71d7a2d5c8eab509d470d (diff) | |
parent | d532912e01eca79fa94df09578e59a94fb7c6511 (diff) |
Merge pull request #7468 from piwik/faster_archiving
Faster archiving by calculating the recursive count only if needed
-rw-r--r-- | core/Archive/DataTableFactory.php | 4 | ||||
-rw-r--r-- | core/ArchiveProcessor.php | 11 | ||||
-rw-r--r-- | core/DataTable/Row.php | 7 | ||||
-rw-r--r-- | plugins/Actions/Archiver.php | 6 | ||||
-rw-r--r-- | plugins/Contents/Archiver.php | 10 | ||||
-rw-r--r-- | plugins/CustomVariables/Archiver.php | 11 | ||||
-rw-r--r-- | plugins/DevicePlugins/Archiver.php | 11 | ||||
-rw-r--r-- | plugins/DevicesDetection/Archiver.php | 11 | ||||
-rw-r--r-- | plugins/Events/Archiver.php | 11 | ||||
-rw-r--r-- | plugins/Goals/Archiver.php | 37 | ||||
-rw-r--r-- | plugins/Provider/Archiver.php | 12 | ||||
-rw-r--r-- | plugins/Referrers/Archiver.php | 11 | ||||
-rw-r--r-- | plugins/Resolution/Archiver.php | 10 | ||||
-rw-r--r-- | plugins/UserCountry/Archiver.php | 13 | ||||
-rw-r--r-- | plugins/UserLanguage/Archiver.php | 10 | ||||
-rw-r--r-- | plugins/VisitTime/Archiver.php | 10 | ||||
-rw-r--r-- | plugins/VisitorInterest/Archiver.php | 10 |
17 files changed, 161 insertions, 34 deletions
diff --git a/core/Archive/DataTableFactory.php b/core/Archive/DataTableFactory.php index 59af8a4e0b..cfc1937813 100644 --- a/core/Archive/DataTableFactory.php +++ b/core/Archive/DataTableFactory.php @@ -326,7 +326,7 @@ class DataTableFactory && $treeLevel >= $this->maxSubtableDepth ) { // unset the subtables so DataTableManager doesn't throw - foreach ($dataTable->getRows() as $row) { + foreach ($dataTable->getRowsWithoutSummaryRow() as $row) { $row->removeSubtable(); } @@ -335,7 +335,7 @@ class DataTableFactory $dataName = reset($this->dataNames); - foreach ($dataTable->getRows() as $row) { + foreach ($dataTable->getRowsWithoutSummaryRow() as $row) { $sid = $row->getIdSubDataTable(); if ($sid === null) { continue; diff --git a/core/ArchiveProcessor.php b/core/ArchiveProcessor.php index 81ac213344..31f79b61a8 100644 --- a/core/ArchiveProcessor.php +++ b/core/ArchiveProcessor.php @@ -187,6 +187,9 @@ class ArchiveProcessor * @param array $columnsToRenameAfterAggregation Columns mapped to new names for columns that must change names * when summed because they cannot be summed, eg, * `array('nb_uniq_visitors' => 'sum_daily_nb_uniq_visitors')`. + * @param bool|array $countRowsRecursive if set to true, will calculate the recursive rows count for all record names + * which makes it slower. If you only need it for some records pass an array of + * recordNames that defines for which ones you need a recursive row count. * @return array Returns the row counts of each aggregated report before truncation, eg, * * array( @@ -203,7 +206,8 @@ class ArchiveProcessor $maximumRowsInSubDataTable = null, $columnToSortByBeforeTruncation = null, &$columnsAggregationOperation = null, - $columnsToRenameAfterAggregation = null) + $columnsToRenameAfterAggregation = null, + $countRowsRecursive = true) { if (!is_array($recordNames)) { $recordNames = array($recordNames); @@ -216,8 +220,9 @@ class ArchiveProcessor $table = $this->aggregateDataTableRecord($recordName, $columnsAggregationOperation, $columnsToRenameAfterAggregation); $nameToCount[$recordName]['level0'] = $table->getRowsCount(); - - $nameToCount[$recordName]['recursive'] = $table->getRowsCountRecursive(); + if ($countRowsRecursive === true || (is_array($countRowsRecursive) && in_array($recordName, $countRowsRecursive))) { + $nameToCount[$recordName]['recursive'] = $table->getRowsCountRecursive(); + } $blob = $table->getSerialized($maximumRowsInDataTableLevelZero, $maximumRowsInSubDataTable, $columnToSortByBeforeTruncation); Common::destroy($table); diff --git a/core/DataTable/Row.php b/core/DataTable/Row.php index 71dde68031..047e19b230 100644 --- a/core/DataTable/Row.php +++ b/core/DataTable/Row.php @@ -229,11 +229,6 @@ class Row implements \ArrayAccess, \IteratorAggregate return $this->c[self::METADATA][$name]; } - private function getColumnsRaw() - { - return $this->c[self::COLUMNS]; - } - /** * Returns true if a column having the given name is already registered. The value will not be evaluated, it will * just check whether a column exists independent of its value. @@ -475,7 +470,7 @@ class Row implements \ArrayAccess, \IteratorAggregate */ public function sumRow(Row $rowToSum, $enableCopyMetadata = true, $aggregationOperations = false) { - foreach ($rowToSum->getColumnsRaw() as $columnToSumName => $columnToSumValue) { + foreach ($rowToSum->getColumns() as $columnToSumName => $columnToSumValue) { if (!$this->isSummableColumn($columnToSumName)) { continue; } diff --git a/plugins/Actions/Archiver.php b/plugins/Actions/Archiver.php index 17fc4f058a..5110130b84 100644 --- a/plugins/Actions/Archiver.php +++ b/plugins/Actions/Archiver.php @@ -497,7 +497,8 @@ class Archiver extends \Piwik\Plugin\Archiver ArchivingHelper::$maximumRowsInSubDataTable, ArchivingHelper::$columnToSortByBeforeTruncation, Metrics::$columnsAggregationOperation, - Metrics::$columnsToRenameAfterAggregation + Metrics::$columnsToRenameAfterAggregation, + $countRowsRecursive = array() ); $dataTableToSum = array( @@ -511,7 +512,8 @@ class Archiver extends \Piwik\Plugin\Archiver ArchivingHelper::$maximumRowsInSubDataTable, ArchivingHelper::$columnToSortByBeforeTruncation, $aggregation, - Metrics::$columnsToRenameAfterAggregation + Metrics::$columnsToRenameAfterAggregation, + $countRowsRecursive = array() ); $this->getProcessor()->aggregateNumericMetrics($this->getMetricNames()); diff --git a/plugins/Contents/Archiver.php b/plugins/Contents/Archiver.php index 6d3a2e05ce..ba8df12d79 100644 --- a/plugins/Contents/Archiver.php +++ b/plugins/Contents/Archiver.php @@ -48,7 +48,15 @@ class Archiver extends \Piwik\Plugin\Archiver public function aggregateMultipleReports() { $dataTableToSum = $this->getRecordNames(); - $this->getProcessor()->aggregateDataTableRecords($dataTableToSum, $this->maximumRowsInDataTable, $this->maximumRowsInSubDataTable, $this->columnToSortByBeforeTruncation); + $columnsAggregationOperation = null; + $this->getProcessor()->aggregateDataTableRecords( + $dataTableToSum, + $this->maximumRowsInDataTable, + $this->maximumRowsInSubDataTable, + $this->columnToSortByBeforeTruncation, + $columnsAggregationOperation, + $columnsToRenameAfterAggregation = null, + $countRowsRecursive = array()); } private function getRecordNames() diff --git a/plugins/CustomVariables/Archiver.php b/plugins/CustomVariables/Archiver.php index 704038b0a1..b613f65d02 100644 --- a/plugins/CustomVariables/Archiver.php +++ b/plugins/CustomVariables/Archiver.php @@ -50,9 +50,16 @@ class Archiver extends \Piwik\Plugin\Archiver public function aggregateMultipleReports() { + $columnsAggregationOperation = null; + $this->getProcessor()->aggregateDataTableRecords( - self::CUSTOM_VARIABLE_RECORD_NAME, $this->maximumRowsInDataTableLevelZero, $this->maximumRowsInSubDataTable, - $columnToSort = Metrics::INDEX_NB_VISITS); + self::CUSTOM_VARIABLE_RECORD_NAME, + $this->maximumRowsInDataTableLevelZero, + $this->maximumRowsInSubDataTable, + $columnToSort = Metrics::INDEX_NB_VISITS, + $columnsAggregationOperation, + $columnsToRenameAfterAggregation = null, + $countRowsRecursive = array()); } public function aggregateDayReport() diff --git a/plugins/DevicePlugins/Archiver.php b/plugins/DevicePlugins/Archiver.php index 1b4b92213e..acec8c0cde 100644 --- a/plugins/DevicePlugins/Archiver.php +++ b/plugins/DevicePlugins/Archiver.php @@ -40,7 +40,16 @@ class Archiver extends \Piwik\Plugin\Archiver $dataTableRecords = array( self::PLUGIN_RECORD_NAME, ); - $this->getProcessor()->aggregateDataTableRecords($dataTableRecords, $this->maximumRows); + $columnsAggregationOperation = null; + $this->getProcessor()->aggregateDataTableRecords( + $dataTableRecords, + $this->maximumRows, + $maximumRowsInSubDataTable = null, + $columnToSortByBeforeTruncation = null, + $columnsAggregationOperation, + $columnsToRenameAfterAggregation = null, + $countRowsRecursive = array() + ); } protected function aggregateByPlugin() diff --git a/plugins/DevicesDetection/Archiver.php b/plugins/DevicesDetection/Archiver.php index 3ecb310028..21e4b4cf01 100644 --- a/plugins/DevicesDetection/Archiver.php +++ b/plugins/DevicesDetection/Archiver.php @@ -56,9 +56,18 @@ class Archiver extends \Piwik\Plugin\Archiver self::BROWSER_ENGINE_RECORD_NAME, self::BROWSER_VERSION_RECORD_NAME ); + + $columnsAggregationOperation = null; + foreach ($dataTablesToSum as $dt) { $this->getProcessor()->aggregateDataTableRecords( - $dt, $this->maximumRows, $this->maximumRows, $columnToSort = "nb_visits"); + $dt, + $this->maximumRows, + $this->maximumRows, + $columnToSort = 'nb_visits', + $columnsAggregationOperation, + $columnsToRenameAfterAggregation = null, + $countRowsRecursive = array()); } } diff --git a/plugins/Events/Archiver.php b/plugins/Events/Archiver.php index 5a5160bdb2..c928ec318e 100644 --- a/plugins/Events/Archiver.php +++ b/plugins/Events/Archiver.php @@ -98,7 +98,16 @@ class Archiver extends \Piwik\Plugin\Archiver public function aggregateMultipleReports() { $dataTableToSum = $this->getRecordNames(); - $this->getProcessor()->aggregateDataTableRecords($dataTableToSum, $this->maximumRowsInDataTable, $this->maximumRowsInSubDataTable, $this->columnToSortByBeforeTruncation); + $columnsAggregationOperation = null; + + $this->getProcessor()->aggregateDataTableRecords( + $dataTableToSum, + $this->maximumRowsInDataTable, + $this->maximumRowsInSubDataTable, + $this->columnToSortByBeforeTruncation, + $columnsAggregationOperation, + $columnsToRenameAfterAggregation = null, + $countRowsRecursive = array()); } protected function getRecordNames() diff --git a/plugins/Goals/Archiver.php b/plugins/Goals/Archiver.php index 379752a6e2..b96d9b94b7 100644 --- a/plugins/Goals/Archiver.php +++ b/plugins/Goals/Archiver.php @@ -361,7 +361,15 @@ class Archiver extends \Piwik\Plugin\Archiver foreach ($this->dimensionRecord as $recordName) { $dataTableToSum[] = self::getItemRecordNameAbandonedCart($recordName); } - $this->getProcessor()->aggregateDataTableRecords($dataTableToSum); + $columnsAggregationOperation = null; + + $this->getProcessor()->aggregateDataTableRecords($dataTableToSum, + $maximumRowsInDataTableLevelZero = null, + $maximumRowsInSubDataTable = null, + $columnToSortByBeforeTruncation = null, + $columnsAggregationOperation, + $columnsToRenameAfterAggregation = null, + $countRowsRecursive = array()); /* * Archive General Goal metrics @@ -383,16 +391,31 @@ class Archiver extends \Piwik\Plugin\Archiver } $this->getProcessor()->aggregateNumericMetrics($fieldsToSum); + $columnsAggregationOperation = null; + foreach ($goalIdsToSum as $goalId) { // sum up the visits to conversion data table & the days to conversion data table - $this->getProcessor()->aggregateDataTableRecords(array( - self::getRecordName(self::VISITS_UNTIL_RECORD_NAME, $goalId), - self::getRecordName(self::DAYS_UNTIL_CONV_RECORD_NAME, $goalId))); + $this->getProcessor()->aggregateDataTableRecords( + array(self::getRecordName(self::VISITS_UNTIL_RECORD_NAME, $goalId), + self::getRecordName(self::DAYS_UNTIL_CONV_RECORD_NAME, $goalId)), + $maximumRowsInDataTableLevelZero = null, + $maximumRowsInSubDataTable = null, + $columnToSortByBeforeTruncation = null, + $columnsAggregationOperation, + $columnsToRenameAfterAggregation = null, + $countRowsRecursive = array()); } + $columnsAggregationOperation = null; // sum up goal overview reports - $this->getProcessor()->aggregateDataTableRecords(array( - self::getRecordName(self::VISITS_UNTIL_RECORD_NAME), - self::getRecordName(self::DAYS_UNTIL_CONV_RECORD_NAME))); + $this->getProcessor()->aggregateDataTableRecords( + array(self::getRecordName(self::VISITS_UNTIL_RECORD_NAME), + self::getRecordName(self::DAYS_UNTIL_CONV_RECORD_NAME)), + $maximumRowsInDataTableLevelZero = null, + $maximumRowsInSubDataTable = null, + $columnToSortByBeforeTruncation = null, + $columnsAggregationOperation, + $columnsToRenameAfterAggregation = null, + $countRowsRecursive = array()); } } diff --git a/plugins/Provider/Archiver.php b/plugins/Provider/Archiver.php index a8a22c9527..ed140b05e7 100644 --- a/plugins/Provider/Archiver.php +++ b/plugins/Provider/Archiver.php @@ -24,6 +24,16 @@ class Archiver extends \Piwik\Plugin\Archiver public function aggregateMultipleReports() { - $this->getProcessor()->aggregateDataTableRecords(array(self::PROVIDER_RECORD_NAME), $this->maximumRows); + $columnsAggregationOperation = null; + + $this->getProcessor()->aggregateDataTableRecords( + array(self::PROVIDER_RECORD_NAME), + $this->maximumRows, + $maximumRowsInSubDataTable = null, + $columnToSortByBeforeTruncation = null, + $columnsAggregationOperation, + $columnsToRenameAfterAggregation = null, + $countRowsRecursive = array() + ); } } diff --git a/plugins/Referrers/Archiver.php b/plugins/Referrers/Archiver.php index 3093449a6c..ea4939f138 100644 --- a/plugins/Referrers/Archiver.php +++ b/plugins/Referrers/Archiver.php @@ -217,7 +217,16 @@ class Archiver extends \Piwik\Plugin\Archiver public function aggregateMultipleReports() { $dataTableToSum = $this->getRecordNames(); - $nameToCount = $this->getProcessor()->aggregateDataTableRecords($dataTableToSum, $this->maximumRowsInDataTableLevelZero, $this->maximumRowsInSubDataTable, $this->columnToSortByBeforeTruncation); + $columnsAggregationOperation = null; + $nameToCount = $this->getProcessor()->aggregateDataTableRecords( + $dataTableToSum, + $this->maximumRowsInDataTableLevelZero, + $this->maximumRowsInSubDataTable, + $this->columnToSortByBeforeTruncation, + $columnsAggregationOperation, + $columnsToRenameAfterAggregation = null, + $countRowsRecursive = array(self::WEBSITES_RECORD_NAME) + ); $mappingFromArchiveName = array( self::METRIC_DISTINCT_SEARCH_ENGINE_RECORD_NAME => diff --git a/plugins/Resolution/Archiver.php b/plugins/Resolution/Archiver.php index f44d744c53..06937c24f3 100644 --- a/plugins/Resolution/Archiver.php +++ b/plugins/Resolution/Archiver.php @@ -39,7 +39,15 @@ class Archiver extends \Piwik\Plugin\Archiver self::RESOLUTION_RECORD_NAME, self::CONFIGURATION_RECORD_NAME, ); - $this->getProcessor()->aggregateDataTableRecords($dataTableRecords, $this->maximumRows); + $columnsAggregationOperation = null; + $this->getProcessor()->aggregateDataTableRecords( + $dataTableRecords, + $this->maximumRows, + $maximumRowsInSubDataTable = null, + $columnToSortByBeforeTruncation = null, + $columnsAggregationOperation, + $columnsToRenameAfterAggregation = null, + $countRowsRecursive = array()); } protected function aggregateByConfiguration() diff --git a/plugins/UserCountry/Archiver.php b/plugins/UserCountry/Archiver.php index d2203e2479..03a789b14c 100644 --- a/plugins/UserCountry/Archiver.php +++ b/plugins/UserCountry/Archiver.php @@ -61,8 +61,17 @@ class Archiver extends \Piwik\Plugin\Archiver self::REGION_RECORD_NAME, self::CITY_RECORD_NAME, ); - - $nameToCount = $this->getProcessor()->aggregateDataTableRecords($dataTableToSum); + $columnsAggregationOperation = null; + + $nameToCount = $this->getProcessor()->aggregateDataTableRecords( + $dataTableToSum, + $maximumRowsInDataTableLevelZero = null, + $maximumRowsInSubDataTable = null, + $columnToSortByBeforeTruncation = null, + $columnsAggregationOperation, + $columnsToRenameAfterAggregation = null, + $countRowsRecursive = array() + ); $this->getProcessor()->insertNumericRecord(self::DISTINCT_COUNTRIES_METRIC, $nameToCount[self::COUNTRY_RECORD_NAME]['level0']); } diff --git a/plugins/UserLanguage/Archiver.php b/plugins/UserLanguage/Archiver.php index 14fd369eec..161630b7f2 100644 --- a/plugins/UserLanguage/Archiver.php +++ b/plugins/UserLanguage/Archiver.php @@ -46,7 +46,15 @@ class Archiver extends \Piwik\Plugin\Archiver $dataTableRecords = array( self::LANGUAGE_RECORD_NAME, ); - $this->getProcessor()->aggregateDataTableRecords($dataTableRecords, $this->maximumRows); + $columnsAggregationOperation = null; + $this->getProcessor()->aggregateDataTableRecords( + $dataTableRecords, + $this->maximumRows, + $maximumRowsInSubDataTable = null, + $columnToSortByBeforeTruncation = null, + $columnsAggregationOperation, + $columnsToRenameAfterAggregation = null, + $countRowsRecursive = array()); } protected function aggregateByLanguage() diff --git a/plugins/VisitTime/Archiver.php b/plugins/VisitTime/Archiver.php index e7fd9c7a19..30bb3dd869 100644 --- a/plugins/VisitTime/Archiver.php +++ b/plugins/VisitTime/Archiver.php @@ -30,7 +30,15 @@ class Archiver extends \Piwik\Plugin\Archiver self::LOCAL_TIME_RECORD_NAME, self::SERVER_TIME_RECORD_NAME, ); - $this->getProcessor()->aggregateDataTableRecords($dataTableRecords); + $columnsAggregationOperation = null; + $this->getProcessor()->aggregateDataTableRecords( + $dataTableRecords, + $maximumRowsInDataTableLevelZero = null, + $maximumRowsInSubDataTable = null, + $columnToSortByBeforeTruncation = null, + $columnsAggregationOperation, + $columnsToRenameAfterAggregation = null, + $countRowsRecursive = array()); } protected function aggregateByServerTime() diff --git a/plugins/VisitorInterest/Archiver.php b/plugins/VisitorInterest/Archiver.php index cba45c6934..8d432a70b2 100644 --- a/plugins/VisitorInterest/Archiver.php +++ b/plugins/VisitorInterest/Archiver.php @@ -128,7 +128,15 @@ class Archiver extends \Piwik\Plugin\Archiver self::VISITS_COUNT_RECORD_NAME, self::DAYS_SINCE_LAST_RECORD_NAME ); - $this->getProcessor()->aggregateDataTableRecords($dataTableRecords); + $columnsAggregationOperation = null; + $this->getProcessor()->aggregateDataTableRecords( + $dataTableRecords, + $maximumRowsInDataTableLevelZero = null, + $maximumRowsInSubDataTable = null, + $columnToSortByBeforeTruncation = null, + $columnsAggregationOperation, + $columnsToRenameAfterAggregation = null, + $countRowsRecursive = array()); } /** |