diff options
author | mattab <matthieu.aubry@gmail.com> | 2013-06-17 09:54:07 +0400 |
---|---|---|
committer | mattab <matthieu.aubry@gmail.com> | 2013-06-17 09:54:07 +0400 |
commit | b22ff3499d0478843aa33c1c4924eb66f1c8fcf1 (patch) | |
tree | acafa0104fd3cec52ea8f85308c60080ffa3a908 /core/ArchiveProcessor.php | |
parent | a82321a712614f062b77955d2fe978766e0e35c6 (diff) |
Creating getMinTimeProcessedForTemporaryArchive and splitting getMinTimeArchivedProcessed to make it clear.
Renaming class to Piwik_DataAccess_ArchiveSelector -- it will specialize in selecting archives
Diffstat (limited to 'core/ArchiveProcessor.php')
-rw-r--r-- | core/ArchiveProcessor.php | 475 |
1 files changed, 228 insertions, 247 deletions
diff --git a/core/ArchiveProcessor.php b/core/ArchiveProcessor.php index 9791735d0e..c475177ee1 100644 --- a/core/ArchiveProcessor.php +++ b/core/ArchiveProcessor.php @@ -10,21 +10,8 @@ */ /** - * The ArchiveProcessor module is a module that reads the Piwik logs from the DB and - * compute all the reports, which are then stored in the database. - * * The ArchiveProcessor class is used by the Archive object to make sure the given Archive is processed and available in the DB. * - * A record in the Database for a given report is defined by - * - idarchive = unique ID that is associated to all the data of this archive (idsite+period+date) - * - idsite = the ID of the website - * - date1 = starting day of the period - * - date2 = ending day of the period - * - period = integer that defines the period (day/week/etc.). @see period::getId() - * - ts_archived = timestamp when the archive was processed (UTC) - * - name = the name of the report (ex: uniq_visitors or search_keywords_by_search_engines) - * - value = the actual data - * * @package Piwik * @subpackage Piwik_ArchiveProcessor */ @@ -75,26 +62,41 @@ abstract class Piwik_ArchiveProcessor protected $tableArchiveBlob; /** - * Minimum timestamp looked at for processed archives - * - * @var int - */ - protected $minDatetimeArchiveProcessedUTC = false; - - /** * Is the current archive temporary. ie. * - today * - current week / month / year */ protected $temporaryArchive; + protected $logAggregator = null; + /** - * This methods reads the subperiods if necessary, - * and computes the archive of the current period. + * @var int Number of visits cached as early as possible */ - abstract protected function compute(); + protected $visitsMetricCached = false; - abstract protected function aggregateCoreVisitsMetrics(); + /** + * @var int Number of visits with conversions, cached when selecting + */ + protected $convertedVisitsMetricCached = false; + + /** + * Site of the current archive + * Can be accessed by plugins (that is why it's public) + * + * @var Piwik_Site + */ + private $site = null; + + /** + * @var Piwik_Period + */ + private $period = null; + + /** + * @var Piwik_Segment + */ + private $segment = null; public function __construct(Piwik_Period $period, Piwik_Site $site, Piwik_Segment $segment) { @@ -103,65 +105,146 @@ abstract class Piwik_ArchiveProcessor $this->segment = $segment; } - protected $logAggregator = null; - /** * @return Piwik_DataAccess_LogAggregator */ public function getLogAggregator() { - if(empty($this->logAggregator)) { - $this->logAggregator = new Piwik_DataAccess_LogAggregator( $this->getPeriod()->getDateStart(), $this->getPeriod()->getDateEnd(), - $this->getSite(), $this->getSegment() ); + if (empty($this->logAggregator)) { + $this->logAggregator = new Piwik_DataAccess_LogAggregator($this->getPeriod()->getDateStart(), $this->getPeriod()->getDateEnd(), + $this->getSite(), $this->getSegment()); } return $this->logAggregator; } + public function preProcessArchive($requestedPlugin, $enforceProcessCoreMetricsOnly = false) + { + $this->idArchive = false; + + $this->setRequestedPlugin($requestedPlugin); + + if (!$enforceProcessCoreMetricsOnly) { + $this->idArchive = $this->loadExistingArchiveIdFromDb($requestedPlugin); + if ($this->isArchivingForcedToTrigger()) { + $this->idArchive = false; + $this->setNumberOfVisits(false); + } + if (!empty($this->idArchive)) { + return $this->idArchive; + } + + $visitsNotKnownYet = $this->getNumberOfVisits() === false; + + $createAnotherArchiveForVisitsSummary = !$this->doesRequestedPluginIncludeVisitsSummary($requestedPlugin) && $visitsNotKnownYet; + + if ($createAnotherArchiveForVisitsSummary) { + // recursive archive creation in case we create another separate one, for VisitsSummary core metrics + // We query VisitsSummary here, as it is needed in the call below ($this->getNumberOfVisits() > 0) + $requestedPlugin = $this->getRequestedPlugin(); + $this->preProcessArchive('VisitsSummary', $pleaseProcessCoreMetricsOnly = true); + $this->setRequestedPlugin($requestedPlugin); + if ($this->getNumberOfVisits() === false) { + throw new Exception("preProcessArchive() is expected to set number of visits to a numeric value."); + } + } + } + + return $this->computeNewArchive($requestedPlugin, $enforceProcessCoreMetricsOnly); + } + + public function setRequestedPlugin($plugin) + { + $this->requestedPlugin = $plugin; + } + /** - * Returns the minimum archive processed datetime to look at - * - * @return string Datetime string, or false if must look at any archive available + * Returns the idArchive if the archive is available in the database for the requested plugin. + * Returns false if the archive needs to be processed. * - * @public for tests + * @return int|false */ - public function getMinTimeArchivedProcessed() + protected function loadExistingArchiveIdFromDb($requestedPlugin) { + $minDatetimeArchiveProcessedUTC = $this->getMinTimeArchivedProcessed(); $site = $this->getSite(); - $segment = $this->getSegment(); $period = $this->getPeriod(); - $dateStart = $this->getDateStart(); - $dateEnd = $this->getDateEnd(); + $segment = $this->getSegment(); + $numericTableName = $this->getTableArchiveNumericName(); + $idAndVisits = Piwik_DataAccess_ArchiveSelector::getArchiveIdAndVisits($numericTableName, $site, $period, $segment, $minDatetimeArchiveProcessedUTC, $requestedPlugin); + if (!$idAndVisits) { + return false; + } + list($idArchive, $visits, $visitsConverted) = $idAndVisits; + $this->setNumberOfVisits($visits, $visitsConverted); + return $idArchive; + } + + protected static function determineIfArchivePermanent(Piwik_Date $dateEnd) + { $now = time(); $endTimestampUTC = strtotime($dateEnd->getDateEndUTC()); if ($endTimestampUTC <= $now) { // - if the period we are looking for is finished, we look for a ts_archived that // is greater than the last day of the archive - $this->temporaryArchive = false; return $endTimestampUTC; } + return false; + } - $this->temporaryArchive = true; - - $minimumArchiveTime = $now - Piwik_ArchiveProcessor_Rules::getTodayArchiveTimeToLive(); + /** + * Returns the minimum archive processed datetime to look at + * + * @return string Datetime string, or false if must look at any archive available + * + * @public for tests + */ + public function getMinTimeArchivedProcessed() + { - $isArchivingDisabled = Piwik_ArchiveProcessor_Rules::isArchivingDisabledFor($segment, $period->getLabel()); + $endDateTimestamp = self::determineIfArchivePermanent($this->getDateEnd()); + $isArchiveTemporary = ($endDateTimestamp === false); + $this->temporaryArchive = $isArchiveTemporary; - if ($isArchivingDisabled) { - if ($endTimestampUTC > $now - && $period->getNumberOfSubperiods() == 0 - && $dateStart->getTimestamp() <= $now - ) { - $minimumArchiveTime = false; - } else { - // However, if archiving is disabled for this request, we shall - // accept any archive that was processed today after 00:00:01 this morning - $timezone = $site->getTimezone(); - $minimumArchiveTime = Piwik_Date::factory(Piwik_Date::factory('now', $timezone)->getDateStartUTC())->setTimezone($timezone)->getTimestamp(); - } + if($endDateTimestamp) { + // Permanent archive + return $endDateTimestamp; } + // Temporary archive + return Piwik_ArchiveProcessor_Rules::getMinTimeProcessedForTemporaryArchive($this->getDateStart(), $this->getPeriod(), $this->getSegment(), $this->getSite()); + } + + /** + * @return Piwik_Date + */ + public function getDateStart() + { + return $this->getPeriod()->getDateStart()->setTimezone($this->getSite()->getTimezone()); + } - return $minimumArchiveTime; + /** + * @return Piwik_Date + */ + public function getDateEnd() + { + return $this->getPeriod()->getDateEnd()->setTimezone($this->getSite()->getTimezone()); + } + + /** + * A flag mechanism to store whether + * @param $visitsMetricCached + * @param bool $convertedVisitsMetricCached + */ + protected function setNumberOfVisits($visitsMetricCached, $convertedVisitsMetricCached = false) + { + if (empty($visitsMetricCached)) { + $visitsMetricCached = 0; + } + if (empty($convertedVisitsMetricCached)) { + $convertedVisitsMetricCached = 0; + } + $this->visitsMetricCached = (int)$visitsMetricCached; + $this->convertedVisitsMetricCached = (int)$convertedVisitsMetricCached; } protected function isArchivingForcedToTrigger() @@ -176,39 +259,9 @@ abstract class Piwik_ArchiveProcessor return Piwik_Config::getInstance()->Debug[$debugSetting]; } - public function preProcessArchive($requestedPlugin, $enforceProcessCoreMetricsOnly = false) + public function getNumberOfVisits() { - $this->idArchive = false; - - $this->setRequestedPlugin($requestedPlugin); - - if( !$enforceProcessCoreMetricsOnly ) { - $this->idArchive = $this->loadExistingArchiveIdFromDb($requestedPlugin); - if ($this->isArchivingForcedToTrigger()) { - $this->idArchive = false; - $this->setNumberOfVisits(false); - } - if (!empty($this->idArchive)) { - return $this->idArchive; - } - - $visitsNotKnownYet = $this->getNumberOfVisits() === false; - - $createAnotherArchiveForVisitsSummary = !$this->doesRequestedPluginIncludeVisitsSummary($requestedPlugin) && $visitsNotKnownYet; - - if ($createAnotherArchiveForVisitsSummary) { - // recursive archive creation in case we create another separate one, for VisitsSummary core metrics - // We query VisitsSummary here, as it is needed in the call below ($this->getNumberOfVisits() > 0) - $requestedPlugin = $this->getRequestedPlugin(); - $this->preProcessArchive('VisitsSummary', $pleaseProcessCoreMetricsOnly = true); - $this->setRequestedPlugin($requestedPlugin); - if($this->getNumberOfVisits() === false) { - throw new Exception("preProcessArchive() is expected to set number of visits to a numeric value."); - } - } - } - - return $this->computeNewArchive($requestedPlugin, $enforceProcessCoreMetricsOnly); + return $this->visitsMetricCached; } protected function doesRequestedPluginIncludeVisitsSummary($requestedPlugin) @@ -226,11 +279,9 @@ abstract class Piwik_ArchiveProcessor */ protected function computeNewArchive($requestedPlugin, $enforceProcessCoreMetricsOnly) { - if (!Piwik_DataAccess_Archiver::getArchiveProcessorLock($this->getSite()->getId(), $this->getPeriod(), $this->getSegment())) { - Piwik::log('SELECT GET_LOCK(?, 1) FAILED to acquire lock. Proceeding anyway...'); - } + Piwik_DataAccess_ArchiveSelector::getArchiveProcessorLock($this->getSite()->getId(), $this->getPeriod(), $this->getSegment()); - $this->idArchive = Piwik_DataAccess_Archiver::allocateNewArchiveId($this->getTableArchiveNumericName(), $this->getSite()->getId()); + $this->idArchive = Piwik_DataAccess_ArchiveSelector::allocateNewArchiveId($this->getTableArchiveNumericName(), $this->getSite()->getId()); $doneFlag = Piwik_ArchiveProcessor_Rules::getDoneStringFlagFor($this->getSegment(), $this->getPeriod()->getLabel(), $requestedPlugin); $this->insertNumericRecord($doneFlag, Piwik_ArchiveProcessor::DONE_ERROR); @@ -250,19 +301,7 @@ abstract class Piwik_ArchiveProcessor } } - $temporary = 'definitive archive'; - if ($this->isArchiveTemporary()) { - $temporary = 'temporary archive'; - } - Piwik::log(sprintf("'%s, idSite = %d (%s), segment '%s', report = '%s', UTC datetime [%s -> %s]", - $this->getPeriod()->getLabel(), - $this->getSite()->getId(), - $temporary, - $this->getSegment()->getString(), - $requestedPlugin, - $this->getDateStart()->getDateStartUTC(), - $this->getDateEnd()->getDateEndUTC() - )); + $this->logStatus($requestedPlugin); if ($this->getNumberOfVisits() > 0 && !$enforceProcessCoreMetricsOnly @@ -270,7 +309,7 @@ abstract class Piwik_ArchiveProcessor $this->compute(); } - Piwik_DataAccess_Archiver::deletePreviousArchiveStatus($this->getTableArchiveNumericName(), $doneFlag, $this->getIdArchive()); + Piwik_DataAccess_ArchiveSelector::deletePreviousArchiveStatus($this->getTableArchiveNumericName(), $doneFlag, $this->getIdArchive()); $flag = Piwik_ArchiveProcessor::DONE_OK; if ($this->isArchiveTemporary()) { @@ -278,77 +317,54 @@ abstract class Piwik_ArchiveProcessor } $this->insertNumericRecord($doneFlag, $flag); - Piwik_DataAccess_Archiver::releaseArchiveProcessorLock($this->getSite()->getId(), $this->getPeriod(), $this->getSegment()); + Piwik_DataAccess_ArchiveSelector::releaseArchiveProcessorLock($this->getSite()->getId(), $this->getPeriod(), $this->getSegment()); return $this->idArchive; } - /** - * Returns the name of the numeric table where the archive numeric values are stored - * - * @return string - */ - public function getTableArchiveNumericName() - { - if (empty($this->tableArchiveNumeric)) { - $this->tableArchiveNumeric = new Piwik_TablePartitioning_Monthly('archive_numeric'); - $this->tableArchiveNumeric->setTimestamp($this->getPeriod()->getDateStart()->getTimestamp()); - } - return $this->tableArchiveNumeric->getTableName(); - } + abstract protected function aggregateCoreVisitsMetrics(); /** - * Returns the name of the blob table where the archive blob values are stored - * - * @return string + * @param $requestedPlugin */ - public function getTableArchiveBlobName() + protected function logStatus($requestedPlugin) { - if (empty($this->tableArchiveBlob)) { - $this->tableArchiveBlob = new Piwik_TablePartitioning_Monthly('archive_blob'); - $this->tableArchiveBlob->setTimestamp($this->getPeriod()->getDateStart()->getTimestamp()); + $temporary = 'definitive archive'; + if ($this->isArchiveTemporary()) { + $temporary = 'temporary archive'; } - return $this->tableArchiveBlob->getTableName(); - } - - public function setRequestedPlugin($plugin) - { - $this->requestedPlugin = $plugin; + Piwik::log(sprintf("'%s, idSite = %d (%s), segment '%s', report = '%s', UTC datetime [%s -> %s]", + $this->getPeriod()->getLabel(), + $this->getSite()->getId(), + $temporary, + $this->getSegment()->getString(), + $requestedPlugin, + $this->getDateStart()->getDateStartUTC(), + $this->getDateEnd()->getDateEndUTC() + )); } - protected function getRequestedPlugin() + public function isArchiveTemporary() { - return $this->requestedPlugin; + return $this->temporaryArchive; } - protected $visitsMetricCached = false; - protected $convertedVisitsMetricCached = false; - /** - * A flag mechanism to store whether - * @param $visitsMetricCached - * @param bool $convertedVisitsMetricCached + * This methods reads the subperiods if necessary, + * and computes the archive of the current period. */ - protected function setNumberOfVisits($visitsMetricCached, $convertedVisitsMetricCached = false) - { - if (empty($visitsMetricCached)) { - $visitsMetricCached = 0; - } - if (empty($convertedVisitsMetricCached)) { - $convertedVisitsMetricCached = 0; - } - $this->visitsMetricCached = (int)$visitsMetricCached; - $this->convertedVisitsMetricCached = (int)$convertedVisitsMetricCached; - } + abstract protected function compute(); - public function getNumberOfVisits() + public function getNumberOfVisitsConverted() { - return $this->visitsMetricCached; + return $this->convertedVisitsMetricCached; } - public function getNumberOfVisitsConverted() + public function insertNumericRecords($numericRecords) { - return $this->convertedVisitsMetricCached; + foreach ($numericRecords as $name => $value) { + $this->insertNumericRecord($name, $value); + } } /** @@ -361,13 +377,6 @@ abstract class Piwik_ArchiveProcessor return $this->insertRecord($name, $value); } - public function insertNumericRecords($numericRecords) - { - foreach ($numericRecords as $name => $value) { - $this->insertNumericRecord($name, $value); - } - } - /** * @param string $name * @param string|array $values @@ -441,21 +450,6 @@ abstract class Piwik_ArchiveProcessor return true; } - protected function getInsertRecordBind() - { - return array($this->getIdArchive(), - $this->getSite()->getId(), - $this->getPeriod()->getDateStart()->toString('Y-m-d'), - $this->getPeriod()->getDateEnd()->toString('Y-m-d'), - $this->getPeriod()->getId(), - date("Y-m-d H:i:s")); - } - - protected function getInsertFields() - { - return array('idarchive', 'idsite', 'date1', 'date2', 'period', 'ts_archived', 'name', 'value'); - } - /** * Inserts a record in the right table (either NUMERIC or BLOB) * @@ -466,7 +460,7 @@ abstract class Piwik_ArchiveProcessor */ protected function insertRecord($name, $value) { - if($this->isRecordZero($name, $value)) { + if ($this->isRecordZero($name, $value)) { return; } @@ -481,6 +475,7 @@ abstract class Piwik_ArchiveProcessor $bindSql[] = $name; $bindSql[] = $value; Piwik_Query($query, $bindSql); + return true; } /** @@ -504,73 +499,56 @@ abstract class Piwik_ArchiveProcessor } /** - * Returns the idArchive if the archive is available in the database for the requested plugin. - * Returns false if the archive needs to be processed. + * Returns the name of the numeric table where the archive numeric values are stored * - * @return int|false + * @return string */ - protected function loadExistingArchiveIdFromDb($requestedPlugin) + public function getTableArchiveNumericName() { - $minDatetimeArchiveProcessedUTC = $this->getMinTimeArchivedProcessed(); - $site = $this->getSite(); - $period = $this->getPeriod(); - $segment = $this->getSegment(); - $numericTableName = $this->getTableArchiveNumericName(); - - $idAndVisits = Piwik_DataAccess_Archiver::getArchiveIdAndVisits($numericTableName, $site, $period, $segment, $minDatetimeArchiveProcessedUTC, $requestedPlugin); - if (!$idAndVisits) { - return false; + if (empty($this->tableArchiveNumeric)) { + $this->tableArchiveNumeric = new Piwik_TablePartitioning_Monthly('archive_numeric'); + $this->tableArchiveNumeric->setTimestamp($this->getPeriod()->getDateStart()->getTimestamp()); } - list($idArchive, $visits, $visitsConverted) = $idAndVisits; - $this->setNumberOfVisits($visits, $visitsConverted); - return $idArchive; + return $this->tableArchiveNumeric->getTableName(); } - /** - * Whether the specified plugin's reports should be archived - * @param string $pluginName - * @return bool - */ - public function shouldProcessReportsForPlugin($pluginName) + public function getPeriod() { - if (Piwik_ArchiveProcessor_Rules::shouldProcessReportsAllPlugins($this->getSegment(), $this->getPeriod()->getLabel())) { - return true; - } - // If any other segment, only process if the requested report belong to this plugin - $pluginBeingProcessed = $this->getRequestedPlugin(); - if ($pluginBeingProcessed == $pluginName) { - return true; - } - if (!Piwik_PluginsManager::getInstance()->isPluginLoaded($pluginBeingProcessed)) { - return true; - } - return false; + return $this->period; } /** - * Site of the current archive - * Can be accessed by plugins (that is why it's public) + * Returns the name of the blob table where the archive blob values are stored * - * @var Piwik_Site + * @return string */ - private $site = null; + public function getTableArchiveBlobName() + { + if (empty($this->tableArchiveBlob)) { + $this->tableArchiveBlob = new Piwik_TablePartitioning_Monthly('archive_blob'); + $this->tableArchiveBlob->setTimestamp($this->getPeriod()->getDateStart()->getTimestamp()); + } + return $this->tableArchiveBlob->getTableName(); + } - /** - * @var Piwik_Period - */ - private $period = null; + protected function getInsertFields() + { + return array('idarchive', 'idsite', 'date1', 'date2', 'period', 'ts_archived', 'name', 'value'); + } - /** - * @var Piwik_Segment - */ - private $segment = null; + protected function getInsertRecordBind() + { + return array($this->getIdArchive(), + $this->getSite()->getId(), + $this->getPeriod()->getDateStart()->toString('Y-m-d'), + $this->getPeriod()->getDateEnd()->toString('Y-m-d'), + $this->getPeriod()->getId(), + date("Y-m-d H:i:s")); + } - /** - * @return Piwik_Segment - */ - public function getSegment() + public function getIdArchive() { - return $this->segment; + return $this->idArchive; } /** @@ -581,34 +559,37 @@ abstract class Piwik_ArchiveProcessor return $this->site; } - public function getPeriod() - { - return $this->period; - } - /** - * @return Piwik_Date + * Whether the specified plugin's reports should be archived + * @param string $pluginName + * @return bool */ - public function getDateEnd() + public function shouldProcessReportsForPlugin($pluginName) { - return $this->getPeriod()->getDateEnd()->setTimezone($this->getSite()->getTimezone()); + if (Piwik_ArchiveProcessor_Rules::shouldProcessReportsAllPlugins($this->getSegment(), $this->getPeriod()->getLabel())) { + return true; + } + // If any other segment, only process if the requested report belong to this plugin + $pluginBeingProcessed = $this->getRequestedPlugin(); + if ($pluginBeingProcessed == $pluginName) { + return true; + } + if (!Piwik_PluginsManager::getInstance()->isPluginLoaded($pluginBeingProcessed)) { + return true; + } + return false; } /** - * @return Piwik_Date + * @return Piwik_Segment */ - public function getDateStart() - { - return $this->getPeriod()->getDateStart()->setTimezone($this->getSite()->getTimezone()); - } - - public function isArchiveTemporary() + public function getSegment() { - return $this->temporaryArchive; + return $this->segment; } - public function getIdArchive() + protected function getRequestedPlugin() { - return $this->idArchive; + return $this->requestedPlugin; } } |