Welcome to mirror list, hosted at ThFree Co, Russian Federation.

github.com/matomo-org/matomo.git - Unnamed repository; edit this file 'description' to name the repository.
summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authormattab <matthieu.aubry@gmail.com>2013-06-17 09:54:07 +0400
committermattab <matthieu.aubry@gmail.com>2013-06-17 09:54:07 +0400
commitb22ff3499d0478843aa33c1c4924eb66f1c8fcf1 (patch)
treeacafa0104fd3cec52ea8f85308c60080ffa3a908 /core/ArchiveProcessor.php
parenta82321a712614f062b77955d2fe978766e0e35c6 (diff)
Creating getMinTimeProcessedForTemporaryArchive and splitting getMinTimeArchivedProcessed to make it clear.
Renaming class to Piwik_DataAccess_ArchiveSelector -- it will specialize in selecting archives
Diffstat (limited to 'core/ArchiveProcessor.php')
-rw-r--r--core/ArchiveProcessor.php475
1 files changed, 228 insertions, 247 deletions
diff --git a/core/ArchiveProcessor.php b/core/ArchiveProcessor.php
index 9791735d0e..c475177ee1 100644
--- a/core/ArchiveProcessor.php
+++ b/core/ArchiveProcessor.php
@@ -10,21 +10,8 @@
*/
/**
- * The ArchiveProcessor module is a module that reads the Piwik logs from the DB and
- * compute all the reports, which are then stored in the database.
- *
* The ArchiveProcessor class is used by the Archive object to make sure the given Archive is processed and available in the DB.
*
- * A record in the Database for a given report is defined by
- * - idarchive = unique ID that is associated to all the data of this archive (idsite+period+date)
- * - idsite = the ID of the website
- * - date1 = starting day of the period
- * - date2 = ending day of the period
- * - period = integer that defines the period (day/week/etc.). @see period::getId()
- * - ts_archived = timestamp when the archive was processed (UTC)
- * - name = the name of the report (ex: uniq_visitors or search_keywords_by_search_engines)
- * - value = the actual data
- *
* @package Piwik
* @subpackage Piwik_ArchiveProcessor
*/
@@ -75,26 +62,41 @@ abstract class Piwik_ArchiveProcessor
protected $tableArchiveBlob;
/**
- * Minimum timestamp looked at for processed archives
- *
- * @var int
- */
- protected $minDatetimeArchiveProcessedUTC = false;
-
- /**
* Is the current archive temporary. ie.
* - today
* - current week / month / year
*/
protected $temporaryArchive;
+ protected $logAggregator = null;
+
/**
- * This methods reads the subperiods if necessary,
- * and computes the archive of the current period.
+ * @var int Number of visits cached as early as possible
*/
- abstract protected function compute();
+ protected $visitsMetricCached = false;
- abstract protected function aggregateCoreVisitsMetrics();
+ /**
+ * @var int Number of visits with conversions, cached when selecting
+ */
+ protected $convertedVisitsMetricCached = false;
+
+ /**
+ * Site of the current archive
+ * Can be accessed by plugins (that is why it's public)
+ *
+ * @var Piwik_Site
+ */
+ private $site = null;
+
+ /**
+ * @var Piwik_Period
+ */
+ private $period = null;
+
+ /**
+ * @var Piwik_Segment
+ */
+ private $segment = null;
public function __construct(Piwik_Period $period, Piwik_Site $site, Piwik_Segment $segment)
{
@@ -103,65 +105,146 @@ abstract class Piwik_ArchiveProcessor
$this->segment = $segment;
}
- protected $logAggregator = null;
-
/**
* @return Piwik_DataAccess_LogAggregator
*/
public function getLogAggregator()
{
- if(empty($this->logAggregator)) {
- $this->logAggregator = new Piwik_DataAccess_LogAggregator( $this->getPeriod()->getDateStart(), $this->getPeriod()->getDateEnd(),
- $this->getSite(), $this->getSegment() );
+ if (empty($this->logAggregator)) {
+ $this->logAggregator = new Piwik_DataAccess_LogAggregator($this->getPeriod()->getDateStart(), $this->getPeriod()->getDateEnd(),
+ $this->getSite(), $this->getSegment());
}
return $this->logAggregator;
}
+ public function preProcessArchive($requestedPlugin, $enforceProcessCoreMetricsOnly = false)
+ {
+ $this->idArchive = false;
+
+ $this->setRequestedPlugin($requestedPlugin);
+
+ if (!$enforceProcessCoreMetricsOnly) {
+ $this->idArchive = $this->loadExistingArchiveIdFromDb($requestedPlugin);
+ if ($this->isArchivingForcedToTrigger()) {
+ $this->idArchive = false;
+ $this->setNumberOfVisits(false);
+ }
+ if (!empty($this->idArchive)) {
+ return $this->idArchive;
+ }
+
+ $visitsNotKnownYet = $this->getNumberOfVisits() === false;
+
+ $createAnotherArchiveForVisitsSummary = !$this->doesRequestedPluginIncludeVisitsSummary($requestedPlugin) && $visitsNotKnownYet;
+
+ if ($createAnotherArchiveForVisitsSummary) {
+ // recursive archive creation in case we create another separate one, for VisitsSummary core metrics
+ // We query VisitsSummary here, as it is needed in the call below ($this->getNumberOfVisits() > 0)
+ $requestedPlugin = $this->getRequestedPlugin();
+ $this->preProcessArchive('VisitsSummary', $pleaseProcessCoreMetricsOnly = true);
+ $this->setRequestedPlugin($requestedPlugin);
+ if ($this->getNumberOfVisits() === false) {
+ throw new Exception("preProcessArchive() is expected to set number of visits to a numeric value.");
+ }
+ }
+ }
+
+ return $this->computeNewArchive($requestedPlugin, $enforceProcessCoreMetricsOnly);
+ }
+
+ public function setRequestedPlugin($plugin)
+ {
+ $this->requestedPlugin = $plugin;
+ }
+
/**
- * Returns the minimum archive processed datetime to look at
- *
- * @return string Datetime string, or false if must look at any archive available
+ * Returns the idArchive if the archive is available in the database for the requested plugin.
+ * Returns false if the archive needs to be processed.
*
- * @public for tests
+ * @return int|false
*/
- public function getMinTimeArchivedProcessed()
+ protected function loadExistingArchiveIdFromDb($requestedPlugin)
{
+ $minDatetimeArchiveProcessedUTC = $this->getMinTimeArchivedProcessed();
$site = $this->getSite();
- $segment = $this->getSegment();
$period = $this->getPeriod();
- $dateStart = $this->getDateStart();
- $dateEnd = $this->getDateEnd();
+ $segment = $this->getSegment();
+ $numericTableName = $this->getTableArchiveNumericName();
+ $idAndVisits = Piwik_DataAccess_ArchiveSelector::getArchiveIdAndVisits($numericTableName, $site, $period, $segment, $minDatetimeArchiveProcessedUTC, $requestedPlugin);
+ if (!$idAndVisits) {
+ return false;
+ }
+ list($idArchive, $visits, $visitsConverted) = $idAndVisits;
+ $this->setNumberOfVisits($visits, $visitsConverted);
+ return $idArchive;
+ }
+
+ protected static function determineIfArchivePermanent(Piwik_Date $dateEnd)
+ {
$now = time();
$endTimestampUTC = strtotime($dateEnd->getDateEndUTC());
if ($endTimestampUTC <= $now) {
// - if the period we are looking for is finished, we look for a ts_archived that
// is greater than the last day of the archive
- $this->temporaryArchive = false;
return $endTimestampUTC;
}
+ return false;
+ }
- $this->temporaryArchive = true;
-
- $minimumArchiveTime = $now - Piwik_ArchiveProcessor_Rules::getTodayArchiveTimeToLive();
+ /**
+ * Returns the minimum archive processed datetime to look at
+ *
+ * @return string Datetime string, or false if must look at any archive available
+ *
+ * @public for tests
+ */
+ public function getMinTimeArchivedProcessed()
+ {
- $isArchivingDisabled = Piwik_ArchiveProcessor_Rules::isArchivingDisabledFor($segment, $period->getLabel());
+ $endDateTimestamp = self::determineIfArchivePermanent($this->getDateEnd());
+ $isArchiveTemporary = ($endDateTimestamp === false);
+ $this->temporaryArchive = $isArchiveTemporary;
- if ($isArchivingDisabled) {
- if ($endTimestampUTC > $now
- && $period->getNumberOfSubperiods() == 0
- && $dateStart->getTimestamp() <= $now
- ) {
- $minimumArchiveTime = false;
- } else {
- // However, if archiving is disabled for this request, we shall
- // accept any archive that was processed today after 00:00:01 this morning
- $timezone = $site->getTimezone();
- $minimumArchiveTime = Piwik_Date::factory(Piwik_Date::factory('now', $timezone)->getDateStartUTC())->setTimezone($timezone)->getTimestamp();
- }
+ if($endDateTimestamp) {
+ // Permanent archive
+ return $endDateTimestamp;
}
+ // Temporary archive
+ return Piwik_ArchiveProcessor_Rules::getMinTimeProcessedForTemporaryArchive($this->getDateStart(), $this->getPeriod(), $this->getSegment(), $this->getSite());
+ }
+
+ /**
+ * @return Piwik_Date
+ */
+ public function getDateStart()
+ {
+ return $this->getPeriod()->getDateStart()->setTimezone($this->getSite()->getTimezone());
+ }
- return $minimumArchiveTime;
+ /**
+ * @return Piwik_Date
+ */
+ public function getDateEnd()
+ {
+ return $this->getPeriod()->getDateEnd()->setTimezone($this->getSite()->getTimezone());
+ }
+
+ /**
+ * A flag mechanism to store whether
+ * @param $visitsMetricCached
+ * @param bool $convertedVisitsMetricCached
+ */
+ protected function setNumberOfVisits($visitsMetricCached, $convertedVisitsMetricCached = false)
+ {
+ if (empty($visitsMetricCached)) {
+ $visitsMetricCached = 0;
+ }
+ if (empty($convertedVisitsMetricCached)) {
+ $convertedVisitsMetricCached = 0;
+ }
+ $this->visitsMetricCached = (int)$visitsMetricCached;
+ $this->convertedVisitsMetricCached = (int)$convertedVisitsMetricCached;
}
protected function isArchivingForcedToTrigger()
@@ -176,39 +259,9 @@ abstract class Piwik_ArchiveProcessor
return Piwik_Config::getInstance()->Debug[$debugSetting];
}
- public function preProcessArchive($requestedPlugin, $enforceProcessCoreMetricsOnly = false)
+ public function getNumberOfVisits()
{
- $this->idArchive = false;
-
- $this->setRequestedPlugin($requestedPlugin);
-
- if( !$enforceProcessCoreMetricsOnly ) {
- $this->idArchive = $this->loadExistingArchiveIdFromDb($requestedPlugin);
- if ($this->isArchivingForcedToTrigger()) {
- $this->idArchive = false;
- $this->setNumberOfVisits(false);
- }
- if (!empty($this->idArchive)) {
- return $this->idArchive;
- }
-
- $visitsNotKnownYet = $this->getNumberOfVisits() === false;
-
- $createAnotherArchiveForVisitsSummary = !$this->doesRequestedPluginIncludeVisitsSummary($requestedPlugin) && $visitsNotKnownYet;
-
- if ($createAnotherArchiveForVisitsSummary) {
- // recursive archive creation in case we create another separate one, for VisitsSummary core metrics
- // We query VisitsSummary here, as it is needed in the call below ($this->getNumberOfVisits() > 0)
- $requestedPlugin = $this->getRequestedPlugin();
- $this->preProcessArchive('VisitsSummary', $pleaseProcessCoreMetricsOnly = true);
- $this->setRequestedPlugin($requestedPlugin);
- if($this->getNumberOfVisits() === false) {
- throw new Exception("preProcessArchive() is expected to set number of visits to a numeric value.");
- }
- }
- }
-
- return $this->computeNewArchive($requestedPlugin, $enforceProcessCoreMetricsOnly);
+ return $this->visitsMetricCached;
}
protected function doesRequestedPluginIncludeVisitsSummary($requestedPlugin)
@@ -226,11 +279,9 @@ abstract class Piwik_ArchiveProcessor
*/
protected function computeNewArchive($requestedPlugin, $enforceProcessCoreMetricsOnly)
{
- if (!Piwik_DataAccess_Archiver::getArchiveProcessorLock($this->getSite()->getId(), $this->getPeriod(), $this->getSegment())) {
- Piwik::log('SELECT GET_LOCK(?, 1) FAILED to acquire lock. Proceeding anyway...');
- }
+ Piwik_DataAccess_ArchiveSelector::getArchiveProcessorLock($this->getSite()->getId(), $this->getPeriod(), $this->getSegment());
- $this->idArchive = Piwik_DataAccess_Archiver::allocateNewArchiveId($this->getTableArchiveNumericName(), $this->getSite()->getId());
+ $this->idArchive = Piwik_DataAccess_ArchiveSelector::allocateNewArchiveId($this->getTableArchiveNumericName(), $this->getSite()->getId());
$doneFlag = Piwik_ArchiveProcessor_Rules::getDoneStringFlagFor($this->getSegment(), $this->getPeriod()->getLabel(), $requestedPlugin);
$this->insertNumericRecord($doneFlag, Piwik_ArchiveProcessor::DONE_ERROR);
@@ -250,19 +301,7 @@ abstract class Piwik_ArchiveProcessor
}
}
- $temporary = 'definitive archive';
- if ($this->isArchiveTemporary()) {
- $temporary = 'temporary archive';
- }
- Piwik::log(sprintf("'%s, idSite = %d (%s), segment '%s', report = '%s', UTC datetime [%s -> %s]",
- $this->getPeriod()->getLabel(),
- $this->getSite()->getId(),
- $temporary,
- $this->getSegment()->getString(),
- $requestedPlugin,
- $this->getDateStart()->getDateStartUTC(),
- $this->getDateEnd()->getDateEndUTC()
- ));
+ $this->logStatus($requestedPlugin);
if ($this->getNumberOfVisits() > 0
&& !$enforceProcessCoreMetricsOnly
@@ -270,7 +309,7 @@ abstract class Piwik_ArchiveProcessor
$this->compute();
}
- Piwik_DataAccess_Archiver::deletePreviousArchiveStatus($this->getTableArchiveNumericName(), $doneFlag, $this->getIdArchive());
+ Piwik_DataAccess_ArchiveSelector::deletePreviousArchiveStatus($this->getTableArchiveNumericName(), $doneFlag, $this->getIdArchive());
$flag = Piwik_ArchiveProcessor::DONE_OK;
if ($this->isArchiveTemporary()) {
@@ -278,77 +317,54 @@ abstract class Piwik_ArchiveProcessor
}
$this->insertNumericRecord($doneFlag, $flag);
- Piwik_DataAccess_Archiver::releaseArchiveProcessorLock($this->getSite()->getId(), $this->getPeriod(), $this->getSegment());
+ Piwik_DataAccess_ArchiveSelector::releaseArchiveProcessorLock($this->getSite()->getId(), $this->getPeriod(), $this->getSegment());
return $this->idArchive;
}
- /**
- * Returns the name of the numeric table where the archive numeric values are stored
- *
- * @return string
- */
- public function getTableArchiveNumericName()
- {
- if (empty($this->tableArchiveNumeric)) {
- $this->tableArchiveNumeric = new Piwik_TablePartitioning_Monthly('archive_numeric');
- $this->tableArchiveNumeric->setTimestamp($this->getPeriod()->getDateStart()->getTimestamp());
- }
- return $this->tableArchiveNumeric->getTableName();
- }
+ abstract protected function aggregateCoreVisitsMetrics();
/**
- * Returns the name of the blob table where the archive blob values are stored
- *
- * @return string
+ * @param $requestedPlugin
*/
- public function getTableArchiveBlobName()
+ protected function logStatus($requestedPlugin)
{
- if (empty($this->tableArchiveBlob)) {
- $this->tableArchiveBlob = new Piwik_TablePartitioning_Monthly('archive_blob');
- $this->tableArchiveBlob->setTimestamp($this->getPeriod()->getDateStart()->getTimestamp());
+ $temporary = 'definitive archive';
+ if ($this->isArchiveTemporary()) {
+ $temporary = 'temporary archive';
}
- return $this->tableArchiveBlob->getTableName();
- }
-
- public function setRequestedPlugin($plugin)
- {
- $this->requestedPlugin = $plugin;
+ Piwik::log(sprintf("'%s, idSite = %d (%s), segment '%s', report = '%s', UTC datetime [%s -> %s]",
+ $this->getPeriod()->getLabel(),
+ $this->getSite()->getId(),
+ $temporary,
+ $this->getSegment()->getString(),
+ $requestedPlugin,
+ $this->getDateStart()->getDateStartUTC(),
+ $this->getDateEnd()->getDateEndUTC()
+ ));
}
- protected function getRequestedPlugin()
+ public function isArchiveTemporary()
{
- return $this->requestedPlugin;
+ return $this->temporaryArchive;
}
- protected $visitsMetricCached = false;
- protected $convertedVisitsMetricCached = false;
-
/**
- * A flag mechanism to store whether
- * @param $visitsMetricCached
- * @param bool $convertedVisitsMetricCached
+ * This methods reads the subperiods if necessary,
+ * and computes the archive of the current period.
*/
- protected function setNumberOfVisits($visitsMetricCached, $convertedVisitsMetricCached = false)
- {
- if (empty($visitsMetricCached)) {
- $visitsMetricCached = 0;
- }
- if (empty($convertedVisitsMetricCached)) {
- $convertedVisitsMetricCached = 0;
- }
- $this->visitsMetricCached = (int)$visitsMetricCached;
- $this->convertedVisitsMetricCached = (int)$convertedVisitsMetricCached;
- }
+ abstract protected function compute();
- public function getNumberOfVisits()
+ public function getNumberOfVisitsConverted()
{
- return $this->visitsMetricCached;
+ return $this->convertedVisitsMetricCached;
}
- public function getNumberOfVisitsConverted()
+ public function insertNumericRecords($numericRecords)
{
- return $this->convertedVisitsMetricCached;
+ foreach ($numericRecords as $name => $value) {
+ $this->insertNumericRecord($name, $value);
+ }
}
/**
@@ -361,13 +377,6 @@ abstract class Piwik_ArchiveProcessor
return $this->insertRecord($name, $value);
}
- public function insertNumericRecords($numericRecords)
- {
- foreach ($numericRecords as $name => $value) {
- $this->insertNumericRecord($name, $value);
- }
- }
-
/**
* @param string $name
* @param string|array $values
@@ -441,21 +450,6 @@ abstract class Piwik_ArchiveProcessor
return true;
}
- protected function getInsertRecordBind()
- {
- return array($this->getIdArchive(),
- $this->getSite()->getId(),
- $this->getPeriod()->getDateStart()->toString('Y-m-d'),
- $this->getPeriod()->getDateEnd()->toString('Y-m-d'),
- $this->getPeriod()->getId(),
- date("Y-m-d H:i:s"));
- }
-
- protected function getInsertFields()
- {
- return array('idarchive', 'idsite', 'date1', 'date2', 'period', 'ts_archived', 'name', 'value');
- }
-
/**
* Inserts a record in the right table (either NUMERIC or BLOB)
*
@@ -466,7 +460,7 @@ abstract class Piwik_ArchiveProcessor
*/
protected function insertRecord($name, $value)
{
- if($this->isRecordZero($name, $value)) {
+ if ($this->isRecordZero($name, $value)) {
return;
}
@@ -481,6 +475,7 @@ abstract class Piwik_ArchiveProcessor
$bindSql[] = $name;
$bindSql[] = $value;
Piwik_Query($query, $bindSql);
+ return true;
}
/**
@@ -504,73 +499,56 @@ abstract class Piwik_ArchiveProcessor
}
/**
- * Returns the idArchive if the archive is available in the database for the requested plugin.
- * Returns false if the archive needs to be processed.
+ * Returns the name of the numeric table where the archive numeric values are stored
*
- * @return int|false
+ * @return string
*/
- protected function loadExistingArchiveIdFromDb($requestedPlugin)
+ public function getTableArchiveNumericName()
{
- $minDatetimeArchiveProcessedUTC = $this->getMinTimeArchivedProcessed();
- $site = $this->getSite();
- $period = $this->getPeriod();
- $segment = $this->getSegment();
- $numericTableName = $this->getTableArchiveNumericName();
-
- $idAndVisits = Piwik_DataAccess_Archiver::getArchiveIdAndVisits($numericTableName, $site, $period, $segment, $minDatetimeArchiveProcessedUTC, $requestedPlugin);
- if (!$idAndVisits) {
- return false;
+ if (empty($this->tableArchiveNumeric)) {
+ $this->tableArchiveNumeric = new Piwik_TablePartitioning_Monthly('archive_numeric');
+ $this->tableArchiveNumeric->setTimestamp($this->getPeriod()->getDateStart()->getTimestamp());
}
- list($idArchive, $visits, $visitsConverted) = $idAndVisits;
- $this->setNumberOfVisits($visits, $visitsConverted);
- return $idArchive;
+ return $this->tableArchiveNumeric->getTableName();
}
- /**
- * Whether the specified plugin's reports should be archived
- * @param string $pluginName
- * @return bool
- */
- public function shouldProcessReportsForPlugin($pluginName)
+ public function getPeriod()
{
- if (Piwik_ArchiveProcessor_Rules::shouldProcessReportsAllPlugins($this->getSegment(), $this->getPeriod()->getLabel())) {
- return true;
- }
- // If any other segment, only process if the requested report belong to this plugin
- $pluginBeingProcessed = $this->getRequestedPlugin();
- if ($pluginBeingProcessed == $pluginName) {
- return true;
- }
- if (!Piwik_PluginsManager::getInstance()->isPluginLoaded($pluginBeingProcessed)) {
- return true;
- }
- return false;
+ return $this->period;
}
/**
- * Site of the current archive
- * Can be accessed by plugins (that is why it's public)
+ * Returns the name of the blob table where the archive blob values are stored
*
- * @var Piwik_Site
+ * @return string
*/
- private $site = null;
+ public function getTableArchiveBlobName()
+ {
+ if (empty($this->tableArchiveBlob)) {
+ $this->tableArchiveBlob = new Piwik_TablePartitioning_Monthly('archive_blob');
+ $this->tableArchiveBlob->setTimestamp($this->getPeriod()->getDateStart()->getTimestamp());
+ }
+ return $this->tableArchiveBlob->getTableName();
+ }
- /**
- * @var Piwik_Period
- */
- private $period = null;
+ protected function getInsertFields()
+ {
+ return array('idarchive', 'idsite', 'date1', 'date2', 'period', 'ts_archived', 'name', 'value');
+ }
- /**
- * @var Piwik_Segment
- */
- private $segment = null;
+ protected function getInsertRecordBind()
+ {
+ return array($this->getIdArchive(),
+ $this->getSite()->getId(),
+ $this->getPeriod()->getDateStart()->toString('Y-m-d'),
+ $this->getPeriod()->getDateEnd()->toString('Y-m-d'),
+ $this->getPeriod()->getId(),
+ date("Y-m-d H:i:s"));
+ }
- /**
- * @return Piwik_Segment
- */
- public function getSegment()
+ public function getIdArchive()
{
- return $this->segment;
+ return $this->idArchive;
}
/**
@@ -581,34 +559,37 @@ abstract class Piwik_ArchiveProcessor
return $this->site;
}
- public function getPeriod()
- {
- return $this->period;
- }
-
/**
- * @return Piwik_Date
+ * Whether the specified plugin's reports should be archived
+ * @param string $pluginName
+ * @return bool
*/
- public function getDateEnd()
+ public function shouldProcessReportsForPlugin($pluginName)
{
- return $this->getPeriod()->getDateEnd()->setTimezone($this->getSite()->getTimezone());
+ if (Piwik_ArchiveProcessor_Rules::shouldProcessReportsAllPlugins($this->getSegment(), $this->getPeriod()->getLabel())) {
+ return true;
+ }
+ // If any other segment, only process if the requested report belong to this plugin
+ $pluginBeingProcessed = $this->getRequestedPlugin();
+ if ($pluginBeingProcessed == $pluginName) {
+ return true;
+ }
+ if (!Piwik_PluginsManager::getInstance()->isPluginLoaded($pluginBeingProcessed)) {
+ return true;
+ }
+ return false;
}
/**
- * @return Piwik_Date
+ * @return Piwik_Segment
*/
- public function getDateStart()
- {
- return $this->getPeriod()->getDateStart()->setTimezone($this->getSite()->getTimezone());
- }
-
- public function isArchiveTemporary()
+ public function getSegment()
{
- return $this->temporaryArchive;
+ return $this->segment;
}
- public function getIdArchive()
+ protected function getRequestedPlugin()
{
- return $this->idArchive;
+ return $this->requestedPlugin;
}
}