diff options
38 files changed, 382 insertions, 1768 deletions
diff --git a/core/Archive.php b/core/Archive.php index f5856d4a99..e1dd558a07 100644 --- a/core/Archive.php +++ b/core/Archive.php @@ -102,6 +102,25 @@ class Piwik_Archive const INDEX_GOAL_ECOMMERCE_REVENUE_DISCOUNT = 7; const INDEX_GOAL_ECOMMERCE_ITEMS = 8; + public static function getVisitsMetricNames() + { + $names = array(); + foreach(self::$metricsAggregatedFromLogs as $metricId) { + $names[$metricId] = self::$mappingFromIdToName[$metricId]; + } + return $names; + } + + protected static $metricsAggregatedFromLogs = array( + Piwik_Archive::INDEX_NB_UNIQ_VISITORS, + Piwik_Archive::INDEX_NB_VISITS, + Piwik_Archive::INDEX_NB_ACTIONS, + Piwik_Archive::INDEX_MAX_ACTIONS, + Piwik_Archive::INDEX_SUM_VISIT_LENGTH, + Piwik_Archive::INDEX_BOUNCE_COUNT, + Piwik_Archive::INDEX_NB_VISITS_CONVERTED, + ); + public static $mappingFromIdToName = array( Piwik_Archive::INDEX_NB_UNIQ_VISITORS => 'nb_uniq_visitors', Piwik_Archive::INDEX_NB_VISITS => 'nb_visits', @@ -240,12 +259,12 @@ class Piwik_Archive /** * Data Access Layer object. * - * @var Piwik_DataAccess_ArchiveQuery + * @var Piwik_DataAccess_Archiver */ private $dataAccess; /** - * Cache of Piwik_ArchiveProcessing instances used when launching the archiving + * Cache of Piwik_ArchiveProcessor instances used when launching the archiving * process. * * @var array @@ -270,7 +289,7 @@ class Piwik_Archive $this->params = $params; $this->forceIndexedBySite = $forceIndexedBySite; $this->forceIndexedByDate = $forceIndexedByDate; - $this->dataAccess = new Piwik_DataAccess_ArchiveQuery(); + $this->dataAccess = new Piwik_DataAccess_Archiver(); } @@ -287,34 +306,37 @@ class Piwik_Archive */ public static function build($idSite, $period, $strDate, $segment = false, $_restrictSitesToLogin = false) { - $forceIndexedBySite = false; - $forceIndexedByDate = false; - - // determine site IDs to query from - if (is_array($idSite) - || $idSite == 'all' - ) { - $forceIndexedBySite = true; - } $sites = Piwik_Site::getIdSitesFromIdSitesString($idSite, $_restrictSitesToLogin); if (self::isMultiplePeriod($strDate, $period)) { $oPeriod = new Piwik_Period_Range($period, $strDate); $allPeriods = $oPeriod->getSubperiods(); - $forceIndexedByDate = true; } else { $timezone = count($sites) == 1 ? Piwik_Site::getTimezoneFor($sites[0]) : false; $oPeriod = Piwik_Archive::makePeriodFromQueryParams($timezone, $period, $strDate); $allPeriods = array($oPeriod); } - $segment = new Piwik_Segment($segment, $sites); + return self::factory($segment, $allPeriods, $sites); + } + + public static function factory(Piwik_Segment $segment, array $periods, array $sites) + { + $forceIndexedBySite = false; + $forceIndexedByDate = false; + + if (count($sites) > 1) { + $forceIndexedBySite = true; + } + if (count($periods) > 1) { + $forceIndexedByDate = true; + } $params = new Piwik_Archive_Parameters(); $params->setIdSites($sites); - $params->setPeriods($allPeriods); + $params->setPeriods($periods); $params->setSegment($segment); - + return new Piwik_Archive($params, $forceIndexedBySite, $forceIndexedByDate); } @@ -445,16 +467,6 @@ class Piwik_Archive $data = $this->get($name, 'blob', 'all'); return $data->getExpandedDataTable($this->getResultIndices(), $idSubtable, $addMetadataSubtableId); } - - /** - * Returns true if we shouldn't launch the archiving process and false if we should. - * - * @return bool - */ - public function isArchivingDisabled() - { - return Piwik_ArchiveProcessing::isArchivingDisabledFor($this->params->getSegment(), $this->getPeriodLabel()); - } /** * Returns true if Segmentation is allowed for this user @@ -594,7 +606,7 @@ class Piwik_Archive * Returns archive IDs for the sites, periods and archive names that are being * queried. This function will use the idarchive cache if it has the right data, * query archive tables for IDs w/o launching archiving, or launch archiving and - * get the idarchive from Piwik_ArchiveProcessing instances. + * get the idarchive from Piwik_ArchiveProcessor instances. */ private function getArchiveIds($archiveNames) { @@ -617,7 +629,7 @@ class Piwik_Archive // cache id archives for plugins we haven't processed yet if (!empty($archiveGroups)) { - if (!$this->isArchivingDisabled()) { + if (!Piwik_ArchiveProcessor_Rules::isArchivingDisabledFor($this->params->getSegment(), $this->getPeriodLabel())) { $this->cacheArchiveIdsAfterLaunching($archiveGroups, $plugins); } else { $this->cacheArchiveIdsWithoutLaunching($plugins); @@ -680,33 +692,21 @@ class Piwik_Archive } // prepare the ArchiveProcessing instance - $processing = Piwik_ArchiveProcessing::factory($period->getLabel()); + $processing = Piwik_ArchiveProcessor::factory($period, $site, $this->params->getSegment()); - $processing->setSite($site); - $processing->setPeriod($period); - $processing->setSegment($this->params->getSegment()); - - $processing->isThereSomeVisits = null; - // process for each plugin as well foreach ($archiveGroups as $plugin) { if ($plugin == 'all') { $plugin = reset($plugins); } - + $doneFlag = $this->getDoneStringForPlugin($plugin); $this->initializeArchiveIdCache($doneFlag); - $processing->setRequestedPlugin($plugin); - - // launch archiving if the requested data hasn't been archived - $idArchive = $processing->loadArchive(); - if (empty($idArchive)) { - $processing->launchArchiving(); - $idArchive = $processing->getIdArchive(); - } - - if (!$processing->isThereSomeVisits()) { + $processing->launchArchiving($plugin); + $idArchive = $processing->getIdArchive(); + + if ($processing->getNumberOfVisits() == 0) { continue; } @@ -715,7 +715,7 @@ class Piwik_Archive } } } - + /** * Gets the IDs of the archives we're querying for and stores them in $this->archives. * This function will not launch the archiving process (and is thus much, much faster @@ -753,7 +753,7 @@ class Piwik_Archive */ private function getDoneStringForPlugin($plugin) { - return Piwik_ArchiveProcessing::getDoneStringFlagFor($this->params->getSegment(), $this->getPeriodLabel(), $plugin); + return Piwik_ArchiveProcessor_Rules::getDoneStringFlagFor($this->params->getSegment(), $this->getPeriodLabel(), $plugin); } private function getPeriodLabel() @@ -862,9 +862,7 @@ class Piwik_Archive public static function getPluginForReport($report) { // Core metrics are always processed in Core, for the requested date/period/segment - if (in_array($report, Piwik_ArchiveProcessing::getCoreMetrics()) - || $report == 'max_actions' - ) { + if (in_array($report, Piwik_Archive::getVisitsMetricNames())) { $report = 'VisitsSummary_CoreMetrics'; } // Goal_* metrics are processed by the Goals plugin (HACK) diff --git a/core/ArchiveProcessing.php b/core/ArchiveProcessing.php deleted file mode 100644 index 5785458e18..0000000000 --- a/core/ArchiveProcessing.php +++ /dev/null @@ -1,935 +0,0 @@ -<?php -/** - * Piwik - Open source web analytics - * - * @link http://piwik.org - * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later - * - * @category Piwik - * @package Piwik - */ - -/** - * The ArchiveProcessing module is a module that reads the Piwik logs from the DB and - * compute all the reports, which are then stored in the database. - * - * The ArchiveProcessing class is used by the Archive object to make sure the given Archive is processed and available in the DB. - * - * A record in the Database for a given report is defined by - * - idarchive = unique ID that is associated to all the data of this archive (idsite+period+date) - * - idsite = the ID of the website - * - date1 = starting day of the period - * - date2 = ending day of the period - * - period = integer that defines the period (day/week/etc.). @see period::getId() - * - ts_archived = timestamp when the archive was processed (UTC) - * - name = the name of the report (ex: uniq_visitors or search_keywords_by_search_engines) - * - value = the actual data - * - * @package Piwik - * @subpackage Piwik_ArchiveProcessing - */ -abstract class Piwik_ArchiveProcessing -{ - /** - * Flag stored at the end of the archiving - * - * @var int - */ - const DONE_OK = 1; - - /** - * Flag stored at the start of the archiving - * When requesting an Archive, we make sure that non-finished archive are not considered valid - * - * @var int - */ - const DONE_ERROR = 2; - - /** - * Flag indicates the archive is over a period that is not finished, eg. the current day, current week, etc. - * Archives flagged will be regularly purged from the DB. - * - * @var int - */ - const DONE_OK_TEMPORARY = 3; - - /** - * A row is created to lock an idarchive for the current archive being processed - * @var string - */ - const PREFIX_SQL_LOCK = "locked_"; - - /** - * Idarchive in the DB for the requested archive - * - * @var int - */ - protected $idArchive; - - /** - * Object used to generate (depending on the $dateStart) the name of the DB table to use to store numeric values - * - * @var Piwik_TablePartitioning - */ - protected $tableArchiveNumeric; - - /** - * Object used to generate (depending on the $dateStart) the name of the DB table to use to store numeric values - * - * @var Piwik_TablePartitioning - */ - protected $tableArchiveBlob; - - /** - * Minimum timestamp looked at for processed archives - * - * @var int - */ - protected $minDatetimeArchiveProcessedUTC = false; - - /** - * Is the current archive temporary. ie. - * - today - * - current week / month / year - */ - protected $temporaryArchive; - - /** - * When set to true, we always archive, even if the archive is already available. - * You can change this settings automatically in the config/global.ini.php always_archive_data under the [Debug] section - * - * @var bool - */ - public $debugAlwaysArchive = false; - - /** - * If the archive has at least 1 visit, this is set to true. - * - * @var bool - */ - public $isThereSomeVisits = null; - - /** - * Flag that will forcefully disable the archiving process. Only set by the tests. - */ - public static $forceDisableArchiving = false; - - - /** - * This methods reads the subperiods if necessary, - * and computes the archive of the current period. - */ - abstract protected function compute(); - - abstract public function isThereSomeVisits(); - - /** - * Returns the Piwik_ArchiveProcessing_Day or Piwik_ArchiveProcessing_Period object - * depending on $name period string - * - * @param string $name day|week|month|year - * @throws Exception - * @return Piwik_ArchiveProcessing Piwik_ArchiveProcessing_Day|Piwik_ArchiveProcessing_Period - */ - public static function factory($name) - { - switch ($name) { - case 'day': - $process = new Piwik_ArchiveProcessing_Day(); - $process->debugAlwaysArchive = Piwik_Config::getInstance()->Debug['always_archive_data_day']; - break; - - case 'week': - case 'month': - case 'year': - $process = new Piwik_ArchiveProcessing_Period(); - $process->debugAlwaysArchive = Piwik_Config::getInstance()->Debug['always_archive_data_period']; - break; - - case 'range': - $process = new Piwik_ArchiveProcessing_Period(); - $process->debugAlwaysArchive = Piwik_Config::getInstance()->Debug['always_archive_data_range']; - break; - - default: - throw new Exception("Unknown Archiving period specified '$name'"); - break; - } - return $process; - } - - /** - * @return Piwik_Archive - */ - protected function makeNewArchive() - { - $params = new Piwik_Archive_Parameters(); - $params->setSegment($this->getSegment()); - $params->setIdSites($this->getSite()->getId()); - $params->setPeriods($this->getPeriod()); - - $archive = new Piwik_Archive($params); - return $archive; - } - - - - const OPTION_TODAY_ARCHIVE_TTL = 'todayArchiveTimeToLive'; - const OPTION_BROWSER_TRIGGER_ARCHIVING = 'enableBrowserTriggerArchiving'; - - public static function getCoreMetrics() - { - return array( - 'nb_uniq_visitors', - 'nb_visits', - 'nb_actions', - 'sum_visit_length', - 'bounce_count', - 'nb_visits_converted', - ); - } - - public static function setTodayArchiveTimeToLive($timeToLiveSeconds) - { - $timeToLiveSeconds = (int)$timeToLiveSeconds; - if ($timeToLiveSeconds <= 0) { - throw new Exception(Piwik_TranslateException('General_ExceptionInvalidArchiveTimeToLive')); - } - Piwik_SetOption(self::OPTION_TODAY_ARCHIVE_TTL, $timeToLiveSeconds, $autoload = true); - } - - public static function getTodayArchiveTimeToLive() - { - $timeToLive = Piwik_GetOption(self::OPTION_TODAY_ARCHIVE_TTL); - if ($timeToLive !== false) { - return $timeToLive; - } - return Piwik_Config::getInstance()->General['time_before_today_archive_considered_outdated']; - } - - public static function setBrowserTriggerArchiving($enabled) - { - if (!is_bool($enabled)) { - throw new Exception('Browser trigger archiving must be set to true or false.'); - } - Piwik_SetOption(self::OPTION_BROWSER_TRIGGER_ARCHIVING, (int)$enabled, $autoload = true); - Piwik_Tracker_Cache::clearCacheGeneral(); - } - - public static function isBrowserTriggerArchivingEnabled() - { - $browserArchivingEnabled = Piwik_GetOption(self::OPTION_BROWSER_TRIGGER_ARCHIVING); - if ($browserArchivingEnabled !== false) { - return (bool)$browserArchivingEnabled; - } - return (bool)Piwik_Config::getInstance()->General['enable_browser_archiving_triggering']; - } - - public function getIdArchive() - { - return $this->idArchive; - } - - private function getDateEnd() - { - return $this->getPeriod()->getDateEnd()->setTimezone($this->getSite()->getTimezone()); - } - - private function getDateStart() - { - return $this->getPeriod()->getDateStart()->setTimezone($this->getSite()->getTimezone()); - } - - public function getStartDatetimeUTC() - { - return $this->getDateStart()->getDateStartUTC(); - } - - public function getEndDatetimeUTC() - { - return $this->getDateEnd()->getDateEndUTC(); - } - - public function isArchiveTemporary() - { - return $this->temporaryArchive; - } - - /** - * Returns the minimum archive processed datetime to look at - * - * @return string Datetime string, or false if must look at any archive available - * - * @public for tests - */ - public function getMinTimeArchivedProcessed() - { - $startTimestampUTC = $this->getDateStart()->getTimestamp(); - $endTimestampUTC = strtotime($this->getEndDatetimeUTC()); - $now = time(); - - $this->temporaryArchive = false; - // if the current archive is a DAY and if it's today, - // we set this minDatetimeArchiveProcessedUTC that defines the lifetime value of today's archive - if ($this->getPeriod()->getNumberOfSubperiods() == 0 - && ($startTimestampUTC <= $now && $endTimestampUTC > $now) - ) { - $this->temporaryArchive = true; - $minDatetimeArchiveProcessedUTC = $now - self::getTodayArchiveTimeToLive(); - // see #1150; if new archives are not triggered from the browser, - // we still want to try and return the latest archive available for today (rather than return nothing) - if ($this->isArchivingDisabled()) { - return false; - } - } // - if the period we are looking for is finished, we look for a ts_archived that - // is greater than the last day of the archive - elseif ($endTimestampUTC <= $now) { - $minDatetimeArchiveProcessedUTC = $endTimestampUTC; - } // - if the period we're looking for is not finished, we look for a recent enough archive - else { - $this->temporaryArchive = true; - - // We choose to only look at archives that are newer than the specified timeout - $minDatetimeArchiveProcessedUTC = $now - self::getTodayArchiveTimeToLive(); - - // However, if archiving is disabled for this request, we shall - // accept any archive that was processed today after 00:00:01 this morning - if ($this->isArchivingDisabled()) { - $timezone = $this->getSite()->getTimezone(); - $minDatetimeArchiveProcessedUTC = Piwik_Date::factory(Piwik_Date::factory('now', $timezone)->getDateStartUTC())->setTimezone($timezone)->getTimestamp(); - } - } - return $minDatetimeArchiveProcessedUTC; - } - - /** - * This method returns the idArchive ; if necessary, it triggers the archiving process. - * - * If the archive was not processed yet, it will launch the archiving process. - * If the current archive needs sub-archives (eg. a month archive needs all the days archive) - * it will recursively launch the archiving (using this loadArchive() on the sub-periods) - * - * @return int|false The idarchive of the archive, false if the archive is not archived yet - */ - public function loadArchive() - { - if ($this->debugAlwaysArchive) { - return false; - } - $this->idArchive = $this->isArchived(); - return $this->getIdArchive(); - } - - /** - * @see loadArchive() - */ - public function launchArchiving() - { - if (!Piwik::getArchiveProcessingLock($this->getSite()->getId(), $this->getPeriod(), $this->getSegment())) { - Piwik::log('Unable to get lock for idSite = ' . $this->getSite()->getId() - . ', period = ' . $this->getPeriod()->getLabel() - . ', UTC datetime [' . $this->getStartDatetimeUTC() . ' -> ' . $this->getEndDatetimeUTC() . ' ]...'); - return; - } - - $this->initCompute(); - $this->compute(); - $this->postCompute(); - - // we execute again the isArchived that does some initialization work - $this->idArchive = $this->isArchived(); - Piwik::releaseArchiveProcessingLock($this->getSite()->getId(), $this->getPeriod(), $this->getSegment()); - } - - /** - * Returns the name of the archive field used to tell the status of an archive, (ie, - * whether the archive was created successfully or not). - * - * @param bool $flagArchiveAsAllPlugins - * @return string - */ - public function getDoneStringFlag($flagArchiveAsAllPlugins = false) - { - return self::getDoneStringFlagFor( - $this->getSegment(), $this->getPeriod()->getLabel(), $this->getRequestedPlugin(), $flagArchiveAsAllPlugins); - } - - /** - * Returns the name of the archive field used to tell the status of an archive, (ie, - * whether the archive was created successfully or not). - * - * @param Piwik_Segment $segment - * @param string $periodLabel - * @param string $plugin - * @param bool $flagArchiveAsAllPlugins - * @return string - */ - public static function getDoneStringFlagFor($segment, $periodLabel, $plugin, $flagArchiveAsAllPlugins = false) - { - $segmentHash = $segment->getHash(); - if (!self::shouldProcessReportsAllPlugins($segment, $periodLabel)) { - if (!Piwik_PluginsManager::getInstance()->isPluginLoaded($plugin) - || $flagArchiveAsAllPlugins - ) { - $plugin = 'all'; - } - $segmentHash .= '.' . $plugin; - } - return 'done' . $segmentHash; - } - - /** - * Init the object before launching the real archive processing - */ - protected function initCompute() - { - $this->loadNextIdArchive(); - $done = $this->getDoneStringFlag(); - $this->insertNumericRecord($done, Piwik_ArchiveProcessing::DONE_ERROR); - - $temporary = 'definitive archive'; - if ($this->isArchiveTemporary()) { - $temporary = 'temporary archive'; - } - Piwik::log(sprintf("'%s, idSite = %d (%s), segment '%s', report = '%s', UTC datetime [%s -> %s]", - $this->getPeriod()->getLabel(), - $this->getSite()->getId(), - $temporary, - $this->getSegment()->getString(), - $this->getRequestedPlugin(), - $this->getStartDatetimeUTC(), - $this->getEndDatetimeUTC() - )); - } - - /** - * Post processing called at the end of the main archive processing. - * Makes sure the new archive is marked as "successful" in the DB - * - * We also try to delete some stuff from memory but really there is still a lot... - */ - protected function postCompute() - { - // delete the first done = ERROR - $done = $this->getDoneStringFlag(); - Piwik_Query("DELETE FROM " . $this->getTableArchiveNumericName() . " - WHERE idarchive = ? AND (name = '" . $done . "' OR name LIKE '" . self::PREFIX_SQL_LOCK . "%')", - array($this->getIdArchive()) - ); - - $flag = Piwik_ArchiveProcessing::DONE_OK; - if ($this->isArchiveTemporary()) { - $flag = Piwik_ArchiveProcessing::DONE_OK_TEMPORARY; - } - $this->insertNumericRecord($done, $flag); - } - - /** - * Returns the name of the numeric table where the archive numeric values are stored - * - * @return string - */ - public function getTableArchiveNumericName() - { - if(empty($this->tableArchiveNumeric)) { - $this->tableArchiveNumeric = new Piwik_TablePartitioning_Monthly('archive_numeric'); - $this->tableArchiveNumeric->setTimestamp($this->getPeriod()->getDateStart()->getTimestamp()); - } - return $this->tableArchiveNumeric->getTableName(); - } - - /** - * Returns the name of the blob table where the archive blob values are stored - * - * @return string - */ - public function getTableArchiveBlobName() - { - if(empty($this->tableArchiveBlob)) { - $this->tableArchiveBlob = new Piwik_TablePartitioning_Monthly('archive_blob'); - $this->tableArchiveBlob->setTimestamp($this->getPeriod()->getDateStart()->getTimestamp()); - } - return $this->tableArchiveBlob->getTableName(); - } - - - public function setRequestedPlugin($plugin) - { - $this->requestedPlugin = $plugin; - } - - protected function getRequestedPlugin() - { - return $this->requestedPlugin; - } - - // exposing the number of visits publicly (number used to compute conversions rates) - protected $nb_visits = null; - protected $nb_visits_converted = null; - - protected function setNumberOfVisits($nb_visits) - { - $this->nb_visits = $nb_visits; - } - - public function getNumberOfVisits() - { - return $this->nb_visits; - } - - protected function setNumberOfVisitsConverted($nb_visits_converted) - { - $this->nb_visits_converted = $nb_visits_converted; - } - - public function getNumberOfVisitsConverted() - { - return $this->nb_visits_converted; - } - - - /** - * Returns the idArchive we will use for the current archive - * - * @throws Exception - * @return int IdArchive to use when saving the current Archive - */ - protected function loadNextIdArchive() - { - $table = $this->getTableArchiveNumericName(); - $dbLockName = "loadNextIdArchive.$table"; - - $db = Zend_Registry::get('db'); - $locked = self::PREFIX_SQL_LOCK . Piwik_Common::generateUniqId(); - $date = date("Y-m-d H:i:s"); - - if (Piwik_GetDbLock($dbLockName, $maxRetries = 30) === false) { - throw new Exception("loadNextIdArchive: Cannot get named lock for table $table."); - } - $db->exec("INSERT INTO $table " - . " SELECT ifnull(max(idarchive),0)+1, - '" . $locked . "', - " . (int)$this->getSite()->getId() . ", - '" . $date . "', - '" . $date . "', - 0, - '" . $date . "', - 0 " - . " FROM $table as tb1"); - Piwik_ReleaseDbLock($dbLockName); - $id = $db->fetchOne("SELECT idarchive FROM $table WHERE name = ? LIMIT 1", $locked); - - $this->idArchive = $id; - } - - /** - * @param string $name - * @param int|float $value - */ - public function insertNumericRecord($name, $value) - { - $value = round($value, 2); - return $this->insertRecord($name, $value); - } - - public function insertNumericRecords($numericRecords) - { - foreach ($numericRecords as $name => $value) { - $this->insertNumericRecord($name, $value); - } - } - - /** - * @param string $name - * @param string|array $values - * @return bool|array - */ - public function insertBlobRecord($name, $values) - { - if (is_array($values)) { - $clean = array(); - foreach ($values as $id => $value) { - // for the parent Table we keep the name - // for example for the Table of searchEngines we keep the name 'referer_search_engine' - // but for the child table of 'Google' which has the ID = 9 the name would be 'referer_search_engine_9' - $newName = $name; - if ($id != 0) { - //FIXMEA: refactor - $newName = $name . '_' . $id; - } - - $value = $this->compress($value); - $clean[] = array($newName, $value); - } - return $this->insertBulkRecords($clean); - } - - $values = $this->compress($values); - - $this->insertRecord($name, $values); - return array($name => $values); - } - - protected function compress( $data) - { - if(Zend_Registry::get('db')->hasBlobDataType()) { - return gzcompress($data); - } - return $data; - } - - protected function insertBulkRecords($records) - { - // Using standard plain INSERT if there is only one record to insert - if ($DEBUG_DO_NOT_USE_BULK_INSERT = false - || count($records) == 1 - ) { - foreach ($records as $record) { - $this->insertRecord($record[0], $record[1]); - } - return; - } - $bindSql = $this->getBindArray(); - $values = array(); - - $valueSeen = false; - foreach ($records as $record) { - // don't record zero - if (empty($record[1])) continue; - - $bind = $bindSql; - $bind[] = $record[0]; // name - $bind[] = $record[1]; // value - $values[] = $bind; - - $valueSeen = $record[1]; - } - if (empty($values)) return; - - $tableName = $this->getTableNameToInsert($valueSeen); - - Piwik::tableInsertBatch($tableName, $this->getInsertFields(), $values); - return true; - } - - protected function getBindArray() - { - return array($this->getIdArchive(), - $this->getSite()->getId(), - $this->getPeriod()->getDateStart()->toString('Y-m-d'), - $this->getPeriod()->getDateEnd()->toString('Y-m-d'), - $this->getPeriod()->getId(), - date("Y-m-d H:i:s")); - } - - protected function getInsertFields() - { - return array('idarchive', 'idsite', 'date1', 'date2', 'period', 'ts_archived', 'name', 'value'); - } - - /** - * Inserts a record in the right table (either NUMERIC or BLOB) - * - * @param string $name - * @param mixed $value - * - * @return void - */ - protected function insertRecord($name, $value) - { - // We choose not to record records with a value of 0 - if ($value == 0) { - return; - } - $tableName = $this->getTableNameToInsert($value); - - // duplicate idarchives are Ignored, see http://dev.piwik.org/trac/ticket/987 - - $query = "INSERT IGNORE INTO " . $tableName . " - (" . implode(", ", $this->getInsertFields()) . ") - VALUES (?,?,?,?,?,?,?,?)"; - $bindSql = $this->getBindArray(); - $bindSql[] = $name; - $bindSql[] = $value; - Piwik_Query($query, $bindSql); - } - - protected function getTableNameToInsert($value) - { - if (is_numeric($value)) { - $tableName = $this->getTableArchiveNumericName(); - return $tableName; - } - $tableName = $this->getTableArchiveBlobName(); - return $tableName; - } - - /** - * Returns the idArchive if the archive is available in the database. - * Returns false if the archive needs to be computed. - * - * An archive is available if - * - for today, the archive was computed less than minDatetimeArchiveProcessedUTC seconds ago - * - for any other day, if the archive was computed once this day was finished - * - for other periods, if the archive was computed once the period was finished - * - * @return int|false - */ - protected function isArchived() - { - $bindSQL = array($this->getSite()->getId(), - $this->getPeriod()->getDateStart()->toString('Y-m-d'), - $this->getPeriod()->getDateEnd()->toString('Y-m-d'), - $this->getPeriod()->getId(), - ); - - $timeStampWhere = ''; - - $minDatetimeArchiveProcessedUTC = $this->getMinTimeArchivedProcessed(); - if ($minDatetimeArchiveProcessedUTC) { - $timeStampWhere = " AND ts_archived >= ? "; - $bindSQL[] = Piwik_Date::factory($minDatetimeArchiveProcessedUTC)->getDatetime(); - } - - // When a Segment is specified, we try and only process the requested report in the archive - // As a limitation, we don't know all the time which plugin should process which report - // There is a catch all flag 'all' appended to archives containing all reports already - // We look for this 'done.ABCDEFG.all', or for an archive that contains only our plugin data 'done.ABDCDEFG.Referers' - $done = $this->getDoneStringFlag(); - $doneAllPluginsProcessed = $this->getDoneStringFlag($flagArchiveAsAllPlugins = true); - - $sqlSegmentsFindArchiveAllPlugins = ''; - - if ($done != $doneAllPluginsProcessed) { - $sqlSegmentsFindArchiveAllPlugins = "OR (name = '" . $doneAllPluginsProcessed . "' AND value = " . Piwik_ArchiveProcessing::DONE_OK . ") - OR (name = '" . $doneAllPluginsProcessed . "' AND value = " . Piwik_ArchiveProcessing::DONE_OK_TEMPORARY . ")"; - } - $sqlQuery = " SELECT idarchive, value, name, date1 as startDate - FROM " . $this->getTableArchiveNumericName() . " - WHERE idsite = ? - AND date1 = ? - AND date2 = ? - AND period = ? - AND ( (name = '" . $done . "' AND value = " . Piwik_ArchiveProcessing::DONE_OK . ") - OR (name = '" . $done . "' AND value = " . Piwik_ArchiveProcessing::DONE_OK_TEMPORARY . ") - $sqlSegmentsFindArchiveAllPlugins - OR name = 'nb_visits') - $timeStampWhere - ORDER BY idarchive DESC"; - $results = Piwik_FetchAll($sqlQuery, $bindSQL); - if (empty($results)) { - return false; - } - - $idarchive = false; - // we look for the more recent idarchive - foreach ($results as $result) { - if ($result['name'] == $done - || $result['name'] == $doneAllPluginsProcessed - ) { - $idarchive = $result['idarchive']; - break; - } - } - - // case when we have a nb_visits entry in the archive, but the process is not finished yet or failed to finish - // therefore we don't have the done=OK - if ($idarchive === false) { - return false; - } - - if ($this->isVisitsSummaryRequested()) { - $this->isThereSomeVisits = false; - } - - // we look for the nb_visits result for this most recent archive - foreach ($results as $result) { - if ($result['name'] == 'nb_visits' - && $result['idarchive'] == $idarchive - ) { - $this->isThereSomeVisits = ($result['value'] > 0); - $this->setNumberOfVisits($result['value']); - break; - } - } - return $idarchive; - } - - protected function isVisitsSummaryRequested() - { - return $this->getRequestedPlugin() == 'VisitsSummary'; - } - /** - * Returns true if, for some reasons, triggering the archiving is disabled. - * Note that when a segment is passed to the function, archiving will always occur - * (since segments are by default not pre-processed) - * - * @return bool - */ - public function isArchivingDisabled() - { - return self::isArchivingDisabledFor($this->getSegment(), $this->getPeriod()->getLabel()); - } - - public static function isArchivingDisabledFor($segment, $periodLabel) - { - if ($periodLabel == 'range') { - return false; - } - $processOneReportOnly = !self::shouldProcessReportsAllPlugins($segment, $periodLabel); - $isArchivingDisabled = !self::isRequestAuthorizedToArchive(); - - if ($processOneReportOnly) { - // When there is a segment, archiving is not necessary allowed - // If browser archiving is allowed, then archiving is enabled - // if browser archiving is not allowed, then archiving is disabled - if (!$segment->isEmpty() - && $isArchivingDisabled - && Piwik_Config::getInstance()->General['browser_archiving_disabled_enforce'] - ) { - Piwik::log("Archiving is disabled because of config setting browser_archiving_disabled_enforce=1"); - return true; - } - return false; - } - return $isArchivingDisabled; - } - - protected static function isRequestAuthorizedToArchive() - { - return !self::$forceDisableArchiving && - (self::isBrowserTriggerArchivingEnabled() - || Piwik_Common::isPhpCliMode() - || (Piwik::isUserIsSuperUser() - && Piwik_Common::isArchivePhpTriggered())); - } - - - /** - * Returns true when - * - there is no segment and period is not range - * - there is a segment that is part of the preprocessed [Segments] list - - * @param Piwik_Segment $segment - * @param string $period - * @return bool - */ - private static function shouldProcessReportsAllPlugins($segment, $periodLabel) - { - if ($segment->isEmpty() && $periodLabel != 'range') { - return true; - } - - $segmentsToProcess = Piwik::getKnownSegmentsToArchive(); - if (!empty($segmentsToProcess)) { - // If the requested segment is one of the segments to pre-process - // we ensure that any call to the API will trigger archiving of all reports for this segment - $segment = $segment->getString(); - if (in_array($segment, $segmentsToProcess)) { - return true; - } - } - return false; - } - - // We check if there is visits for the requested date / site / segment - // If no specified Segment - // Or if a segment is passed and we specifically process VisitsSummary - // Then we check the logs. This is to ensure that this query is ran only once for this day/site/segment (rather than running it for every plugin) - protected function isProcessingEnabled() - { - return $this->shouldProcessReportsAllPlugins($this->getSegment(), $this->getPeriod()->getLabel()) - || $this->isVisitsSummaryRequested(); - } - - /** - * When a segment is set, we shall only process the requested report (no more). - * The requested data set will return a lot faster if we only process these reports rather than all plugins. - * Similarly, when a period=range is requested, we shall only process the requested report for the range itself. - * - * @param string $pluginName - * @return bool - */ - public function shouldProcessReportsForPlugin($pluginName) - { - if ($this->shouldProcessReportsAllPlugins($this->getSegment(), $this->getPeriod()->getLabel())) { - return true; - } - - // If any other segment, only process if the requested report belong to this plugin - // or process all plugins if the requested report plugin couldn't be guessed - $pluginBeingProcessed = $this->getRequestedPlugin(); - return $pluginBeingProcessed == $pluginName - || !Piwik_PluginsManager::getInstance()->isPluginLoaded($pluginBeingProcessed); - } - - /** - * Site of the current archive - * Can be accessed by plugins (that is why it's public) - * - * @var Piwik_Site - */ - private $site = null; - - - /** - * Period of the current archive - * Can be accessed by plugins (that is why it's public) - * - * @var Piwik_Period - */ - private $period = null; - - - - /** - * @var Piwik_Segment - */ - private $segment = null; - - - /** - * Set the period - * - * @param Piwik_Period $period - */ - public function setPeriod(Piwik_Period $period) - { - $this->period = $period; - } - - //FIXMEA remove - public function setSegment(Piwik_Segment $segment) - { - $this->segment = $segment; - } - - public function getSegment() - { - return $this->segment; - } - - //FIXMEA remove, only via constructor - /** - * Set the site - * - * @param Piwik_Site $site - */ - public function setSite(Piwik_Site $site) - { - $this->site = $site; - } - - /** - * @return Piwik_Site - */ - public function getSite() - { - return $this->site; - } - - public function getPeriod() - { - return $this->period; - } -} diff --git a/core/ArchiveProcessing/Period.php b/core/ArchiveProcessing/Period.php deleted file mode 100644 index 2293c6f2bd..0000000000 --- a/core/ArchiveProcessing/Period.php +++ /dev/null @@ -1,512 +0,0 @@ -<?php -/** - * Piwik - Open source web analytics - * - * @link http://piwik.org - * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later - * - * @category Piwik - * @package Piwik - */ - -/** - * Handles the archiving process for a period - * - * This class provides generic methods to archive data for a period (week / month / year). - * - * These methods are called by the plugins that do the logic of archiving their own data. \ - * They hook on the event 'ArchiveProcessing_Period.compute' - * - * @package Piwik - * @subpackage Piwik_ArchiveProcessing - */ -class Piwik_ArchiveProcessing_Period extends Piwik_ArchiveProcessing -{ - /** - * Array of (column name before => column name renamed) of the columns for which sum operation is invalid. - * The summed value is not accurate and these columns will be renamed accordingly. - * @var array - */ - static public $invalidSummedColumnNameToRenamedName = array( - Piwik_Archive::INDEX_NB_UNIQ_VISITORS => Piwik_Archive::INDEX_SUM_DAILY_NB_UNIQ_VISITORS - ); - - /** - * @var Piwik_Archive_Single[] - */ - //public $archives = array(); - - public $archive = null; - - /** - * Set the period - * - * @param Piwik_Period $period - */ - public function setPeriod( Piwik_Period $period ) - { - parent::setPeriod($period); - $this->resetSubperiodArchiveQuery(); - } - - /** - * Sets the segment. - * - * @param Piwik_Segment $segment - */ - public function setSegment( Piwik_Segment $segment) - { - parent::setSegment($segment); - $this->resetSubperiodArchiveQuery(); - } - - /** - * Set the site - * - * @param Piwik_Site $site - */ - public function setSite( Piwik_Site $site ) - { - parent::setSite($site); - $this->resetSubperiodArchiveQuery(); - } - - /** - * Sums all values for the given field names $aNames over the period - * See @archiveNumericValuesGeneral for more information - * - * @param string|array $aNames - * @return array - */ - public function archiveNumericValuesSum($aNames) - { - return $this->archiveNumericValuesGeneral($aNames, 'sum'); - } - - /** - * Get the maximum value for all values for the given field names $aNames over the period - * See @archiveNumericValuesGeneral for more information - * - * @param string|array $aNames - * @return array - */ - public function archiveNumericValuesMax($aNames) - { - return $this->archiveNumericValuesGeneral($aNames, 'max'); - } - - /** - * Given a list of fields names, the method will fetch all their values over the period, and archive them using the given operation. - * - * For example if $operationToApply = 'sum' and $aNames = array('nb_visits', 'sum_time_visit') - * it will sum all values of nb_visits for the period (for example give the number of visits for the month by summing the visits of every day) - * - * @param array|string $aNames Array of strings or string containg the field names to select - * @param string $operationToApply Available operations = sum, max, min - * @throws Exception - * @return array - */ - private function archiveNumericValuesGeneral($aNames, $operationToApply) - { - $this->loadSubPeriods(); - if (!is_array($aNames)) { - $aNames = array($aNames); - } - - // remove nb_uniq_visitors if present - foreach ($aNames as $i => $name) { - if ($name == 'nb_uniq_visitors') { - $results['nb_uniq_visitors'] = 0; - unset($aNames[$i]); - - break; - } - } - - // data will be array mapping each period w/ result row for period - $data = $this->archive->getNumeric($aNames); - foreach ($data as $dateRange => $row) { - foreach ($row as $name => $value) { - switch ($operationToApply) { - case 'sum': - if (!isset($results[$name])) { - $results[$name] = 0; - } - - $results[$name] += $value; - break; - case 'max': - if (!isset($results[$name])) { - $results[$name] = 0; - } - - $results[$name] = max($results[$name], $value); - break; - case 'min': - if (!isset($results[$name])) { - $results[$name] = $value; - } - - $results[$name] = min($results[$name], $value); - break; - default: - throw new Exception("Operation not applicable."); - break; - } - } - } - - // set default for metrics that weren't found - foreach ($aNames as $name) { - if (!isset($results[$name])) { - $results[$name] = 0; - } - } - - if (!Piwik::isUniqueVisitorsEnabled($this->getPeriod()->getLabel())) { - unset($results['nb_uniq_visitors']); - } - - foreach($results as $name => $value) { - if ($name == 'nb_uniq_visitors') { - $value = (float) $this->computeNbUniqVisitors(); - } - $this->insertRecord($name, $value); - } - - // if asked for only one field to sum - if (count($results) == 1) { - return reset($results); - } - - // returns the array of records once summed - return $results; - } - - /** - * This method will compute the sum of DataTables over the period for the given fields $aRecordName. - * The resulting DataTable will be then added to queue of data to be recorded in the database. - * It will usually be called in a plugin that listens to the hook 'ArchiveProcessing_Period.compute' - * - * For example if $aRecordName = 'UserCountry_country' the method will select all UserCountry_country DataTable for the period - * (eg. the 31 dataTable of the last month), sum them, then record it in the DB - * - * - * This method works on recursive dataTable. For example for the 'Actions' it will select all subtables of all dataTable of all the sub periods - * and get the sum. - * - * It returns an array that gives information about the "final" DataTable. The array gives for every field name, the number of rows in the - * final DataTable (ie. the number of distinct LABEL over the period) (eg. the number of distinct keywords over the last month) - * - * @param string|array $aRecordName Field name(s) of DataTable to select so we can get the sum - * @param array $invalidSummedColumnNameToRenamedName (current_column_name => new_column_name) for columns that must change names when summed - * (eg. unique visitors go from nb_uniq_visitors to sum_daily_nb_uniq_visitors) - * @param int $maximumRowsInDataTableLevelZero Max row count of parent datatable to archive - * @param int $maximumRowsInSubDataTable Max row count of children datatable(s) to archive - * @param string $columnToSortByBeforeTruncation Column name to sort by, before truncating rows (ie. if there are more rows than the specified max row count) - * @param array $columnAggregationOperations Operations for aggregating columns, @see Piwik_DataTable_Row::sumRow() - * - * @return array array ( - * nameTable1 => number of rows, - * nameTable2 => number of rows, - * ) - */ - public function archiveDataTable($aRecordName, - $invalidSummedColumnNameToRenamedName = null, - $maximumRowsInDataTableLevelZero = null, - $maximumRowsInSubDataTable = null, - $columnToSortByBeforeTruncation = null, - &$columnAggregationOperations = null) - { - // We clean up below all tables created during this function call (and recursive calls) - $latestUsedTableId = Piwik_DataTable_Manager::getInstance()->getMostRecentTableId(); - - $this->loadSubPeriods(); - if (!is_array($aRecordName)) { - $aRecordName = array($aRecordName); - } - - $nameToCount = array(); - foreach ($aRecordName as $recordName) { - $table = $this->getRecordDataTableSum($recordName, $invalidSummedColumnNameToRenamedName, $columnAggregationOperations); - - $nameToCount[$recordName]['level0'] = $table->getRowsCount(); - $nameToCount[$recordName]['recursive'] = $table->getRowsCountRecursive(); - - $blob = $table->getSerialized($maximumRowsInDataTableLevelZero, $maximumRowsInSubDataTable, $columnToSortByBeforeTruncation); - destroy($table); - $this->insertBlobRecord($recordName, $blob); - } - Piwik_DataTable_Manager::getInstance()->deleteAll($latestUsedTableId); - - return $nameToCount; - } - - /** - * This method selects all DataTables that have the name $name over the period. - * It calls the appropriate methods that sum all these tables together. - * The resulting DataTable is returned. - * - * @param string $name - * @param array $invalidSummedColumnNameToRenamedName columns in the array (old name, new name) to be renamed as the sum operation is not valid on them (eg. nb_uniq_visitors->sum_daily_nb_uniq_visitors) - * @param array $columnAggregationOperations Operations for aggregating columns, @see Piwik_DataTable_Row::sumRow() - * @return Piwik_DataTable - */ - protected function getRecordDataTableSum($name, $invalidSummedColumnNameToRenamedName, &$columnAggregationOperations = null) - { - $table = new Piwik_DataTable(); - if ($columnAggregationOperations !== null) { - $table->setColumnAggregationOperations($columnAggregationOperations); - } - - $data = $this->archive->getDataTableExpanded($name, $idSubTable = null, $addMetadataSubtableId = false); - foreach ($data->getArray() as $dateRange => $datatableToSum) - { - $table->addDataTable($datatableToSum); - } - - unset($data); - - if(is_null($invalidSummedColumnNameToRenamedName)) - { - $invalidSummedColumnNameToRenamedName = self::$invalidSummedColumnNameToRenamedName; - } - foreach($invalidSummedColumnNameToRenamedName as $oldName => $newName) - { - $table->renameColumn($oldName, $newName); - } - return $table; - } - - protected function initCompute() - { - parent::initCompute(); - } - - /** - * Returns an archive instance that can be used to query for data in each - * subperiod of the period we're archiving data for. - * - * @return Piwik_Archive - */ - protected function loadSubperiodsArchive() - { - $params = new Piwik_Archive_Parameters(); - $params->setSegment( $this->getSegment() ); - $params->setIdSites( $this->getSite()->getId() ); - $params->setPeriods( $this->getPeriod()->getSubperiods() ); - - return new Piwik_Archive( - $params, - $forceIndexedBySite = false, - $forceIndexedByDate = true - ); - } - - /** - * Main method to process logs for a period. - * The only logic done here is computing the number of visits, actions, etc. - * - * All the other reports are computed inside plugins listening to the event 'ArchiveProcessing_Period.compute'. - * See some of the plugins for an example. - */ - protected function compute() - { - if (!$this->isThereSomeVisits()) { - return; - } - Piwik_PostEvent('ArchiveProcessing_Period.compute', $this); - } - - protected function loadSubPeriods() - { - if(is_null($this->archive)) - { - $this->archive = $this->loadSubperiodsArchive(); - } - } - - /** - * - * @see Piwik_ArchiveProcessing_Day::isThereSomeVisits() - * @return bool|null - */ - public function isThereSomeVisits() - { - if (!is_null($this->isThereSomeVisits)) { - return $this->isThereSomeVisits; - } - - $this->loadSubPeriods(); - if ($this->isProcessingEnabled()) { - $toSum = self::getCoreMetrics(); - $record = $this->archiveNumericValuesSum($toSum); - $this->archiveNumericValuesMax('max_actions'); - - if (!isset($record['nb_visits'])) { - $nbVisits = $nbVisitsConverted = 0; - } else { - $nbVisitsConverted = $record['nb_visits_converted']; - $nbVisits = $record['nb_visits']; - } - } else { - - $archive = $this->makeNewArchive(); - - $metrics = $archive->getNumeric(array('nb_visits', 'nb_visits_converted')); - if (!isset($metrics['nb_visits'])) { - $nbVisits = $nbVisitsConverted = 0; - } else { - $nbVisits = $metrics['nb_visits']; - $nbVisitsConverted = $metrics['nb_visits_converted']; - } - } - - $this->setNumberOfVisits($nbVisits); - $this->setNumberOfVisitsConverted($nbVisitsConverted); - $this->isThereSomeVisits = ($nbVisits > 0); - return $this->isThereSomeVisits; - } - - - /** - * Processes number of unique visitors for the given period - * - * This is the only metric we process from the logs directly, - * since unique visitors cannot be summed like other metrics. - * - * @return int - */ - protected function computeNbUniqVisitors() - { - $select = "count(distinct log_visit.idvisitor) as nb_uniq_visitors"; - $from = "log_visit"; - $where = "log_visit.visit_last_action_time >= ? - AND log_visit.visit_last_action_time <= ? - AND log_visit.idsite = ?"; - - $bind = array($this->getStartDatetimeUTC(), $this->getEndDatetimeUTC(), $this->getSite()->getId()); - - $query = $this->getSegment()->getSelectQuery($select, $from, $where, $bind); - - return Zend_Registry::get('db')->fetchOne($query['sql'], $query['bind']); - } - - /** - * Called at the end of the archiving process. - * Does some cleaning job in the database. - */ - protected function postCompute() - { - parent::postCompute(); - - self::doPurgeOutdatedArchives($this->getTableArchiveNumericName(), $this->isArchiveTemporary()); - - $this->resetSubperiodArchiveQuery(); - } - - const FLAG_TABLE_PURGED = 'lastPurge_'; - - // Used to disable Purge Outdated reports during test data setup - static public $enablePurgeOutdated = true; - - /** - * Given a monthly archive table, will delete all reports that are now outdated, - * or reports that ended with an error - */ - static public function doPurgeOutdatedArchives($numericTable) - { - if (!self::$enablePurgeOutdated) { - return; - } - $blobTable = str_replace("numeric", "blob", $numericTable); - $key = self::FLAG_TABLE_PURGED . $blobTable; - $timestamp = Piwik_GetOption($key); - - // we shall purge temporary archives after their timeout is finished, plus an extra 6 hours - // in case archiving is disabled or run once a day, we give it this extra time to run - // and re-process more recent records... - // TODO: Instead of hardcoding 6 we should put the actual number of hours between 2 archiving runs - $temporaryArchivingTimeout = self::getTodayArchiveTimeToLive(); - $purgeEveryNSeconds = max($temporaryArchivingTimeout, 6 * 3600); - - // we only delete archives if we are able to process them, otherwise, the browser might process reports - // when &segment= is specified (or custom date range) and would below, delete temporary archives that the - // browser is not able to process until next cron run (which could be more than 1 hour away) - if (self::isRequestAuthorizedToArchive() - && (!$timestamp - || $timestamp < time() - $purgeEveryNSeconds) - ) { - Piwik_SetOption($key, time()); - - // If Browser Archiving is enabled, it is likely there are many more temporary archives - // We delete more often which is safe, since reports are re-processed on demand - if (self::isBrowserTriggerArchivingEnabled()) { - $purgeArchivesOlderThan = Piwik_Date::factory(time() - 2 * $temporaryArchivingTimeout)->getDateTime(); - } // If archive.php via Cron is building the reports, we should keep all temporary reports from today - else { - $purgeArchivesOlderThan = Piwik_Date::factory('today')->getDateTime(); - } - $result = Piwik_FetchAll(" - SELECT idarchive - FROM $numericTable - WHERE name LIKE 'done%' - AND (( value = " . Piwik_ArchiveProcessing::DONE_OK_TEMPORARY . " - AND ts_archived < ?) - OR value = " . Piwik_ArchiveProcessing::DONE_ERROR . ")", - array($purgeArchivesOlderThan) - ); - - $idArchivesToDelete = array(); - if (!empty($result)) { - foreach ($result as $row) { - $idArchivesToDelete[] = $row['idarchive']; - } - $query = "DELETE - FROM %s - WHERE idarchive IN (" . implode(',', $idArchivesToDelete) . ") - "; - - Piwik_Query(sprintf($query, $numericTable)); - - // Individual blob tables could be missing - try { - Piwik_Query(sprintf($query, $blobTable)); - } catch (Exception $e) { - } - } - Piwik::log("Purging temporary archives: done [ purged archives older than $purgeArchivesOlderThan from $blobTable and $numericTable ] [Deleted IDs: " . implode(',', $idArchivesToDelete) . "]"); - - // Deleting "Custom Date Range" reports after 1 day, since they can be re-processed - // and would take up unecessary space - $yesterday = Piwik_Date::factory('yesterday')->getDateTime(); - $query = "DELETE - FROM %s - WHERE period = ? - AND ts_archived < ?"; - $bind = array(Piwik::$idPeriods['range'], $yesterday); - Piwik::log("Purging Custom Range archives: done [ purged archives older than $yesterday from $blobTable and $numericTable ]"); - - Piwik_Query(sprintf($query, $numericTable), $bind); - - // Individual blob tables could be missing - try { - Piwik_Query(sprintf($query, $blobTable), $bind); - } catch (Exception $e) { - } - - // these tables will be OPTIMIZEd daily in a scheduled task, to claim lost space - } else { - Piwik::log("Purging temporary archives: skipped."); - } - } - - //FIXMEA - private function resetSubperiodArchiveQuery() - { - if ($this->archive !== null) { - destroy($this->archive); - $this->archive = null; - } - } -} diff --git a/core/ArchiveProcessing/Day.php b/core/ArchiveProcessor/Day.php index b662ca54d0..ccef3d5b2e 100644 --- a/core/ArchiveProcessing/Day.php +++ b/core/ArchiveProcessor/Day.php @@ -17,12 +17,14 @@ * All the logic of the archiving is done inside the plugins listening to the event 'ArchiveProcessing_Day.compute' * * @package Piwik - * @subpackage Piwik_ArchiveProcessing + * @subpackage Piwik_ArchiveProcessor */ -class Piwik_ArchiveProcessing_Day extends Piwik_ArchiveProcessing +class Piwik_ArchiveProcessor_Day extends Piwik_ArchiveProcessor { - const FIELDS_SEPARATOR = ", \n\t\t\t"; + const LOG_VISIT_TABLE = 'log_visit'; + const LOG_ACTIONS_TABLE = 'log_link_visit_action'; const LOG_CONVERSION_TABLE = "log_conversion"; + const REVENUE_SUBTOTAL_FIELD = 'revenue_subtotal'; const REVENUE_TAX_FIELD = 'revenue_tax'; const REVENUE_SHIPPING_FIELD = 'revenue_shipping'; @@ -30,16 +32,12 @@ class Piwik_ArchiveProcessing_Day extends Piwik_ArchiveProcessing const TOTAL_REVENUE_FIELD = 'revenue'; const ITEMS_COUNT_FIELD = "items"; - const IDGOAL_FIELD = 'idgoal'; - const CONVERSION_DATETIME_FIELD = "server_time"; const ACTION_DATETIME_FIELD = "server_time"; - const VISIT_DATETIME_FIELD = 'visit_last_action_time'; + const IDGOAL_FIELD = 'idgoal'; - const LOG_ACTIONS_TABLE = 'log_link_visit_action'; - - const LOG_VISIT_TABLE = 'log_visit'; + const FIELDS_SEPARATOR = ", \n\t\t\t"; public function queryActionsByDimension($dimensions, $where = '', $additionalSelects = array(), $metrics = false, $rankingQuery = null, $joinLogActionOnColumn = false) { @@ -103,12 +101,16 @@ class Piwik_ArchiveProcessing_Day extends Piwik_ArchiveProcessing protected function query($select, $from, $where, $groupBy, $orderBy) { - $bind = array($this->getStartDatetimeUTC(), $this->getEndDatetimeUTC(), $this->getSite()->getId()); - + $bind = $this->getBindDatetimeSite(); $query = $this->getSegment()->getSelectQuery($select, $from, $where, $bind, $orderBy, $groupBy); return $query; } + protected function getBindDatetimeSite() + { + return array($this->getDateStart()->getDateStartUTC(), $this->getDateEnd()->getDateEndUTC(), $this->getSite()->getId()); + } + /** * @see queryVisitsByDimension() Similar to this function, * but queries metrics for the requested dimensionRecord, @@ -193,10 +195,7 @@ class Piwik_ArchiveProcessing_Day extends Piwik_ArchiveProcessing // Segment not supported yet // $query = $this->query($select, $from, $where, $groupBy, $orderBy); - $bind = array($this->getStartDatetimeUTC(), - $this->getEndDatetimeUTC(), - $this->getSite()->getId() - ); + $bind = $this->getBindDatetimeSite(); $query = $this->getDb()->query($query, $bind); return $query; } @@ -280,44 +279,35 @@ class Piwik_ArchiveProcessing_Day extends Piwik_ArchiveProcessing return $metrics; } - /** - * Returns true if there are logs for the current archive. - * - * If the current archive is for a specific plugin (for example, Referers), - * (for example when a Segment is defined and the Keywords report is requested) - * Then the function will create the Archive for the Core metrics 'VisitsSummary' which will in turn process the number of visits - * - * If there is no specified segment, the SQL query will always run. - * - * @return bool|null - */ - public function isThereSomeVisits() - { - if (!is_null($this->isThereSomeVisits)) { - return $this->isThereSomeVisits; - } - if (!$this->isProcessingEnabled()) { - return $this->makeArchiveToCheckForVisits(); - } + protected function aggregateCoreVisitsMetrics() + { $query = $this->queryVisitsByDimension(); $data = $query->fetch(); - // no visits found - if (!is_array($data) || $data[Piwik_Archive::INDEX_NB_VISITS] == 0) { - return $this->isThereSomeVisits = false; + if (empty($data[Piwik_Archive::INDEX_NB_VISITS])) { + $this->setNumberOfVisits(false); } + $metrics = $this->convertMetricsIdToName($data); + $this->setNumberOfVisits($metrics['nb_visits'], $metrics['nb_visits_converted']); + + $this->insertNumericRecords($metrics); + return $metrics; + } + + /** + * @param $data + * @return array + */ + protected function convertMetricsIdToName($data) + { $metrics = array(); - foreach($data as $metricId => $value) { + foreach ($data as $metricId => $value) { $readableMetric = Piwik_Archive::$mappingFromIdToName[$metricId]; $metrics[$readableMetric] = $value; } - $this->insertNumericRecords($metrics); - - $this->setNumberOfVisits($data[Piwik_Archive::INDEX_NB_VISITS]); - $this->setNumberOfVisitsConverted($data[Piwik_Archive::INDEX_NB_VISITS_CONVERTED]); - return $this->isThereSomeVisits = true; + return $metrics; } /** @@ -468,41 +458,11 @@ class Piwik_ArchiveProcessing_Day extends Piwik_ArchiveProcessing return $dimensions; } - /** - * Main method to process logs for a day. The only logic done here is computing the number of visits, actions, etc. - * All the other reports are computed inside plugins listening to the event 'ArchiveProcessing_Day.compute'. - * See some of the plugins for an example eg. 'Provider' - */ protected function compute() { - if (!$this->isThereSomeVisits()) { - return; - } Piwik_PostEvent('ArchiveProcessing_Day.compute', $this); } - /** - * If a segment is specified but a plugin other than 'VisitsSummary' is being requested, - * we create an archive for processing VisitsSummary Core Metrics, which will in turn - * execute the query above (in isThereSomeVisits) - * - * @return bool|null - */ - private function makeArchiveToCheckForVisits() - { - $archive = $this->makeNewArchive(); - $nbVisits = $archive->getNumeric('nb_visits'); - $this->isThereSomeVisits = $nbVisits > 0; - - if ($this->isThereSomeVisits) { - $nbVisitsConverted = $archive->getNumeric('nb_visits_converted'); - $this->setNumberOfVisits($nbVisits); - $this->setNumberOfVisitsConverted($nbVisitsConverted); - } - - return $this->isThereSomeVisits; - } - public function getDb() { return Zend_Registry::get('db'); diff --git a/core/DataAccess/ArchiveQuery.php b/core/DataAccess/Archiver.php index b56e58acad..e01605676c 100644 --- a/core/DataAccess/ArchiveQuery.php +++ b/core/DataAccess/Archiver.php @@ -10,10 +10,104 @@ */ /** - * Data Access object used to query archive data. + * Data Access object used to query archives, create new archives, and insert data for them. */ -class Piwik_DataAccess_ArchiveQuery +class Piwik_DataAccess_Archiver { + public static function deleteTemporaryLockedArchive($numericTable, $requestedPlugin, $segment, $period, $idArchive) + { + $done = Piwik_ArchiveProcessor_Rules::getDoneStringFlagFor($segment, $period->getLabel(), $requestedPlugin); + Piwik_Query("DELETE FROM " . $numericTable . " + WHERE idarchive = ? AND (name = '" . $done . "' OR name LIKE '" . self::PREFIX_SQL_LOCK . "%')", + array($idArchive) + ); + return $done; + } + + /** + * Generate advisory lock name + * + * @param int $idsite + * @param Piwik_Period $period + * @param Piwik_Segment $segment + * @return string + */ + static protected function getArchiveProcessingLockName($idsite, $period, Piwik_Segment $segment) + { + $config = Piwik_Config::getInstance(); + + $lockName = 'piwik.' + . $config->database['dbname'] . '.' + . $config->database['tables_prefix'] . '/' + . $idsite . '/' + . (!$segment->isEmpty() ? $segment->getHash() . '/' : '') + . $period->getId() . '/' + . $period->getDateStart()->toString('Y-m-d') . ',' + . $period->getDateEnd()->toString('Y-m-d'); + return $lockName . '/' . md5($lockName . Piwik_Common::getSalt()); + } + + /** + * Get an advisory lock + * + * @param int $idsite + * @param Piwik_Period $period + * @param Piwik_Segment $segment + * @return bool True if lock acquired; false otherwise + */ + static public function getArchiveProcessingLock($idsite, $period, $segment) + { + $lockName = self::getArchiveProcessingLockName($idsite, $period, $segment); + return Piwik_GetDbLock($lockName, $maxRetries = 30); + } + + /** + * Release an advisory lock + * + * @param int $idsite + * @param Piwik_Period $period + * @param Piwik_Segment $segment + * @return bool True if lock released; false otherwise + */ + static public function releaseArchiveProcessingLock($idsite, $period, $segment) + { + $lockName = self::getArchiveProcessingLockName($idsite, $period, $segment); + return Piwik_ReleaseDbLock($lockName); + } + + /** + * A row is created to lock an idarchive for the current archive being processed + * @var string + */ + const PREFIX_SQL_LOCK = "locked_"; + + + public static function allocateNewArchiveId($table, $idSite) + { + $dbLockName = "allocateNewArchiveId.$table"; + + $db = Zend_Registry::get('db'); + $locked = self::PREFIX_SQL_LOCK . Piwik_Common::generateUniqId(); + $date = date("Y-m-d H:i:s"); + + if (Piwik_GetDbLock($dbLockName, $maxRetries = 30) === false) { + throw new Exception("allocateNewArchiveId: Cannot get named lock for table $table."); + } + $db->exec("INSERT INTO $table " + . " SELECT ifnull(max(idarchive),0)+1, + '" . $locked . "', + " . (int)$idSite . ", + '" . $date . "', + '" . $date . "', + 0, + '" . $date . "', + 0 " + . " FROM $table as tb1"); + Piwik_ReleaseDbLock($dbLockName); + $id = $db->fetchOne("SELECT idarchive FROM $table WHERE name = ? LIMIT 1", $locked); + return $id; + } + /** * Queries and returns archive IDs for a set of sites, periods, and a segment. * @@ -31,7 +125,7 @@ class Piwik_DataAccess_ArchiveQuery public function getArchiveIds($siteIds, $periods, $segment, $plugins) { $periodType = reset($periods)->getLabel(); - + $getArchiveIdsSql = "SELECT idsite, name, date1, date2, MAX(idarchive) as idarchive FROM %s WHERE period = ? @@ -45,8 +139,7 @@ class Piwik_DataAccess_ArchiveQuery foreach ($this->getPeriodsByTableMonth($periods) as $tableMonth => $periods) { $firstPeriod = reset($periods); $table = Piwik_Common::prefixTable("archive_numeric_$tableMonth"); -// echo $periods; - + Piwik_TablePartitioning_Monthly::createArchiveTablesIfAbsent($firstPeriod); // if looking for a range archive. NOTE: we assume there's only one period if its a range. @@ -122,7 +215,6 @@ class Piwik_DataAccess_ArchiveQuery $table = Piwik_Common::prefixTable($archiveTableType."_".$tableMonth); $sql = sprintf($getValuesSql, $table, implode(',', $ids)); - foreach (Piwik_FetchAll($sql, $bind) as $row) { $rows[] = $row; } @@ -146,8 +238,8 @@ class Piwik_DataAccess_ArchiveQuery // if it was completed $doneFlags = array(); foreach ($plugins as $plugin) { - $done = Piwik_ArchiveProcessing::getDoneStringFlagFor($segment, $periodType, $plugin); - $donePlugins = Piwik_ArchiveProcessing::getDoneStringFlagFor($segment, $periodType, $plugin, true); + $done = Piwik_ArchiveProcessor_Rules::getDoneStringFlagFor($segment, $periodType, $plugin); + $donePlugins = Piwik_ArchiveProcessor_Rules::getDoneStringFlagFor($segment, $periodType, $plugin, true); $doneFlags[$done] = $done; $doneFlags[$donePlugins] = $donePlugins; @@ -157,8 +249,8 @@ class Piwik_DataAccess_ArchiveQuery // create the SQL to find archives that are DONE return "(name IN ($allDoneFlags)) AND - (value = '".Piwik_ArchiveProcessing::DONE_OK."' OR - value = '".Piwik_ArchiveProcessing::DONE_OK_TEMPORARY."')"; + (value = '".Piwik_ArchiveProcessor::DONE_OK."' OR + value = '".Piwik_ArchiveProcessor::DONE_OK_TEMPORARY."')"; } /** @@ -191,4 +283,55 @@ class Piwik_DataAccess_ArchiveQuery { return str_replace('-', '_', substr($dateRange, 0, 7)); } + + static public function purgeOutdatedArchives($numericTable, $blobTable, $purgeArchivesOlderThan) + { + $result = Piwik_FetchAll(" + SELECT idarchive + FROM $numericTable + WHERE name LIKE 'done%' + AND (( value = " . Piwik_ArchiveProcessor::DONE_OK_TEMPORARY . " + AND ts_archived < ?) + OR value = " . Piwik_ArchiveProcessor::DONE_ERROR . ")", + array($purgeArchivesOlderThan) + ); + + $idArchivesToDelete = array(); + if (!empty($result)) { + foreach ($result as $row) { + $idArchivesToDelete[] = $row['idarchive']; + } + $query = "DELETE + FROM %s + WHERE idarchive IN (" . implode(',', $idArchivesToDelete) . ") + "; + + Piwik_Query(sprintf($query, $numericTable)); + + // Individual blob tables could be missing + try { + Piwik_Query(sprintf($query, $blobTable)); + } catch (Exception $e) { + } + } + Piwik::log("Purging temporary archives: done [ purged archives older than $purgeArchivesOlderThan from $blobTable and $numericTable ] [Deleted IDs: " . implode(',', $idArchivesToDelete) . "]"); + + // Deleting "Custom Date Range" reports after 1 day, since they can be re-processed + // and would take up unecessary space + $yesterday = Piwik_Date::factory('yesterday')->getDateTime(); + $query = "DELETE + FROM %s + WHERE period = ? + AND ts_archived < ?"; + $bind = array(Piwik::$idPeriods['range'], $yesterday); + Piwik::log("Purging Custom Range archives: done [ purged archives older than $yesterday from $blobTable and $numericTable ]"); + + Piwik_Query(sprintf($query, $numericTable), $bind); + + // Individual blob tables could be missing + try { + Piwik_Query(sprintf($query, $blobTable), $bind); + } catch (Exception $e) { + } + } } diff --git a/core/Piwik.php b/core/Piwik.php index eca11f1da2..15fbcee79a 100644 --- a/core/Piwik.php +++ b/core/Piwik.php @@ -2425,57 +2425,6 @@ class Piwik } /** - * Generate advisory lock name - * - * @param int $idsite - * @param Piwik_Period $period - * @param Piwik_Segment $segment - * @return string - */ - static public function getArchiveProcessingLockName($idsite, $period, Piwik_Segment $segment) - { - $config = Piwik_Config::getInstance(); - - $lockName = 'piwik.' - . $config->database['dbname'] . '.' - . $config->database['tables_prefix'] . '/' - . $idsite . '/' - . (!$segment->isEmpty() ? $segment->getHash() . '/' : '') - . $period->getId() . '/' - . $period->getDateStart()->toString('Y-m-d') . ',' - . $period->getDateEnd()->toString('Y-m-d'); - return $lockName . '/' . md5($lockName . Piwik_Common::getSalt()); - } - - /** - * Get an advisory lock - * - * @param int $idsite - * @param Piwik_Period $period - * @param Piwik_Segment $segment - * @return bool True if lock acquired; false otherwise - */ - static public function getArchiveProcessingLock($idsite, $period, $segment) - { - $lockName = self::getArchiveProcessingLockName($idsite, $period, $segment); - return Piwik_GetDbLock($lockName, $maxRetries = 30); - } - - /** - * Release an advisory lock - * - * @param int $idsite - * @param Piwik_Period $period - * @param Piwik_Segment $segment - * @return bool True if lock released; false otherwise - */ - static public function releaseArchiveProcessingLock($idsite, $period, $segment) - { - $lockName = self::getArchiveProcessingLockName($idsite, $period, $segment); - return Piwik_ReleaseDbLock($lockName); - } - - /** * Cached result of isLockprivilegeGranted function. * * Public so tests can simulate the situation where the lock tables privilege isn't granted. diff --git a/core/PluginsArchiver.php b/core/PluginsArchiver.php index dca85a7c77..89fd74453a 100644 --- a/core/PluginsArchiver.php +++ b/core/PluginsArchiver.php @@ -17,7 +17,7 @@ abstract class Piwik_PluginsArchiver { protected $processor; - public function __construct(Piwik_ArchiveProcessing $processing) + public function __construct(Piwik_ArchiveProcessor $processing) { $this->maximumRows = Piwik_Config::getInstance()->General['datatable_archiving_maximum_rows_standard']; $this->processor = $processing; @@ -36,7 +36,7 @@ abstract class Piwik_PluginsArchiver } /** - * @return Piwik_ArchiveProcessing_Day|Piwik_ArchiveProcessing_Period + * @return Piwik_ArchiveProcessor_Day|Piwik_ArchiveProcessor_Period */ protected function getProcessor() { diff --git a/core/Tracker/Cache.php b/core/Tracker/Cache.php index b537cc31a8..29ffe5cce2 100644 --- a/core/Tracker/Cache.php +++ b/core/Tracker/Cache.php @@ -94,7 +94,7 @@ class Piwik_Tracker_Cache Piwik_Tracker::initCorePiwikInTrackerMode(); $cacheContent = array( - 'isBrowserTriggerArchivingEnabled' => Piwik_ArchiveProcessing::isBrowserTriggerArchivingEnabled(), + 'isBrowserTriggerEnabled' => Piwik_ArchiveProcessor_Rules::isBrowserTriggerEnabled(), 'lastTrackerCronRun' => Piwik_GetOption('lastTrackerCronRun'), 'currentLocationProviderId' => Piwik_UserCountry_LocationProvider::getCurrentProviderId(), ); diff --git a/misc/cron/archive.php b/misc/cron/archive.php index 88c7b1924f..6d1d514a85 100644 --- a/misc/cron/archive.php +++ b/misc/cron/archive.php @@ -125,8 +125,8 @@ class Archiving $this->log("Notes"); // Information about timeout - $this->todayArchiveTimeToLive = Piwik_ArchiveProcessing::getTodayArchiveTimeToLive(); - $this->log("- Reports for today will be processed at most every " . Piwik_ArchiveProcessing::getTodayArchiveTimeToLive() + $this->todayArchiveTimeToLive = Piwik_ArchiveProcessor_Rules::getTodayArchiveTimeToLive(); + $this->log("- Reports for today will be processed at most every " . $this->todayArchiveTimeToLive . " seconds. You can change this value in Piwik UI > Settings > General Settings."); $this->log("- Reports for the current week/month/year will be refreshed at most every " . $this->processPeriodsMaximumEverySeconds . " seconds."); @@ -272,11 +272,14 @@ class Archiving continue; } $visitsToday = end($response); + if(empty($visitsToday)) { + $visitsToday = 0; + } $this->requests++; $processed++; // If there is no visit today and we don't need to process this website, we can skip remaining archives - if ($visitsToday <= 0 + if ($visitsToday == 0 && !$shouldArchivePeriods ) { $this->log("Skipped website id $idsite, no visit today, " . $timerWebsite->__toString()); @@ -667,7 +670,7 @@ class Archiving ) // in case --force-timeout-for-periods= without [seconds] specified { // Ensure the cache for periods is at least as high as cache for today - $todayTTL = Piwik_ArchiveProcessing::getTodayArchiveTimeToLive(); + $todayTTL = Piwik_ArchiveProcessor_Rules::getTodayArchiveTimeToLive(); if ($forceTimeoutPeriod < $todayTTL) { $this->log("WARNING: Automatically increasing --force-timeout-for-periods from $forceTimeoutPeriod to " . $todayTTL @@ -678,7 +681,7 @@ class Archiving } // Recommend to disable browser archiving when using this script - if (Piwik_ArchiveProcessing::isBrowserTriggerArchivingEnabled()) { + if (Piwik_ArchiveProcessor_Rules::isBrowserTriggerEnabled()) { $this->log("NOTE: if you execute this script at least once per hour (or more often) in a crontab, you may disable 'Browser trigger archiving' in Piwik UI > Settings > General Settings. "); $this->log(" see doc at: http://piwik.org/docs/setup-auto-archiving/"); } diff --git a/plugins/Actions/Actions.php b/plugins/Actions/Actions.php index d54a11c65d..7d401b3c59 100644 --- a/plugins/Actions/Actions.php +++ b/plugins/Actions/Actions.php @@ -573,7 +573,7 @@ class Piwik_Actions extends Piwik_Plugin */ public function archiveDay($notification) { - /* @var $archiveProcessing Piwik_ArchiveProcessing_Day */ + /* @var $archiveProcessing Piwik_ArchiveProcessor_Day */ $archiveProcessing = $notification->getNotificationObject(); $archiving = new Piwik_Actions_Archiver($archiveProcessing); diff --git a/plugins/Actions/Archiver.php b/plugins/Actions/Archiver.php index 1a8e902f16..d7e8e93b23 100644 --- a/plugins/Actions/Archiver.php +++ b/plugins/Actions/Archiver.php @@ -66,7 +66,7 @@ class Piwik_Actions_Archiver extends Piwik_PluginsArchiver /** * Archives Actions reports for a Day * - * @param Piwik_ArchiveProcessing $this->getProcessor() + * @param Piwik_ArchiveProcessor $this->getProcessor() * @return bool */ public function archiveDay() @@ -252,20 +252,15 @@ class Piwik_Actions_Archiver extends Piwik_PluginsArchiver // to the outer select. therefore, $segment needs to know about it. $select = sprintf($select, $sprintfField); - $bind = array(); + $bind = array($this->getProcessor()->getDateStart()->getDateStartUTC(), + $this->getProcessor()->getDateEnd()->getDateEndUTC(), + $this->getProcessor()->getSite()->getId() + ); // get query with segmentation $query = $this->getProcessor()->getSegment()->getSelectQuery( $select, $from, $where, $bind, $orderBy, $groupBy); - // extend bindings - $bind = array_merge(array($this->getProcessor()->getStartDatetimeUTC(), - $this->getProcessor()->getEndDatetimeUTC(), - $this->getProcessor()->getSite()->getId() - ), - $query['bind'] - ); - // replace the rest of the %s $querySql = str_replace("%s", $sprintfField, $query['sql']); @@ -562,25 +557,27 @@ class Piwik_Actions_Archiver extends Piwik_PluginsArchiver self::PAGE_TITLES_RECORD_NAME, self::PAGE_URLS_RECORD_NAME, ); - $this->getProcessor()->archiveDataTable($dataTableToSum, - self::$invalidSummedColumnNameToRenamedNameFromPeriodArchive, + $this->getProcessor()->aggregateDataTableReports($dataTableToSum, Piwik_Actions_ArchivingHelper::$maximumRowsInDataTableLevelZero, Piwik_Actions_ArchivingHelper::$maximumRowsInSubDataTable, Piwik_Actions_ArchivingHelper::$columnToSortByBeforeTruncation, - self::$actionColumnAggregationOperations); + self::$actionColumnAggregationOperations, + self::$invalidSummedColumnNameToRenamedNameFromPeriodArchive + ); $dataTableToSum = array( self::DOWNLOADS_RECORD_NAME, self::OUTLINKS_RECORD_NAME, self::SITE_SEARCH_RECORD_NAME, ); - $nameToCount = $this->getProcessor()->archiveDataTable($dataTableToSum, - self::$invalidSummedColumnNameToRenamedNameFromPeriodArchive, + $nameToCount = $this->getProcessor()->aggregateDataTableReports($dataTableToSum, Piwik_Actions_ArchivingHelper::$maximumRowsInDataTableLevelZero, Piwik_Actions_ArchivingHelper::$maximumRowsInSubDataTable, - Piwik_Actions_ArchivingHelper::$columnToSortByBeforeTruncation); + Piwik_Actions_ArchivingHelper::$columnToSortByBeforeTruncation, + $aggregation = null, + self::$invalidSummedColumnNameToRenamedNameFromPeriodArchive); - $this->getProcessor()->archiveNumericValuesSum(array( + $this->getProcessor()->archiveNumericValuesGeneral(array( self::METRIC_PAGEVIEWS_RECORD_NAME, self::METRIC_UNIQ_PAGEVIEWS_RECORD_NAME, self::METRIC_DOWNLOADS_RECORD_NAME, diff --git a/plugins/CoreAdminHome/Controller.php b/plugins/CoreAdminHome/Controller.php index c915205bc2..e002157276 100644 --- a/plugins/CoreAdminHome/Controller.php +++ b/plugins/CoreAdminHome/Controller.php @@ -29,8 +29,8 @@ class Piwik_CoreAdminHome_Controller extends Piwik_Controller_Admin $view = Piwik_View::factory('generalSettings'); if (Piwik::isUserIsSuperUser()) { - $enableBrowserTriggerArchiving = Piwik_ArchiveProcessing::isBrowserTriggerArchivingEnabled(); - $todayArchiveTimeToLive = Piwik_ArchiveProcessing::getTodayArchiveTimeToLive(); + $enableBrowserTriggerArchiving = Piwik_ArchiveProcessor_Rules::isBrowserTriggerEnabled(); + $todayArchiveTimeToLive = Piwik_ArchiveProcessor_Rules::getTodayArchiveTimeToLive(); $showWarningCron = false; if (!$enableBrowserTriggerArchiving && $todayArchiveTimeToLive < 3600 @@ -81,8 +81,8 @@ class Piwik_CoreAdminHome_Controller extends Piwik_Controller_Admin $enableBrowserTriggerArchiving = Piwik_Common::getRequestVar('enableBrowserTriggerArchiving'); $todayArchiveTimeToLive = Piwik_Common::getRequestVar('todayArchiveTimeToLive'); - Piwik_ArchiveProcessing::setBrowserTriggerArchiving((bool)$enableBrowserTriggerArchiving); - Piwik_ArchiveProcessing::setTodayArchiveTimeToLive($todayArchiveTimeToLive); + Piwik_ArchiveProcessor_Rules::setBrowserTriggerArchiving((bool)$enableBrowserTriggerArchiving); + Piwik_ArchiveProcessor_Rules::setTodayArchiveTimeToLive($todayArchiveTimeToLive); // Update email settings $mail = array(); diff --git a/plugins/CoreAdminHome/CoreAdminHome.php b/plugins/CoreAdminHome/CoreAdminHome.php index c77f509e9f..4f75d274c0 100644 --- a/plugins/CoreAdminHome/CoreAdminHome.php +++ b/plugins/CoreAdminHome/CoreAdminHome.php @@ -114,7 +114,7 @@ class Piwik_CoreAdminHome extends Piwik_Plugin $archiveTables = Piwik::getTablesArchivesInstalled(); foreach ($archiveTables as $table) { if (strpos($table, 'numeric') !== false) { - Piwik_ArchiveProcessing_Period::doPurgeOutdatedArchives($table); + Piwik_ArchiveProcessor_Rules::doPurgeOutdatedArchives($table); } } } diff --git a/plugins/CoreUpdater/Controller.php b/plugins/CoreUpdater/Controller.php index 8e20f9c82a..76ab9bf71d 100644 --- a/plugins/CoreUpdater/Controller.php +++ b/plugins/CoreUpdater/Controller.php @@ -163,7 +163,7 @@ class Piwik_CoreUpdater_Controller extends Piwik_Controller /* * Make sure the execute bit is set for this shell script */ - if (!Piwik_ArchiveProcessing::isBrowserTriggerArchivingEnabled()) { + if (!Piwik_ArchiveProcessor_Rules::isBrowserTriggerEnabled()) { @chmod($this->pathRootExtractedPiwik . '/misc/cron/archive.sh', 0755); } diff --git a/plugins/CustomVariables/Archiver.php b/plugins/CustomVariables/Archiver.php index 15cc2fdf23..e456178ea8 100644 --- a/plugins/CustomVariables/Archiver.php +++ b/plugins/CustomVariables/Archiver.php @@ -74,7 +74,7 @@ class Piwik_CustomVariables_Archiver extends Piwik_PluginsArchiver protected function getSelectAveragePrice() { - return Piwik_ArchiveProcessing_Day::getSqlRevenue("AVG(log_link_visit_action.custom_var_v2)") + return Piwik_ArchiveProcessor_Day::getSqlRevenue("AVG(log_link_visit_action.custom_var_v2)") . " as `" . Piwik_Archive::INDEX_ECOMMERCE_ITEM_PRICE_VIEWED . "`"; } @@ -191,8 +191,8 @@ class Piwik_CustomVariables_Archiver extends Piwik_PluginsArchiver public function archivePeriod() { - $nameToCount = $this->getProcessor()->archiveDataTable( - self::CUSTOM_VARIABLE_RECORD_NAME, null, $this->maximumRowsInDataTableLevelZero, $this->maximumRowsInSubDataTable, + $nameToCount = $this->getProcessor()->aggregateDataTableReports( + self::CUSTOM_VARIABLE_RECORD_NAME, $this->maximumRowsInDataTableLevelZero, $this->maximumRowsInSubDataTable, $columnToSort = Piwik_Archive::INDEX_NB_VISITS); } }
\ No newline at end of file diff --git a/plugins/DevicesDetection/Archiver.php b/plugins/DevicesDetection/Archiver.php index 8bedcc9d39..4d74e28ca2 100644 --- a/plugins/DevicesDetection/Archiver.php +++ b/plugins/DevicesDetection/Archiver.php @@ -57,8 +57,8 @@ class Piwik_DevicesDetection_Archiver extends Piwik_PluginsArchiver self::BROWSER_VERSION_RECORD_NAME ); foreach ($dataTablesToSum as $dt) { - $this->getProcessor()->archiveDataTable( - $dt, null, $this->maximumRows, $this->maximumRows, $columnToSort = "nb_visits"); + $this->getProcessor()->aggregateDataTableReports( + $dt, $this->maximumRows, $this->maximumRows, $columnToSort = "nb_visits"); } } }
\ No newline at end of file diff --git a/plugins/Goals/Archiver.php b/plugins/Goals/Archiver.php index 0a5a154f3b..1946594122 100644 --- a/plugins/Goals/Archiver.php +++ b/plugins/Goals/Archiver.php @@ -146,10 +146,11 @@ class Piwik_Goals_Archiver extends Piwik_PluginsArchiver $this->insertReports(self::DAYS_UNTIL_CONV_RECORD_NAME, $daysToConversions); // Stats for all goals + $nbConvertedVisits = $this->getProcessor()->getNumberOfVisitsConverted(); $metrics = array( - self::getRecordName('conversion_rate') => $this->getConversionRate($this->getProcessor()->getNumberOfVisitsConverted()), + self::getRecordName('conversion_rate') => $this->getConversionRate($nbConvertedVisits), self::getRecordName('nb_conversions') => $totalConversions, - self::getRecordName('nb_visits_converted') => $this->getProcessor()->getNumberOfVisitsConverted(), + self::getRecordName('nb_visits_converted') => $nbConvertedVisits, self::getRecordName('revenue') => $totalRevenue, ); $this->getProcessor()->insertNumericRecords($metrics); @@ -380,7 +381,7 @@ class Piwik_Goals_Archiver extends Piwik_PluginsArchiver foreach ($this->dimensionRecord as $recordName) { $dataTableToSum[] = self::getItemRecordNameAbandonedCart($recordName); } - $this->getProcessor()->archiveDataTable($dataTableToSum); + $this->getProcessor()->aggregateDataTableReports($dataTableToSum); } /* @@ -402,7 +403,7 @@ class Piwik_Goals_Archiver extends Piwik_PluginsArchiver $fieldsToSum[] = self::getRecordName($metricName, $goalId); } } - $records = $this->getProcessor()->archiveNumericValuesSum($fieldsToSum); + $records = $this->getProcessor()->archiveNumericValuesGeneral($fieldsToSum); // also recording conversion_rate for each goal foreach ($goalIdsToSum as $goalId) { @@ -411,13 +412,13 @@ class Piwik_Goals_Archiver extends Piwik_PluginsArchiver $this->getProcessor()->insertNumericRecord(self::getRecordName('conversion_rate', $goalId), $conversion_rate); // sum up the visits to conversion data table & the days to conversion data table - $this->getProcessor()->archiveDataTable(array( + $this->getProcessor()->aggregateDataTableReports(array( self::getRecordName(self::VISITS_UNTIL_RECORD_NAME, $goalId), self::getRecordName(self::DAYS_UNTIL_CONV_RECORD_NAME, $goalId))); } // sum up goal overview reports - $this->getProcessor()->archiveDataTable(array( + $this->getProcessor()->aggregateDataTableReports(array( self::getRecordName(self::VISITS_UNTIL_RECORD_NAME), self::getRecordName(self::DAYS_UNTIL_CONV_RECORD_NAME))); } diff --git a/plugins/Goals/Goals.php b/plugins/Goals/Goals.php index 024cd562f1..ecda104804 100644 --- a/plugins/Goals/Goals.php +++ b/plugins/Goals/Goals.php @@ -509,7 +509,7 @@ class Piwik_Goals extends Piwik_Plugin function archiveDay($notification) { /** - * @var Piwik_ArchiveProcessing_Day + * @var Piwik_ArchiveProcessor_Day */ $archiveProcessing = $notification->getNotificationObject(); diff --git a/plugins/Live/API.php b/plugins/Live/API.php index d2d74a74bf..0f3b70b5e2 100644 --- a/plugins/Live/API.php +++ b/plugins/Live/API.php @@ -537,11 +537,11 @@ class Piwik_Live_API $sql = "SELECT case idgoal when " . Piwik_Tracker_GoalManager::IDGOAL_CART . " then '" . Piwik_Archive::LABEL_ECOMMERCE_CART . "' else '" . Piwik_Archive::LABEL_ECOMMERCE_ORDER . "' end as type, idorder as orderId, - " . Piwik_ArchiveProcessing_Day::getSqlRevenue('revenue') . " as revenue, - " . Piwik_ArchiveProcessing_Day::getSqlRevenue('revenue_subtotal') . " as revenueSubTotal, - " . Piwik_ArchiveProcessing_Day::getSqlRevenue('revenue_tax') . " as revenueTax, - " . Piwik_ArchiveProcessing_Day::getSqlRevenue('revenue_shipping') . " as revenueShipping, - " . Piwik_ArchiveProcessing_Day::getSqlRevenue('revenue_discount') . " as revenueDiscount, + " . Piwik_ArchiveProcessor_Day::getSqlRevenue('revenue') . " as revenue, + " . Piwik_ArchiveProcessor_Day::getSqlRevenue('revenue_subtotal') . " as revenueSubTotal, + " . Piwik_ArchiveProcessor_Day::getSqlRevenue('revenue_tax') . " as revenueTax, + " . Piwik_ArchiveProcessor_Day::getSqlRevenue('revenue_shipping') . " as revenueShipping, + " . Piwik_ArchiveProcessor_Day::getSqlRevenue('revenue_discount') . " as revenueDiscount, items as items, log_conversion.server_time as serverTimePretty @@ -578,7 +578,7 @@ class Piwik_Live_API log_action_sku.name as itemSKU, log_action_name.name as itemName, log_action_category.name as itemCategory, - " . Piwik_ArchiveProcessing_Day::getSqlRevenue('price') . " as price, + " . Piwik_ArchiveProcessor_Day::getSqlRevenue('price') . " as price, quantity as quantity FROM " . Piwik_Common::prefixTable('log_conversion_item') . " INNER JOIN " . Piwik_Common::prefixTable('log_action') . " AS log_action_sku diff --git a/plugins/MultiSites/API.php b/plugins/MultiSites/API.php index 1cbcfc39b3..bf4250ff0e 100755 --- a/plugins/MultiSites/API.php +++ b/plugins/MultiSites/API.php @@ -232,18 +232,22 @@ class Piwik_MultiSites_API // if the period isn't a range & a lastN/previousN date isn't used, we get the same // data for the last period to show the evolution of visits/actions/revenue list($strLastDate, $lastPeriod) = Piwik_Period_Range::getLastDate($date, $period); - if ($strLastDate !== false) { + if ( + false && + + $strLastDate !== false) { if ($lastPeriod !== false) { // NOTE: no easy way to set last period date metadata when range of dates is requested. // will be easier if DataTable_Array::metadata is removed, and metadata that is // put there is put directly in Piwik_DataTable::metadata. $dataTable->setMetadata(self::getLastPeriodMetadataName('date'), $lastPeriod); } - $pastArchive = Piwik_Archive::build('all', $period, $strLastDate, $segment, $_restrictSitesToLogin); $pastData = $pastArchive->getDataTableFromNumeric($fieldsToGet); - $pastData = $pastData->mergeChildren(); + if(!($dataTable instanceof Piwik_DataTable_Simple)) { + $pastData = $pastData->mergeChildren(); + } // use past data to calculate evolution percentages $this->calculateEvolutionPercentages($dataTable, $pastData, $apiMetrics); @@ -315,7 +319,7 @@ class Piwik_MultiSites_API private function calculateEvolutionPercentages($currentData, $pastData, $apiMetrics) { if ($currentData instanceof Piwik_DataTable_Array) { - $pastArray = $pastData->getArray(); + $pastArray = $pastData->getRows(); foreach ($currentData->getArray() as $subTable) { $this->calculateEvolutionPercentages($subTable, current($pastArray), $apiMetrics); next($pastArray); diff --git a/plugins/MultiSites/Controller.php b/plugins/MultiSites/Controller.php index cb63dc5746..a4e5a98eab 100644 --- a/plugins/MultiSites/Controller.php +++ b/plugins/MultiSites/Controller.php @@ -68,50 +68,60 @@ class Piwik_MultiSites_Controller extends Piwik_Controller foreach ($siteIds as $idSite) { $isEcommerceEnabled = Piwik_Site::isEcommerceEnabledFor($idSite); - $digestableData[$idSite] = array( - 'idsite' => $idSite, - 'main_url' => Piwik_Site::getMainUrlFor($idSite), - 'name' => Piwik_Site::getNameFor($idSite), - 'visits' => 0, - 'pageviews' => 0 +// allSites[{$i}] = new setRowData( +// 0 {$site.idsite}, +// 1 {$site.visits}, +// 2 {$site.pageviews}, +// 3 {if empty($site.revenue)}0{else}{$site.revenue}{/if}, +// 4 '{$site.name|escape:"javascript"}', +// 5 '{$site.main_url|escape:"javascript"}', +// 6 '{if isset($site.visits_evolution)}{$site.visits_evolution|replace:",":"."}{/if}', +// 7 '{if isset($site.pageviews_evolution)}{$site.pageviews_evolution|replace:",":"."}{/if}', +// 8 '{if isset($site.revenue_evolution)}{$site.revenue_evolution|replace:",":"."}{/if}'); + + $siteData = array($idSite,0,0.0, + Piwik_Site::getMainUrlFor($idSite), + Piwik_Site::getNameFor($idSite), ); if ($period != 'range') { - $digestableData[$idSite]['visits_evolution'] = 0; - $digestableData[$idSite]['pageviews_evolution'] = 0; + $siteData[6] = 0; + $siteData[7] = 0; } if ($displayRevenueColumn) { $revenueDefault = $isEcommerceEnabled ? 0 : "'-'"; if ($period != 'range') { - $digestableData[$idSite]['revenue_evolution'] = $revenueDefault; + $siteData[8] = $revenueDefault; } } + $digestableData[$idSite] = $siteData; } foreach ($dataTable->getRows() as $row) { $idsite = (int)$row->getMetadata('idsite'); - $site = & $digestableData[$idsite]; + $siteData = array(); - $site['visits'] = (int)$row->getColumn('nb_visits'); - $site['pageviews'] = (int)$row->getColumn('nb_pageviews'); + $siteData[1] = (int)$row->getColumn('nb_visits');http://pastebin.com/raw.php?i=1dvHmEUA + $siteData[2] = (int)$row->getColumn('nb_pageviews'); if ($displayRevenueColumn) { if ($row->getColumn('revenue') !== false) { - $site['revenue'] = $row->getColumn('revenue'); + $siteData[3] = $row->getColumn('revenue'); } } if ($period != 'range') { - $site['visits_evolution'] = $row->getColumn('visits_evolution'); - $site['pageviews_evolution'] = $row->getColumn('pageviews_evolution'); + $siteData[6] = $row->getColumn('visits_evolution'); + $siteData[7] = $row->getColumn('pageviews_evolution'); if ($displayRevenueColumn) { - $site['revenue_evolution'] = $row->getColumn('revenue_evolution'); + $siteData[8] = $row->getColumn('revenue_evolution'); } } + $digestableData[$idsite] = array_merge($digestableData[$idsite], $siteData); } $this->applyPrettyMoney($digestableData); diff --git a/plugins/Provider/Archiver.php b/plugins/Provider/Archiver.php index 692c3a72d2..e90489f51a 100644 --- a/plugins/Provider/Archiver.php +++ b/plugins/Provider/Archiver.php @@ -22,6 +22,6 @@ class Piwik_Provider_Archiver extends Piwik_PluginsArchiver public function archivePeriod() { - $this->getProcessor()->archiveDataTable(array(self::PROVIDER_RECORD_NAME), null, $this->maximumRows); + $this->getProcessor()->aggregateDataTableReports(array(self::PROVIDER_RECORD_NAME), $this->maximumRows); } }
\ No newline at end of file diff --git a/plugins/Referers/Archiver.php b/plugins/Referers/Archiver.php index ddae742131..47ce247550 100644 --- a/plugins/Referers/Archiver.php +++ b/plugins/Referers/Archiver.php @@ -189,7 +189,7 @@ class Piwik_Referers_Archiver extends Piwik_PluginsArchiver /** * Records the daily stats (numeric or datatable blob) into the archive tables. * - * @param Piwik_ArchiveProcessing $this->getProcessor() + * @param Piwik_ArchiveProcessor $this->getProcessor() */ protected function recordDayReports() { @@ -223,7 +223,7 @@ class Piwik_Referers_Archiver extends Piwik_PluginsArchiver public function archivePeriod() { $dataTableToSum = $this->getRecordNames(); - $nameToCount = $this->getProcessor()->archiveDataTable($dataTableToSum, null, $this->maximumRowsInDataTableLevelZero, $this->maximumRowsInSubDataTable, $this->columnToSortByBeforeTruncation); + $nameToCount = $this->getProcessor()->aggregateDataTableReports($dataTableToSum, $this->maximumRowsInDataTableLevelZero, $this->maximumRowsInSubDataTable, $this->columnToSortByBeforeTruncation); $mappingFromArchiveName = array( self::METRIC_DISTINCT_SEARCH_ENGINE_RECORD_NAME => diff --git a/plugins/SegmentEditor/SegmentEditor.php b/plugins/SegmentEditor/SegmentEditor.php index 3ac66afb1e..5af09785ae 100644 --- a/plugins/SegmentEditor/SegmentEditor.php +++ b/plugins/SegmentEditor/SegmentEditor.php @@ -45,9 +45,7 @@ class Piwik_SegmentEditor extends Piwik_Plugin public function getKnownSegmentsToArchiveAllSites($notification) { $segments =& $notification->getNotificationObject(); - $segmentsToAutoArchive = Piwik_SegmentEditor_API::getInstance()->getAll( - $idSite = false, $returnAutoArchived = true); - + $segmentsToAutoArchive = Piwik_SegmentEditor_API::getInstance()->getAll($idSite = false, $returnAutoArchived = true); foreach ($segmentsToAutoArchive as $segment) { $segments[] = $segment['definition']; } diff --git a/plugins/Transitions/API.php b/plugins/Transitions/API.php index 897b14923a..7e17fa2996 100644 --- a/plugins/Transitions/API.php +++ b/plugins/Transitions/API.php @@ -62,10 +62,10 @@ class Piwik_Transitions_API } // prepare archive processing that can be used by the archiving code - $archiveProcessing = new Piwik_ArchiveProcessing_Day(); - $archiveProcessing->setSite(new Piwik_Site($idSite)); - $archiveProcessing->setPeriod(Piwik_Period::advancedFactory($period, $date)); - $archiveProcessing->setSegment(new Piwik_Segment($segment, $idSite)); + $segment = new Piwik_Segment($segment, $idSite); + $site = new Piwik_Site($idSite); + $period = Piwik_Period::advancedFactory($period, $date); + $archiveProcessing = Piwik_ArchiveProcessor::factory($period, $site, $segment); // prepare the report $report = array( diff --git a/plugins/Transitions/Transitions.php b/plugins/Transitions/Transitions.php index b92ef87b8c..b887669a70 100644 --- a/plugins/Transitions/Transitions.php +++ b/plugins/Transitions/Transitions.php @@ -64,7 +64,7 @@ class Piwik_Transitions extends Piwik_Plugin * * @param $idaction * @param $actionType - * @param Piwik_ArchiveProcessing_Day $archiveProcessing + * @param Piwik_ArchiveProcessor_Day $archiveProcessing * @param $limitBeforeGrouping * @return Piwik_DataTable */ @@ -130,7 +130,7 @@ class Piwik_Transitions extends Piwik_Plugin //FIXMEA refactor after integration tests written $array = new Piwik_DataArray($referrerData, $referrerSubData); - return Piwik_ArchiveProcessing_Day::getDataTableFromDataArray($array); + return Piwik_ArchiveProcessor_Day::getDataTableFromDataArray($array); } /** @@ -138,7 +138,7 @@ class Piwik_Transitions extends Piwik_Plugin * * @param $idaction * @param $actionType - * @param Piwik_ArchiveProcessing_Day $archiveProcessing + * @param Piwik_ArchiveProcessor_Day $archiveProcessing * @param $limitBeforeGrouping * @return array(previousPages:Piwik_DataTable, loops:integer) */ @@ -257,12 +257,12 @@ class Piwik_Transitions extends Piwik_Plugin * * @param $idaction * @param $actionType - * @param Piwik_ArchiveProcessing_Day $archiveProcessing + * @param Piwik_ArchiveProcessor_Day $archiveProcessing * @param $limitBeforeGrouping * @param $includeLoops * @return array(followingPages:Piwik_DataTable, outlinks:Piwik_DataTable, downloads:Piwik_DataTable) */ - public function queryFollowingActions($idaction, $actionType, Piwik_ArchiveProcessing_Day $archiveProcessing, + public function queryFollowingActions($idaction, $actionType, Piwik_ArchiveProcessor_Day $archiveProcessing, $limitBeforeGrouping = false, $includeLoops = false) { $types = array(); diff --git a/plugins/UserCountry/Archiver.php b/plugins/UserCountry/Archiver.php index fb69e89ff8..33f7d28e90 100644 --- a/plugins/UserCountry/Archiver.php +++ b/plugins/UserCountry/Archiver.php @@ -124,15 +124,15 @@ class Piwik_UserCountry_Archiver extends Piwik_PluginsArchiver protected function recordDayReports() { - $tableCountry = Piwik_ArchiveProcessing_Day::getDataTableFromDataArray($this->arrays[self::COUNTRY_FIELD]); + $tableCountry = Piwik_ArchiveProcessor_Day::getDataTableFromDataArray($this->arrays[self::COUNTRY_FIELD]); $this->getProcessor()->insertBlobRecord(self::COUNTRY_RECORD_NAME, $tableCountry->getSerialized()); $this->getProcessor()->insertNumericRecord(self::DISTINCT_COUNTRIES_METRIC, $tableCountry->getRowsCount()); - $tableRegion = Piwik_ArchiveProcessing_Day::getDataTableFromDataArray($this->arrays[self::REGION_FIELD]); + $tableRegion = Piwik_ArchiveProcessor_Day::getDataTableFromDataArray($this->arrays[self::REGION_FIELD]); $serialized = $tableRegion->getSerialized($this->maximumRows, $this->maximumRows, Piwik_Archive::INDEX_NB_VISITS); $this->getProcessor()->insertBlobRecord(self::REGION_RECORD_NAME, $serialized); - $tableCity = Piwik_ArchiveProcessing_Day::getDataTableFromDataArray($this->arrays[self::CITY_FIELD]); + $tableCity = Piwik_ArchiveProcessor_Day::getDataTableFromDataArray($this->arrays[self::CITY_FIELD]); $this->setLatitudeLongitude($tableCity); $serialized = $tableCity->getSerialized($this->maximumRows, $this->maximumRows, Piwik_Archive::INDEX_NB_VISITS); $this->getProcessor()->insertBlobRecord(self::CITY_RECORD_NAME, $serialized); @@ -167,7 +167,7 @@ class Piwik_UserCountry_Archiver extends Piwik_PluginsArchiver self::CITY_RECORD_NAME, ); - $nameToCount = $this->getProcessor()->archiveDataTable($dataTableToSum); + $nameToCount = $this->getProcessor()->aggregateDataTableReports($dataTableToSum); $this->getProcessor()->insertNumericRecord(self::DISTINCT_COUNTRIES_METRIC, $nameToCount[self::COUNTRY_RECORD_NAME]['level0']); } diff --git a/plugins/UserSettings/Archiver.php b/plugins/UserSettings/Archiver.php index a27db178ca..91df3d2c0c 100644 --- a/plugins/UserSettings/Archiver.php +++ b/plugins/UserSettings/Archiver.php @@ -146,7 +146,7 @@ class Piwik_UserSettings_Archiver extends Piwik_PluginsArchiver self::PLUGIN_RECORD_NAME, self::LANGUAGE_RECORD_NAME, ); - $this->getProcessor()->archiveDataTable($dataTableToSum, null, $this->maximumRows); + $this->getProcessor()->aggregateDataTableReports($dataTableToSum, $this->maximumRows); } } diff --git a/plugins/VisitTime/API.php b/plugins/VisitTime/API.php index 637aa97cbc..65952ba24d 100644 --- a/plugins/VisitTime/API.php +++ b/plugins/VisitTime/API.php @@ -74,14 +74,15 @@ class Piwik_VisitTime_API } // metrics to query - $metrics = Piwik_ArchiveProcessing::getCoreMetrics(); + $metrics = Piwik_Archive::getVisitsMetricNames(); + unset($metrics[Piwik_Archive::INDEX_MAX_ACTIONS]); // get metric data for every day within the supplied period $oPeriod = Piwik_Archive::makePeriodFromQueryParams(Piwik_Site::getTimezoneFor($idSite), $period, $date); $dateRange = $oPeriod->getDateStart()->toString() . ',' . $oPeriod->getDateEnd()->toString(); $archive = Piwik_Archive::build($idSite, 'day', $dateRange, $segment); - $dataTable = $archive->getDataTableFromNumeric($metrics)->mergeChildren(); + $dataTable = $archive->getDataTableFromNumeric($metrics); // if there's no data for this report, don't bother w/ anything else // TODO: with changes to getDataTableFromNumeric, this code would have to check if every row has 0 column values. is it really necessary? (assuming no for now) diff --git a/plugins/VisitTime/Archiver.php b/plugins/VisitTime/Archiver.php index af811e0d2f..2f11c61d2a 100644 --- a/plugins/VisitTime/Archiver.php +++ b/plugins/VisitTime/Archiver.php @@ -46,7 +46,7 @@ class Piwik_VisitTime_Archiver extends Piwik_PluginsArchiver protected function convertTimeToLocalTimezone(Piwik_DataArray &$array) { - $date = Piwik_Date::factory($this->getProcessor()->getStartDatetimeUTC())->toString(); + $date = Piwik_Date::factory($this->getProcessor()->getDateStart()->getDateStartUTC())->toString(); $timezone = $this->getProcessor()->getSite()->getTimezone(); $converted = array(); @@ -75,6 +75,6 @@ class Piwik_VisitTime_Archiver extends Piwik_PluginsArchiver self::LOCAL_TIME_RECORD_NAME, self::SERVER_TIME_RECORD_NAME, ); - $this->getProcessor()->archiveDataTable($dataTableToSum); + $this->getProcessor()->aggregateDataTableReports($dataTableToSum); } }
\ No newline at end of file diff --git a/plugins/VisitorGenerator/Controller.php b/plugins/VisitorGenerator/Controller.php index 54ebc3aea0..acebd0a2fc 100644 --- a/plugins/VisitorGenerator/Controller.php +++ b/plugins/VisitorGenerator/Controller.php @@ -76,7 +76,7 @@ class Piwik_VisitorGenerator_Controller extends Piwik_Controller_Admin $api = Piwik_CoreAdminHome_API::getInstance(); $api->invalidateArchivedReports($idSite, implode($dates, ",")); - $browserArchiving = Piwik_ArchiveProcessing::isBrowserTriggerArchivingEnabled(); + $browserArchiving = Piwik_ArchiveProcessor_Rules::isBrowserTriggerEnabled(); // Init view $view = Piwik_View::factory('generate'); diff --git a/plugins/VisitorInterest/Archiver.php b/plugins/VisitorInterest/Archiver.php index 5f0a15d3d0..6cbeb82317 100644 --- a/plugins/VisitorInterest/Archiver.php +++ b/plugins/VisitorInterest/Archiver.php @@ -141,6 +141,6 @@ class Piwik_VisitorInterest_Archiver extends Piwik_PluginsArchiver self::VISITS_COUNT_RECORD_NAME, self::DAYS_SINCE_LAST_RECORD_NAME ); - $this->getProcessor()->archiveDataTable($dataTableToSum); + $this->getProcessor()->aggregateDataTableReports($dataTableToSum); } }
\ No newline at end of file diff --git a/plugins/VisitsSummary/VisitsSummary.php b/plugins/VisitsSummary/VisitsSummary.php index 87b320c947..0744b0424a 100644 --- a/plugins/VisitsSummary/VisitsSummary.php +++ b/plugins/VisitsSummary/VisitsSummary.php @@ -12,7 +12,7 @@ /** * Note: This plugin does not hook on Daily and Period Archiving like other Plugins because it reports the * very core metrics (visits, actions, visit duration, etc.) which are processed in the Core - * Piwik_ArchiveProcessing_Day class directly. + * Piwik_ArchiveProcessor_Day class directly. * These metrics can be used by other Plugins so they need to be processed up front. * * @package Piwik_VisitsSummary diff --git a/tests/PHPUnit/Benchmarks/ArchiveQueryBenchmark.php b/tests/PHPUnit/Benchmarks/ArchiveQueryBenchmark.php index 4b456f9df3..225bbc6b3f 100644 --- a/tests/PHPUnit/Benchmarks/ArchiveQueryBenchmark.php +++ b/tests/PHPUnit/Benchmarks/ArchiveQueryBenchmark.php @@ -33,8 +33,8 @@ class ArchiveQueryBenchmark extends BenchmarkTestCase if ($this->archivingLaunched) { echo "NOTE: Had to archive tables, memory results will not be accurate. Run again for better results."; } - - Piwik_ArchiveProcessing::$forceDisableArchiving = true; + + Piwik_ArchiveProcessor_Rules::$archivingDisabledByTests = true; $period = Piwik_Period::factory(self::$fixture->period, Piwik_Date::factory(self::$fixture->date)); $dateRange = $period->getDateStart().','.$period->getDateEnd(); diff --git a/tests/PHPUnit/Benchmarks/Fixtures/SqlDump.php b/tests/PHPUnit/Benchmarks/Fixtures/SqlDump.php index 488814877f..08843aff65 100755 --- a/tests/PHPUnit/Benchmarks/Fixtures/SqlDump.php +++ b/tests/PHPUnit/Benchmarks/Fixtures/SqlDump.php @@ -62,6 +62,6 @@ class Piwik_Test_Fixture_SqlDump } // make sure archiving will be called - Piwik_ArchiveProcessing::setBrowserTriggerArchiving(true); + Piwik_ArchiveProcessor_Rules::setBrowserTriggerArchiving(true); } } diff --git a/tests/PHPUnit/Core/ArchiveProcessingTest.php b/tests/PHPUnit/Core/ArchiveProcessingTest.php index a2198aaee8..566502f803 100644 --- a/tests/PHPUnit/Core/ArchiveProcessingTest.php +++ b/tests/PHPUnit/Core/ArchiveProcessingTest.php @@ -44,19 +44,16 @@ class ArchiveProcessingTest extends DatabaseTestCase * @param string $periodLabel * @param string $dateLabel * @param string $siteTimezone - * @return Piwik_ArchiveProcessing + * @return Piwik_ArchiveProcessor */ private function _createArchiveProcessing($periodLabel, $dateLabel, $siteTimezone) { $site = $this->_createWebsite($siteTimezone); $date = Piwik_Date::factory($dateLabel); $period = Piwik_Period::factory($periodLabel, $date); + $segment = new Piwik_Segment('', $site->getId()); - $archiveProcessing = Piwik_ArchiveProcessing::factory($periodLabel); - $archiveProcessing->setSite($site); - $archiveProcessing->setPeriod($period); - $archiveProcessing->setSegment(new Piwik_Segment('', $site->getId())); - return $archiveProcessing; + return Piwik_ArchiveProcessor::factory($period, $site, $segment); } /** @@ -82,7 +79,7 @@ class ArchiveProcessingTest extends DatabaseTestCase // min finished timestamp considered when looking at archive timestamp - $timeout = Piwik_ArchiveProcessing::getTodayArchiveTimeToLive(); + $timeout = Piwik_ArchiveProcessor_Rules::getTodayArchiveTimeToLive(); $this->assertTrue($timeout >= 10); $dateMinArchived = $now - $timeout; @@ -103,8 +100,8 @@ class ArchiveProcessingTest extends DatabaseTestCase $dateMinArchived = Piwik_Date::factory('2010-01-02')->getTimestamp(); $this->assertEquals($archiveProcessing->getMinTimeArchivedProcessed() + 1, $dateMinArchived); - $this->assertEquals('2010-01-01 00:00:00', $archiveProcessing->getStartDatetimeUTC()); - $this->assertEquals('2010-01-01 23:59:59', $archiveProcessing->getEndDatetimeUTC()); + $this->assertEquals('2010-01-01 00:00:00', $archiveProcessing->getDateStart()->getDateStartUTC()); + $this->assertEquals('2010-01-01 23:59:59', $archiveProcessing->getDateEnd()->getDateEndUTC()); $this->assertFalse($archiveProcessing->isArchiveTemporary()); } @@ -121,8 +118,8 @@ class ArchiveProcessingTest extends DatabaseTestCase $dateMinArchived = Piwik_Date::factory('2010-01-01 18:30:00'); $this->assertEquals($archiveProcessing->getMinTimeArchivedProcessed() + 1, $dateMinArchived->getTimestamp()); - $this->assertEquals('2009-12-31 18:30:00', $archiveProcessing->getStartDatetimeUTC()); - $this->assertEquals('2010-01-01 18:29:59', $archiveProcessing->getEndDatetimeUTC()); + $this->assertEquals('2009-12-31 18:30:00', $archiveProcessing->getDateStart()->getDateStartUTC()); + $this->assertEquals('2010-01-01 18:29:59', $archiveProcessing->getDateEnd()->getDateEndUTC()); $this->assertFalse($archiveProcessing->isArchiveTemporary()); } @@ -139,8 +136,8 @@ class ArchiveProcessingTest extends DatabaseTestCase $dateMinArchived = Piwik_Date::factory('2010-02-01 05:30:00'); $this->assertEquals($archiveProcessing->getMinTimeArchivedProcessed() + 1, $dateMinArchived->getTimestamp()); - $this->assertEquals('2010-01-01 05:30:00', $archiveProcessing->getStartDatetimeUTC()); - $this->assertEquals('2010-02-01 05:29:59', $archiveProcessing->getEndDatetimeUTC()); + $this->assertEquals('2010-01-01 05:30:00', $archiveProcessing->getDateStart()->getDateStartUTC()); + $this->assertEquals('2010-02-01 05:29:59', $archiveProcessing->getDateEnd()->getDateEndUTC()); $this->assertFalse($archiveProcessing->isArchiveTemporary()); } @@ -156,19 +153,19 @@ class ArchiveProcessingTest extends DatabaseTestCase $timestamp = Piwik_Date::factory('now', $siteTimezone)->getTimestamp(); $dateLabel = date('Y-m-d', $timestamp); - Piwik_ArchiveProcessing::setBrowserTriggerArchiving(true); + Piwik_ArchiveProcessor_Rules::setBrowserTriggerArchiving(true); $archiveProcessing = $this->_createArchiveProcessing('day', $dateLabel, $siteTimezone); $archiveProcessing->time = $now; // we look at anything processed within the time to live range - $dateMinArchived = $now - Piwik_ArchiveProcessing::getTodayArchiveTimeToLive(); + $dateMinArchived = $now - Piwik_ArchiveProcessor_Rules::getTodayArchiveTimeToLive(); $this->assertEquals($dateMinArchived, $archiveProcessing->getMinTimeArchivedProcessed()); $this->assertTrue($archiveProcessing->isArchiveTemporary()); // when browsers don't trigger archives, we force ArchiveProcessing // to fetch any of the most recent archive - Piwik_ArchiveProcessing::setBrowserTriggerArchiving(false); + Piwik_ArchiveProcessor_Rules::setBrowserTriggerArchiving(false); // see isArchivingDisabled() // Running in CLI doesn't impact the time to live today's archive we are loading // From CLI, we will not return data that is 'stale' @@ -177,8 +174,8 @@ class ArchiveProcessingTest extends DatabaseTestCase } $this->assertEquals($archiveProcessing->getMinTimeArchivedProcessed(), $dateMinArchived); - $this->assertEquals(date('Y-m-d', $timestamp) . ' 01:00:00', $archiveProcessing->getStartDatetimeUTC()); - $this->assertEquals(date('Y-m-d', $timestamp + 86400) . ' 00:59:59', $archiveProcessing->getEndDatetimeUTC()); + $this->assertEquals(date('Y-m-d', $timestamp) . ' 01:00:00', $archiveProcessing->getDateStart()->getDateStartUTC()); + $this->assertEquals(date('Y-m-d', $timestamp + 86400) . ' 00:59:59', $archiveProcessing->getDateEnd()->getDateEndUTC()); $this->assertTrue($archiveProcessing->isArchiveTemporary()); } @@ -198,19 +195,19 @@ class ArchiveProcessingTest extends DatabaseTestCase $timestamp = Piwik_Date::factory('now', $siteTimezone)->getTimestamp(); $dateLabel = date('Y-m-d', $timestamp); - Piwik_ArchiveProcessing::setBrowserTriggerArchiving(true); + Piwik_ArchiveProcessor_Rules::setBrowserTriggerArchiving(true); $archiveProcessing = $this->_createArchiveProcessing('day', $dateLabel, $siteTimezone); $archiveProcessing->time = $now; // we look at anything processed within the time to live range - $dateMinArchived = $now - Piwik_ArchiveProcessing::getTodayArchiveTimeToLive(); + $dateMinArchived = $now - Piwik_ArchiveProcessor_Rules::getTodayArchiveTimeToLive(); $this->assertEquals($archiveProcessing->getMinTimeArchivedProcessed(), $dateMinArchived); $this->assertTrue($archiveProcessing->isArchiveTemporary()); // when browsers don't trigger archives, we force ArchiveProcessing // to fetch any of the most recent archive - Piwik_ArchiveProcessing::setBrowserTriggerArchiving(false); + Piwik_ArchiveProcessor_Rules::setBrowserTriggerArchiving(false); // see isArchivingDisabled() // Running in CLI doesn't impact the time to live today's archive we are loading // From CLI, we will not return data that is 'stale' @@ -220,10 +217,10 @@ class ArchiveProcessingTest extends DatabaseTestCase $this->assertEquals($archiveProcessing->getMinTimeArchivedProcessed(), $dateMinArchived); // this test varies with DST - $this->assertTrue($archiveProcessing->getStartDatetimeUTC() == date('Y-m-d', $timestamp - 86400) . ' 22:00:00' || - $archiveProcessing->getStartDatetimeUTC() == date('Y-m-d', $timestamp - 86400) . ' 23:00:00'); - $this->assertTrue($archiveProcessing->getEndDatetimeUTC() == date('Y-m-d', $timestamp) . ' 21:59:59' || - $archiveProcessing->getEndDatetimeUTC() == date('Y-m-d', $timestamp) . ' 22:59:59'); + $this->assertTrue($archiveProcessing->getDateStart()->getDateStartUTC() == date('Y-m-d', $timestamp - 86400) . ' 22:00:00' || + $archiveProcessing->getDateStart()->getDateStartUTC() == date('Y-m-d', $timestamp - 86400) . ' 23:00:00'); + $this->assertTrue($archiveProcessing->getDateEnd()->getDateEndUTC() == date('Y-m-d', $timestamp) . ' 21:59:59' || + $archiveProcessing->getDateEnd()->getDateEndUTC() == date('Y-m-d', $timestamp) . ' 22:59:59'); $this->assertTrue($archiveProcessing->isArchiveTemporary()); } @@ -244,19 +241,19 @@ class ArchiveProcessingTest extends DatabaseTestCase $timestamp = Piwik_Date::factory('now', $siteTimezone)->getTimestamp(); $dateLabel = date('Y-m-d', $timestamp); - Piwik_ArchiveProcessing::setBrowserTriggerArchiving(true); + Piwik_ArchiveProcessor_Rules::setBrowserTriggerArchiving(true); $archiveProcessing = $this->_createArchiveProcessing('day', $dateLabel, $siteTimezone); $archiveProcessing->time = $now; // we look at anything processed within the time to live range - $dateMinArchived = $now - Piwik_ArchiveProcessing::getTodayArchiveTimeToLive(); + $dateMinArchived = $now - Piwik_ArchiveProcessor_Rules::getTodayArchiveTimeToLive(); $this->assertEquals($archiveProcessing->getMinTimeArchivedProcessed(), $dateMinArchived); $this->assertTrue($archiveProcessing->isArchiveTemporary()); // when browsers don't trigger archives, we force ArchiveProcessing // to fetch any of the most recent archive - Piwik_ArchiveProcessing::setBrowserTriggerArchiving(false); + Piwik_ArchiveProcessor_Rules::setBrowserTriggerArchiving(false); // see isArchivingDisabled() // Running in CLI doesn't impact the time to live today's archive we are loading // From CLI, we will not return data that is 'stale' @@ -266,10 +263,10 @@ class ArchiveProcessingTest extends DatabaseTestCase $this->assertEquals($archiveProcessing->getMinTimeArchivedProcessed(), $dateMinArchived); // this test varies with DST - $this->assertTrue($archiveProcessing->getStartDatetimeUTC() == date('Y-m-d', $timestamp) . ' 04:00:00' || - $archiveProcessing->getStartDatetimeUTC() == date('Y-m-d', $timestamp) . ' 05:00:00'); - $this->assertTrue($archiveProcessing->getEndDatetimeUTC() == date('Y-m-d', $timestamp + 86400) . ' 03:59:59' || - $archiveProcessing->getEndDatetimeUTC() == date('Y-m-d', $timestamp + 86400) . ' 04:59:59'); + $this->assertTrue($archiveProcessing->getDateStart()->getDateStartUTC() == date('Y-m-d', $timestamp) . ' 04:00:00' || + $archiveProcessing->getDateStart()->getDateStartUTC() == date('Y-m-d', $timestamp) . ' 05:00:00'); + $this->assertTrue($archiveProcessing->getDateEnd()->getDateEndUTC() == date('Y-m-d', $timestamp + 86400) . ' 03:59:59' || + $archiveProcessing->getDateEnd()->getDateEndUTC() == date('Y-m-d', $timestamp + 86400) . ' 04:59:59'); $this->assertTrue($archiveProcessing->isArchiveTemporary()); } diff --git a/tests/PHPUnit/IntegrationTestCase.php b/tests/PHPUnit/IntegrationTestCase.php index 881d5e182f..17356d6175 100755 --- a/tests/PHPUnit/IntegrationTestCase.php +++ b/tests/PHPUnit/IntegrationTestCase.php @@ -950,9 +950,9 @@ abstract class IntegrationTestCase extends PHPUnit_Framework_TestCase $this->_setCallableApi($api); if (isset($params['disableArchiving']) && $params['disableArchiving'] === true) { - Piwik_ArchiveProcessing::$forceDisableArchiving = true; + Piwik_ArchiveProcessor_Rules::$archivingDisabledByTests = true; } else { - Piwik_ArchiveProcessing::$forceDisableArchiving = false; + Piwik_ArchiveProcessor_Rules::$archivingDisabledByTests = false; } if (isset($params['language'])) { diff --git a/tests/PHPUnit/Plugins/PrivacyManagerTest.php b/tests/PHPUnit/Plugins/PrivacyManagerTest.php index aad3d0e65f..dfb1596af1 100755 --- a/tests/PHPUnit/Plugins/PrivacyManagerTest.php +++ b/tests/PHPUnit/Plugins/PrivacyManagerTest.php @@ -44,12 +44,12 @@ class PrivacyManagerTest extends IntegrationTestCase // Temporarily disable the purge of old archives so that getNumeric('nb_visits') // in _addReportData does not trigger the data purge of data we've just imported - Piwik_ArchiveProcessing_Period::$enablePurgeOutdated = false; + Piwik_ArchiveProcessor_Rules::$purgeDisabledByTests = false; self::_addLogData(); self::_addReportData(); - Piwik_ArchiveProcessing_Period::$enablePurgeOutdated = true; + Piwik_ArchiveProcessor_Rules::$purgeDisabledByTests = true; self::$dbData = self::getDbTablesWithData(); } |