diff options
author | Benaka Moorthi <benaka.moorthi@gmail.com> | 2013-05-31 11:36:25 +0400 |
---|---|---|
committer | Benaka Moorthi <benaka.moorthi@gmail.com> | 2013-05-31 11:36:25 +0400 |
commit | 6abbacb9575087161ade571de2b0ad373b03d115 (patch) | |
tree | 4da27e6b31ffc2f62e0cb0b7ee2cecfd15de2c22 /core | |
parent | cbc929d0e6a10337c29c3459048611c23a2d2d65 (diff) |
Refactored archive data querying code. Removed tree-like data structure from Piwik_Archive, added memory optimizations for archiving (not storing archive processing instances), added optimization for querying data indexed by site + date, added ability to select multiple blobs/metrics at once (from different plugins), added optimization to Piwik_ArchiveProcessing_Period where numerics/records are selected all at once instead of one at a time.
Also added data access layer object for archive querying code, allowed anonymous function objects to be used as DataTable filters, and removed 'timestamp' DataTable metadata.
Diffstat (limited to 'core')
-rw-r--r-- | core/Archive.php | 816 | ||||
-rw-r--r-- | core/Archive/Array.php | 147 | ||||
-rw-r--r-- | core/Archive/Array/IndexedByDate.php | 131 | ||||
-rw-r--r-- | core/Archive/Array/IndexedBySite.php | 266 | ||||
-rw-r--r-- | core/Archive/DataCollection.php | 326 | ||||
-rw-r--r-- | core/Archive/DataTableFactory.php | 363 | ||||
-rw-r--r-- | core/Archive/Single.php | 632 | ||||
-rw-r--r-- | core/ArchiveProcessing.php | 30 | ||||
-rw-r--r-- | core/ArchiveProcessing/Day.php | 6 | ||||
-rw-r--r-- | core/ArchiveProcessing/Period.php | 249 | ||||
-rw-r--r-- | core/DataAccess/ArchiveQuery.php | 189 | ||||
-rw-r--r-- | core/DataTable.php | 20 | ||||
-rw-r--r-- | core/DataTable/Manager.php | 2 | ||||
-rw-r--r-- | core/DataTable/Renderer/Rss.php | 2 | ||||
-rw-r--r-- | core/Period.php | 5 | ||||
-rw-r--r-- | core/Site.php | 5 | ||||
-rw-r--r-- | core/TablePartitioning.php | 40 |
17 files changed, 1770 insertions, 1459 deletions
diff --git a/core/Archive.php b/core/Archive.php index 5492c1b4a8..42690fb379 100644 --- a/core/Archive.php +++ b/core/Archive.php @@ -12,9 +12,14 @@ /** * The archive object is used to query specific data for a day or a period of statistics for a given website. * + * Limitations: + * - If you query w/ a range period, you can only query for ONE at a time. + * - If you query w/ a non-range period, you can query for multiple periods, but they must + * all be of the same type (ie, day, week, month, year). + * * Example: * <pre> - * $archive = Piwik_Archive::build($idSite = 1, $period = 'week', '2008-03-08' ); + * $archive = Piwik_Archive::build($idSite = 1, $period = 'week', '2008-03-08'); * $dataTable = $archive->getDataTable('Provider_hostnameExt'); * $dataTable->queueFilter('ReplaceColumnNames'); * return $dataTable; @@ -22,17 +27,19 @@ * * Example bis: * <pre> - * $archive = Piwik_Archive::build($idSite = 3, $period = 'day', $date = 'today' ); + * $archive = Piwik_Archive::build($idSite = 3, $period = 'day', $date = 'today'); * $nbVisits = $archive->getNumeric('nb_visits'); * return $nbVisits; * </pre> * * If the requested statistics are not yet processed, Archive uses ArchiveProcessing to archive the statistics. - * + * + * TODO: create ticket for this: when building archives, should use each site's timezone (ONLY FOR 'now'). + * * @package Piwik * @subpackage Piwik_Archive */ -abstract class Piwik_Archive +class Piwik_Archive { /** * When saving DataTables in the DB, we sometimes replace the columns name by these IDs so we save up lots of bytes @@ -185,86 +192,230 @@ abstract class Piwik_Archive const LABEL_ECOMMERCE_CART = 'ecommerceAbandonedCart'; const LABEL_ECOMMERCE_ORDER = 'ecommerceOrder'; - + /** - * Website Piwik_Site - * - * @var Piwik_Site + * The list of site IDs to query archive data for. + * + * @var array */ - protected $site = null; - + private $siteIds; + + /** + * The list of Piwik_Period's to query archive data for. + * + * @var array + */ + private $periods; + /** - * Segment applied to the visits set + * Segment applied to the visits set. + * * @var Piwik_Segment */ - protected $segment = false; + private $segment; + + /** + * List of archive IDs for the sites, periods and segment we are querying with. + * Archive IDs are indexed by done flag and period, ie: + * + * array( + * 'done.Referers' => array( + * '2010-01-01' => 1, + * '2010-01-02' => 2, + * ), + * 'done.VisitsSummary' => array( + * '2010-01-01' => 3, + * '2010-01-02' => 4, + * ), + * ) + * + * or, + * + * array( + * 'done.all' => array( + * '2010-01-01' => 1, + * '2010-01-02' => 2 + * ) + * ) + * + * @var array + */ + private $idarchives = array(); + + /** + * If set to true, the result of all get functions (ie, getNumeric, getBlob, etc.) + * will be indexed by the site ID, even if we're only querying data for one site. + * + * @var bool + */ + private $forceIndexedBySite; + + /** + * If set to true, the result of all get functions (ie, getNumeric, getBlob, etc.) + * will be indexed by the period, even if we're only querying data for one period. + * + * @var bool + */ + private $forceIndexedByDate; + + /** + * Data Access Layer object. + * + * @var Piwik_DataAccess_ArchiveQuery + */ + private $dataAccess; + + /** + * Cache of Piwik_ArchiveProcessing instances used when launching the archiving + * process. + * + * @var array + */ + private $processingCache = array(); + + /** + * Constructor. + * + * @param array|int $siteIds List of site IDs to query data for. + * @param array|Piwik_Period $periods List of periods to query data for. + * @param Piwik_Segment $segment The segment used to narrow the visits set. + * @param bool $forceIndexedBySite Whether to force index the result of a query by site ID. + * @param bool $forceIndexedByDate Whether to force index the result of a query by period. + */ + public function __construct($siteIds, $periods, Piwik_Segment $segment, $forceIndexedBySite = false, + $forceIndexedByDate = false) + { + $this->siteIds = $this->getAsNonEmptyArray($siteIds, 'siteIds'); + + $periods = $this->getAsNonEmptyArray($periods, 'periods'); + $this->periods = array(); + foreach ($periods as $period) { + $this->periods[$period->getRangeString()] = $period; + } + + $this->segment = $segment; + $this->forceIndexedBySite = $forceIndexedBySite; + $this->forceIndexedByDate = $forceIndexedByDate; + $this->dataAccess = new Piwik_DataAccess_ArchiveQuery(); + } + + /** + * Destructor. + */ + public function __destruct() + { + $this->periods = null; + $this->siteIds = null; + $this->segment = null; + $this->idarchives = array(); + $this->processingCache = array(); + } + + /** + * Returns the IDs of sites we are querying archive data for. + * + * @return array + */ + public function getSiteIds() + { + return $this->siteIds; + } + + /** + * Returns the periods we are querying archive data for. + * + * @return array + */ + public function getPeriods() + { + return $this->periods; + } + + /** + * Returns the segment used to limit the visit set. + * + * @return Piwik_Segment|null + */ + public function getSegment() + { + return $this->segment; + } /** - * Builds an Archive object or returns the same archive if previously built. + * Builds an Archive object using query parameter values. * - * @param int|string $idSite integer, or comma separated list of integer - * @param string $period 'week' 'day' etc. - * @param Piwik_Date|string $strDate 'YYYY-MM-DD' or magic keywords 'today' @see Piwik_Date::factory() - * @param bool|string $segment Segment definition - defaults to false for Backward Compatibility - * @param bool|string $_restrictSitesToLogin Used only when running as a scheduled task + * @param int|string $idSite Integer, or comma separated list of integer site IDs. + * @param string $period 'day', 'week', 'month', 'year' or 'range' + * @param Piwik_Date|string $strDate 'YYYY-MM-DD', magic keywords (ie, 'today'; @see Piwik_Date::factory()) + * or date range (ie, 'YYYY-MM-DD,YYYY-MM-DD'). + * @param false|string $segment Segment definition - defaults to false for backward compatibility. + * @param false|string $_restrictSitesToLogin Used only when running as a scheduled task. * @return Piwik_Archive */ public static function build($idSite, $period, $strDate, $segment = false, $_restrictSitesToLogin = false) { - if ($idSite === 'all') { - $sites = Piwik_SitesManager_API::getInstance()->getSitesIdWithAtLeastViewAccess($_restrictSitesToLogin); - } else { - $sites = Piwik_Site::getIdSitesFromIdSitesString($idSite); - } - - if (!($segment instanceof Piwik_Segment)) { - $segment = new Piwik_Segment($segment, $idSite); + $forceIndexedBySite = false; + $forceIndexedByDate = false; + + // determine site IDs to query from + if (is_array($idSite) + || $idSite == 'all' + ) { + $forceIndexedBySite = true; } - - // idSite=1,3 or idSite=all - if ($idSite === 'all' - || is_array($idSite) - || count($sites) > 1 + $sites = Piwik_Site::getIdSitesFromIdSitesString($idSite, $_restrictSitesToLogin); + + // if a period date string is detected: either 'last30', 'previous10' or 'YYYY-MM-DD,YYYY-MM-DD' + if (is_string($strDate) + && self::isMultiplePeriod($strDate, $period) ) { - $archive = new Piwik_Archive_Array_IndexedBySite($sites, $period, $strDate, $segment, $_restrictSitesToLogin); - } // if a period date string is detected: either 'last30', 'previous10' or 'YYYY-MM-DD,YYYY-MM-DD' - elseif (is_string($strDate) && self::isMultiplePeriod($strDate, $period)) { - $oSite = new Piwik_Site($idSite); - $archive = new Piwik_Archive_Array_IndexedByDate($oSite, $period, $strDate, $segment); - } // case we request a single archive - else { - $oSite = new Piwik_Site($idSite); + $oPeriod = new Piwik_Period_Range($period, $strDate); + $allPeriods = $oPeriod->getSubperiods(); + $forceIndexedByDate = true; + } else { + if (count($sites) == 1) { + $oSite = new Piwik_Site($sites[0]); + } else { + $oSite = null; + } + $oPeriod = Piwik_Archive::makePeriodFromQueryParams($oSite, $period, $strDate); - - $archive = new Piwik_Archive_Single(); - $archive->setPeriod($oPeriod); - $archive->setSite($oSite); - $archive->setSegment($segment); + $allPeriods = array($oPeriod); } - return $archive; + + return new Piwik_Archive( + $sites, $allPeriods, new Piwik_Segment($segment, $sites), $forceIndexedBySite, $forceIndexedByDate); } /** * Creates a period instance using a Piwik_Site instance and two strings describing * the period & date. * - * @param Piwik_Site $site + * @param Piwik_Site|null $site * @param string $strPeriod The period string: day, week, month, year, range * @param string $strDate The date or date range string. * @return Piwik_Period */ public static function makePeriodFromQueryParams($site, $strPeriod, $strDate) { - $tz = $site->getTimezone(); + if ($site === null) { + $tz = 'UTC'; + } else { + $tz = $site->getTimezone(); + } if ($strPeriod == 'range') { $oPeriod = new Piwik_Period_Range('range', $strDate, $tz, Piwik_Date::factory('today', $tz)); } else { $oDate = $strDate; if (!($strDate instanceof Piwik_Date)) { - if ($strDate == 'now' || $strDate == 'today') { + if ($strDate == 'now' + || $strDate == 'today' + ) { $strDate = date('Y-m-d', Piwik_Date::factory('now', $tz)->getTimestamp()); - } elseif ($strDate == 'yesterday' || $strDate == 'yesterdaySameTime') { + } elseif ($strDate == 'yesterday' + || $strDate == 'yesterdaySameTime' + ) { $strDate = date('Y-m-d', Piwik_Date::factory('now', $tz)->subDay(1)->getTimestamp()); } $oDate = Piwik_Date::factory($strDate); @@ -275,63 +426,197 @@ abstract class Piwik_Archive return $oPeriod; } - - abstract public function prepareArchive(); - + /** - * Returns the value of the element $name from the current archive + * Returns the value of the element $name from the current archive * The value to be returned is a numeric value and is stored in the archive_numeric_* tables * - * @param string $name For example Referers_distinctKeywords - * @return float|int|false False if no value with the given name + * @param string|array $names One or more archive names, eg, 'nb_visits', 'Referers_distinctKeywords', + * etc. + * @return numeric|array|false False if no value with the given name, numeric if only one site + * and date and we're not forcing an index, and array if multiple + * sites/dates are queried. */ - abstract public function getNumeric($name); - + public function getNumeric($names) + { + $data = $this->get($names, 'numeric'); + + $resultIndices = $this->getResultIndices(); + $result = $data->getArray($resultIndices); + + // if only one metric is returned, just return it as a numeric value + if (empty($resultIndices) + && count($result) <= 1 + ) { + $result = (float)reset($result); // convert to float in case $result is empty + } + + return $result; + } + /** - * Returns the value of the element $name from the current archive - * - * The value to be returned is a blob value and is stored in the archive_numeric_* tables - * + * Returns the value of the elements in $names from the current archive. + * + * The value to be returned is a blob value and is stored in the archive_blob_* tables. + * * It can return anything from strings, to serialized PHP arrays or PHP objects, etc. * - * @param string $name For example Referers_distinctKeywords - * @return mixed False if no value with the given name + * @param string|array $names One or more archive names, eg, 'Referers_keywordBySearchEngine'. + * @return string|array|false False if no value with the given name, numeric if only one site + * and date and we're not forcing an index, and array if multiple + * sites/dates are queried. */ - abstract public function getBlob($name); - + public function getBlob($names, $idSubtable = null) + { + $data = $this->get($names, 'blob', $idSubtable); + return $data->getArray($this->getResultIndices()); + } + /** - * - * @param $fields - * @return Piwik_DataTable + * Returns the numeric values of the elements in $names as a DataTable. + * + * Note: Every DataTable instance returned will have at most one row in it. + * + * @param string|array $names One or more archive names, eg, 'nb_visits', 'Referers_distinctKeywords', + * etc. + * @return Piwik_DataTable|false False if no value with the given names. Based on the number + * of sites/periods, the result can be a DataTable_Array, which + * contains DataTable instances. */ - abstract public function getDataTableFromNumeric($fields); + public function getDataTableFromNumeric($names) + { + $data = $this->get($names, 'numeric'); + return $data->getDataTable($this->getResultIndices()); + } /** * This method will build a dataTable from the blob value $name in the current archive. + * + * For example $name = 'Referers_searchEngineByKeyword' will return a + * Piwik_DataTable containing all the keywords. If a $idSubtable is given, the method + * will return the subTable of $name. If 'all' is supplied for $idSubtable every subtable + * will be returned. + * + * @param string $name The name of the record to get. + * @param int|string|null $idSubtable The subtable ID (if any) or 'all' if requesting every datatable. + * @return Piwik_DataTable|false + */ + public function getDataTable($name, $idSubtable = null) + { + $data = $this->get($name, 'blob', $idSubtable); + return $data->getDataTable($this->getResultIndices()); + } + + /** + * Same as getDataTable() except that it will also load in memory all the subtables + * for the DataTable $name. You can then access the subtables by using the + * Piwik_DataTable_Manager::getTable() function. * - * For example $name = 'Referers_searchEngineByKeyword' will return a Piwik_DataTable containing all the keywords - * If a idSubTable is given, the method will return the subTable of $name - * - * @param string $name - * @param int $idSubTable or null if requesting the parent table + * @param string $name The name of the record to get. + * @param int|string|null $idSubtable The subtable ID (if any) or 'all' if requesting every datatable. + * @param bool $addMetadataSubtableId Whether to add the DB subtable ID as metadata to each datatable, + * or not. * @return Piwik_DataTable - * @throws exception If the value cannot be found */ - abstract public function getDataTable($name, $idSubTable = null); + public function getDataTableExpanded($name, $idSubtable = null, $addMetadataSubtableId = true) + { + $data = $this->get($name, 'blob', 'all'); + return $data->getExpandedDataTable($this->getResultIndices(), $idSubtable, $addMetadataSubtableId); + } + + /** + * Returns true if we shouldn't launch the archiving process and false if we should. + * + * @return bool + */ + public function isArchivingDisabled() + { + return Piwik_ArchiveProcessing::isArchivingDisabledFor($this->segment, $this->getPeriodLabel()); + } /** - * Same as getDataTable() except that it will also load in memory - * all the subtables for the DataTable $name. - * You can then access the subtables by using the Piwik_DataTable_Manager getTable() + * Returns true if Segmentation is allowed for this user * - * @param string $name - * @param int|null $idSubTable null if requesting the parent table - * @return Piwik_DataTable + * @return bool */ - abstract public function getDataTableExpanded($name, $idSubTable = null); + public static function isSegmentationEnabled() + { + return !Piwik::isUserIsAnonymous() + || Piwik_Config::getInstance()->General['anonymous_user_enable_use_segments_API']; + } + /** + * Indicate if $dateString and $period correspond to multiple periods + * + * @static + * @param $dateString + * @param $period + * @return boolean + */ + public static function isMultiplePeriod($dateString, $period) + { + return (preg_match('/^(last|previous){1}([0-9]*)$/D', $dateString, $regs) + || Piwik_Period_Range::parseDateRange($dateString)) + && $period != 'range'; + } /** + * Indicate if $idSiteString corresponds to multiple sites. + * + * @param string $idSiteString + * @return bool + */ + public static function isMultipleSites($idSiteString) + { + return $idSiteString == 'all' || strpos($idSiteString, ',') !== false; + } + + /** + * Returns the report names for a list of metric/record names. + * + * @see getRequestedReport + * + * @param array $archiveNames + */ + public function getRequestedReports($archiveNames) + { + $result = array(); + foreach ($archiveNames as $name) { + $result[] = self::getRequestedReport($name); + } + return array_unique($result); + } + + /** + * Returns the report name for a metric/record name. + * + * A report name has the following format: {$pluginName}_{$reportId}, eg. VisitFrequency_Metrics. + * The report ID is not used anywhere in Piwik. + */ + public static function getRequestedReport($archiveName) + { + // Core metrics are always processed in Core, for the requested date/period/segment + if (in_array($archiveName, Piwik_ArchiveProcessing::getCoreMetrics()) + || $archiveName == 'max_actions' + ) { + return 'VisitsSummary_CoreMetrics'; + } + // VisitFrequency metrics don't follow the same naming convention (HACK) + else if(strpos($archiveName, '_returning') > 0 + // ignore Goal_visitor_returning_1_1_nb_conversions + && strpos($archiveName, 'Goal_') === false + ) { + return 'VisitFrequency_Metrics'; + } + // Goal_* metrics are processed by the Goals plugin (HACK) + else if(strpos($archiveName, 'Goal_') === 0) { + return 'Goals_Metrics'; + } else { + return $archiveName; + } + } + + /** * Helper - Loads a DataTable from the Archive. * Optionally loads the table recursively, * or optionally fetches a given subtable with $idSubtable @@ -363,98 +648,337 @@ abstract class Piwik_Archive return $dataTable; } - - protected function formatNumericValue($value) + + /** + * Queries archive tables for data and returns the result. + */ + private function get($archiveNames, $archiveDataType, $idSubtable = null) { - // If there is no dot, we return as is - // Note: this could be an integer bigger than 32 bits - if (strpos($value, '.') === false) { - if ($value === false) { - return 0; + if (!is_array($archiveNames)) { + $archiveNames = array($archiveNames); + } + + // apply idSubtable + if ($idSubtable !== null + && $idSubtable != 'all' + ) { + foreach ($archiveNames as &$name) { + $name .= "_$idSubtable"; } - return (float)$value; } - - // Round up the value with 2 decimals - // we cast the result as float because returns false when no visitors - $value = round((float)$value, 2); - return $value; - } - - public function getSegment() - { - return $this->segment; + + $result = new Piwik_Archive_DataCollection( + $archiveNames, $archiveDataType, $this->siteIds, $this->periods, $defaultRow = null); + + $archiveIds = $this->getArchiveIds($archiveNames); + if (empty($archiveIds)) { + return $result; + } + + $archiveData = $this->dataAccess->getArchiveData($archiveIds, $archiveNames, $archiveDataType, $idSubtable); + foreach ($archiveData as $row) { + // values are grouped by idsite (site ID), date1-date2 (date range), then name (field name) + $idSite = $row['idsite']; + $periodStr = $row['date1'].",".$row['date2']; + + if ($archiveDataType == 'numeric') { + $value = $this->formatNumericValue($row['value']); + } else { + $value = $this->uncompress($row['value']); + $result->addMetadata($idSite, $periodStr, 'ts_archived', $row['ts_archived']); + } + + $resultRow = &$result->get($idSite, $periodStr); + $resultRow[$row['name']] = $value; + } + + return $result; } - - public function setSegment(Piwik_Segment $segment) + + /** + * Returns archive IDs for the sites, periods and archive names that are being + * queried. This function will use the idarchive cache if it has the right data, + * query archive tables for IDs w/o launching archiving, or launch archiving and + * get the idarchive from Piwik_ArchiveProcessing instances. + */ + private function getArchiveIds($archiveNames) { - $this->segment = $segment; + $requestedReports = $this->getRequestedReports($archiveNames); + + // figure out which archives haven't been processed (if an archive has been processed, + // then we have the archive IDs in $this->idarchives) + $doneFlags = array(); + $archiveGroups = array(); + foreach ($requestedReports as $report) { + $doneFlag = Piwik_ArchiveProcessing::getDoneStringFlagFor( + $this->segment, $this->getPeriodLabel(), $report); + + $doneFlags[$doneFlag] = true; + if (!isset($this->idarchives[$doneFlag])) { + $archiveGroups[] = $this->getArchiveGroupOfReport($report); + } + } + + $archiveGroups = array_unique($archiveGroups); + + // cache id archives for plugins we haven't processed yet + if (!empty($archiveGroups)) { + if (!$this->isArchivingDisabled()) { + $this->cacheArchiveIdsAfterLaunching($archiveGroups, $requestedReports); + } else { + $this->cacheArchiveIdsWithoutLaunching($requestedReports); + } + } + + // order idarchives by the table month they belong to + $idArchivesByMonth = array(); + foreach (array_keys($doneFlags) as $doneFlag) { + if (empty($this->idarchives[$doneFlag])) { + continue; + } + + foreach ($this->idarchives[$doneFlag] as $dateRange => $idarchives) { + foreach ($idarchives as $id) { + $idArchivesByMonth[$dateRange][] = $id; + } + } + } + + return $idArchivesByMonth; } - + /** - * Sets the site - * - * @param Piwik_Site $site + * Gets the IDs of the archives we're querying for and stores them in $this->archives. + * This function will launch the archiving process for each period/site/plugin if + * metrics/reports have not been calculated/archived already. + * + * @param array $archiveGroups @see getArchiveGroupOfReport + * @param array $requestedReports @see getRequestedReport */ - public function setSite(Piwik_Site $site) + private function cacheArchiveIdsAfterLaunching($archiveGroups, $requestedReports) { - $this->site = $site; + $today = Piwik_Date::today(); + + // for every individual query permutation, launch the archiving process and get the archive ID + foreach ($this->periods as $period) { + $periodStr = $period->getRangeString(); + + $twoDaysBeforePeriod = $period->getDateStart()->subDay(2); + $twoDaysAfterPeriod = $period->getDateEnd()->addDay(2); + + foreach ($this->siteIds as $idSite) { + $site = new Piwik_Site($idSite); + + // if the END of the period is BEFORE the website creation date + // we already know there are no stats for this period + // we add one day to make sure we don't miss the day of the website creation + if ($twoDaysAfterPeriod->isEarlier($site->getCreationDate())) { + $archiveDesc = $this->getArchiveDescriptor($idSite, $period); + Piwik::log("Archive $archiveDesc skipped, archive is before the website was created."); + continue; + } + + // if the starting date is in the future we know there is no visit + if ($twoDaysBeforePeriod->isLater($today)) { + $archiveDesc = $this->getArchiveDescriptor($idSite, $period); + Piwik::log("Archive $archiveDesc skipped, archive is after today."); + continue; + } + + // prepare the ArchiveProcessing instance + $processing = $this->getArchiveProcessingInstance($period); + $processing->setSite($site); + $processing->setPeriod($period); + $processing->setSegment($this->segment); + + $processing->isThereSomeVisits = null; + + // process for each requested report as well + foreach ($archiveGroups as $pluginOrAll) { + if ($pluginOrAll == 'all') { + $pluginOrAll = $this->getPluginForReport(reset($requestedReports)); + } + $report = $pluginOrAll.'_reportsAndMetrics'; + + $doneFlag = Piwik_ArchiveProcessing::getDoneStringFlagFor( + $this->segment, $period->getLabel(), $report); + $this->initializeArchiveIdCache($doneFlag); + + $processing->init(); + $processing->setRequestedReport($report); + + // launch archiving if the requested data hasn't been archived + $idArchive = $processing->loadArchive(); + if (empty($idArchive)) { + $processing->launchArchiving(); + $idArchive = $processing->getIdArchive(); + } + + if (!$processing->isThereSomeVisits()) { + continue; + } + + $this->idarchives[$doneFlag][$periodStr][] = $idArchive; + } + } + } } - + /** - * Gets the site - * - * @return Piwik_Site + * Gets the IDs of the archives we're querying for and stores them in $this->archives. + * This function will not launch the archiving process (and is thus much, much faster + * than cacheArchiveIdsAfterLaunching). + * + * @param array $requestedReports @see getRequestedReport */ - public function getSite() + private function cacheArchiveIdsWithoutLaunching($requestedReports) { - return $this->site; + $periodType = $this->getPeriodLabel(); + + $idarchivesByReport = $this->dataAccess->getArchiveIds( + $this->siteIds, $this->periods, $this->segment, $requestedReports); + + // initialize archive ID cache for each report + foreach ($requestedReports as $report) { + $doneFlag = Piwik_ArchiveProcessing::getDoneStringFlagFor($this->segment, $periodType, $report); + $this->initializeArchiveIdCache($doneFlag); + } + + foreach ($idarchivesByReport as $doneFlag => $idarchivesByDate) { + foreach ($idarchivesByDate as $dateRange => $idarchives) { + foreach ($idarchives as $idarchive) { + $this->idarchives[$doneFlag][$dateRange][] = $idarchive; + } + } + } } - + /** - * Returns the Id site associated with this archive - * - * @return int + * Returns an ArchiveProcessing instance that should be used for a specific + * period. + * + * @param Piwik_Period $period */ - public function getIdSite() + private function getArchiveProcessingInstance($period) + { + $label = $period->getLabel(); + if (!isset($this->processingCache[$label])) { + $this->processingCache[$label] = Piwik_ArchiveProcessing::factory($label); + } + return $this->processingCache[$label]; + } + + private function getPeriodLabel() { - return $this->site->getId(); + return reset($this->periods)->getLabel(); } - + /** - * Returns true if Segmentation is allowed for this user - * - * @return bool + * Returns an array describing what metadata to use when indexing a query result. + * For use with Piwik_Archive_DataCollection. + * + * @return array */ - public static function isSegmentationEnabled() + private function getResultIndices() { - return !Piwik::isUserIsAnonymous() - || Piwik_Config::getInstance()->General['anonymous_user_enable_use_segments_API']; + $indices = array(); + + if (count($this->siteIds) > 1 + || $this->forceIndexedBySite + ) { + $indices['site'] = 'idSite'; + } + + if (count($this->periods) > 1 + || $this->forceIndexedByDate + ) { + $indices['period'] = 'date'; + } + + return $indices; } + private function formatNumericValue($value) + { + // If there is no dot, we return as is + // Note: this could be an integer bigger than 32 bits + if (strpos($value, '.') === false) { + if ($value === false) { + return 0; + } + return (float)$value; + } + + // Round up the value with 2 decimals + // we cast the result as float because returns false when no visitors + return round((float)$value, 2); + } + + private function getArchiveDescriptor($idSite, $period) + { + return "site $idSite, {$period->getLabel()} ({$period->getPrettyString()})"; + } + + private function uncompress($data) + { + return @gzuncompress($data); + } + + private function getAsNonEmptyArray($array, $paramName) + { + if (!is_array($array)) { + $array = array($array); + } + + if (empty($array)) { + throw new Exception("Piwik_Archive::__construct: \$$paramName is empty."); + } + + return $array; + } + /** - * Indicate if $dateString and $period correspond to multiple periods - * - * @static - * @param $dateString - * @param $period - * @return boolean + * Initializes the archive ID cache ($this->idarchives) for a particular 'done' flag. + * + * It is necessary that each archive ID caching function call this method for each + * unique 'done' flag it encounters, since the getArchiveIds function determines + * whether archiving should be launched based on whether $this->idarchives has a + * an entry for a specific 'done' flag. + * + * If this function is not called, then periods with no visits will not add + * entries to the cache. If the archive is used again, SQL will be executed to + * try and find the archive IDs even though we know there are none. */ - public static function isMultiplePeriod($dateString, $period) + private function initializeArchiveIdCache($doneFlag) { - return (preg_match('/^(last|previous){1}([0-9]*)$/D', $dateString, $regs) - || Piwik_Period_Range::parseDateRange($dateString)) - && $period != 'range'; + if (!isset($this->idarchives[$doneFlag])) { + $this->idarchives[$doneFlag] = array(); + } } - + /** - * Indicate if $idSiteString corresponds to multiple sites. - * - * @param string $idSiteString - * @return bool + * Returns the archiving group identifier of a report. + * + * More than one plugin can be called at once. In such a case we don't want to + * launch archiving three times for three plugins if doing it once is enough, + * so getArchiveIds makes sure to get the archive group of all reports. + * + * If the period isn't a range, then all plugins' archiving code is executed. + * If the period is a range, then archiving code is executed individually for + * each plugin. */ - public static function isMultipleSites($idSiteString) + private function getArchiveGroupOfReport($report) { - return $idSiteString == 'all' || strpos($idSiteString, ',') !== false; + if ($this->getPeriodLabel() != 'range') { + return 'all'; + } + + return $this->getPluginForReport($report); + } + + private function getPluginForReport($report) + { + $parts = explode('_', $report); + return $parts[0]; } } diff --git a/core/Archive/Array.php b/core/Archive/Array.php deleted file mode 100644 index 896cc1e17c..0000000000 --- a/core/Archive/Array.php +++ /dev/null @@ -1,147 +0,0 @@ -<?php -/** - * Piwik - Open source web analytics - * - * @link http://piwik.org - * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later - * - * @category Piwik - * @package Piwik - */ - -/** - * Piwik_Archive_Array is used to store multiple archives, - * for example one archive for a given day for each Piwik website - * - * @package Piwik - * @subpackage Piwik_Archive - */ -abstract class Piwik_Archive_Array extends Piwik_Archive -{ - /** - * This array contains one Piwik_Archive per entry in the period - * - * @var Piwik_Archive[] - */ - protected $archives = array(); - - abstract protected function getIndexName(); - - abstract protected function getDataTableLabelValue($archive); - - /** - * Destructor - */ - public function __destruct() - { - foreach ($this->archives as $archive) { - destroy($archive); - } - $this->archives = array(); - } - - /** - * Prepares each archive in the array - */ - public function prepareArchive() - { - foreach ($this->archives as $archive) { - $archive->prepareArchive(); - } - } - - /** - * Returns a newly created Piwik_DataTable_Array. - * - * @return Piwik_DataTable_Array - */ - protected function getNewDataTableArray() - { - $table = new Piwik_DataTable_Array(); - $table->setKeyName($this->getIndexName()); - return $table; - } - - /** - * Returns a DataTable_Array containing numeric values - * of the element $name from the archives in this Archive_Array. - * - * @param string $name Name of the mysql table field to load eg. Referers_distinctKeywords - * @return Piwik_DataTable_Array containing the requested numeric value for each Archive - */ - public function getNumeric($name) - { - $table = $this->getNewDataTableArray(); - - foreach ($this->archives as $archive) { - $numeric = $archive->getNumeric($name); - $subTable = $archive->makeDataTable($isSimple = true); - $subTable->addRowsFromArray(array($numeric)); - $table->addTable($subTable, $this->getDataTableLabelValue($archive)); - } - - return $table; - } - - /** - * Returns a DataTable_Array containing values - * of the element $name from the archives in this Archive_Array. - * - * The value to be returned are blob values (stored in the archive_numeric_* tables in the DB). * - * It can return anything from strings, to serialized PHP arrays or PHP objects, etc. - * - * @param string $name Name of the mysql table field to load eg. Referers_keywordBySearchEngine - * @return Piwik_DataTable_Array containing the requested blob values for each Archive - */ - public function getBlob($name) - { - $table = $this->getNewDataTableArray(); - - foreach ($this->archives as $archive) { - $blob = $archive->getBlob($name); - $subTable = $archive->makeNewDataTable($isSimple = true); - $subTable->addRowsFromArray(array('blob' => $blob)); - $table->addTable($subTable, $this->getDataTableLabelValue($archive)); - } - return $table; - } - - /** - * Given a BLOB field name (eg. 'Referers_searchEngineByKeyword'), it will return a Piwik_DataTable_Array - * which is an array of Piwik_DataTable, ordered by chronological order - * - * @param string $name Name of the mysql table field to load - * @param int $idSubTable optional idSubDataTable - * @return Piwik_DataTable_Array - * @throws Exception If the value cannot be found - */ - public function getDataTable($name, $idSubTable = null) - { - $table = $this->getNewDataTableArray(); - foreach ($this->archives as $archive) { - $subTable = $archive->getDataTable($name, $idSubTable); - $table->addTable($subTable, $this->getDataTableLabelValue($archive)); - } - return $table; - } - - - /** - * Same as getDataTable() except that it will also load in memory - * all the subtables for the DataTable $name. - * You can then access the subtables by using the Piwik_DataTable_Manager::getInstance()->getTable($idSubTable); - * - * @param string $name Name of the mysql table field to load - * @param int $idSubTable optional idSubDataTable - * @return Piwik_DataTable_Array - */ - public function getDataTableExpanded($name, $idSubTable = null) - { - $table = $this->getNewDataTableArray(); - foreach ($this->archives as $archive) { - $subTable = $archive->getDataTableExpanded($name, $idSubTable); - $table->addTable($subTable, $this->getDataTableLabelValue($archive)); - } - return $table; - } -} diff --git a/core/Archive/Array/IndexedByDate.php b/core/Archive/Array/IndexedByDate.php deleted file mode 100644 index 9a59712d4a..0000000000 --- a/core/Archive/Array/IndexedByDate.php +++ /dev/null @@ -1,131 +0,0 @@ -<?php -/** - * Piwik - Open source web analytics - * - * @link http://piwik.org - * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later - * - * @category Piwik - * @package Piwik - */ - -/** - * @package Piwik - * @subpackage Piwik_Archive - */ -class Piwik_Archive_Array_IndexedByDate extends Piwik_Archive_Array -{ - /** - * Builds an array of Piwik_Archive of a given date range - * - * @param Piwik_Site $oSite - * @param string $strPeriod eg. 'day' 'week' etc. - * @param string $strDate A date range, eg. 'last10', 'previous5' or 'YYYY-MM-DD,YYYY-MM-DD' - * @param Piwik_Segment $segment - */ - public function __construct(Piwik_Site $oSite, $strPeriod, $strDate, Piwik_Segment $segment) - { - $rangePeriod = new Piwik_Period_Range($strPeriod, $strDate, $oSite->getTimezone()); - foreach ($rangePeriod->getSubperiods() as $subPeriod) { - $startDate = $subPeriod->getDateStart(); - $archive = Piwik_Archive::build($oSite->getId(), $strPeriod, $startDate, $segment->getString()); - $archive->setSegment($segment); - $this->archives[] = $archive; - } - $this->setSite($oSite); - } - - /** - * @return string - */ - protected function getIndexName() - { - return 'date'; - } - - /** - * @param Piwik_Archive $archive - * @return mixed - */ - protected function getDataTableLabelValue($archive) - { - return $archive->getPrettyDate(); - } - - /** - * Given a list of fields defining numeric values, it will return a Piwik_DataTable_Array - * which is an array of Piwik_DataTable_Simple, ordered by chronological order - * - * @param array|string $fields array( fieldName1, fieldName2, ...) Names of the mysql table fields to load - * @return Piwik_DataTable_Array - */ - public function getDataTableFromNumeric($fields) - { - $inNames = Piwik_Common::getSqlStringFieldsArray($fields); - - // we select in different shots - // one per distinct table (case we select last 300 days, maybe we will select from 10 different tables) - $queries = array(); - foreach ($this->archives as $archive) { - $archive->setRequestedReport(is_string($fields) ? $fields : current($fields)); - $archive->prepareArchive(); - if (!$archive->isThereSomeVisits) { - continue; - } - - $table = $archive->archiveProcessing->getTableArchiveNumericName(); - - // for every query store IDs - $queries[$table][] = $archive->getIdArchive(); - } - // we select the requested value - $db = Zend_Registry::get('db'); - - // date => array( 'field1' =>X, 'field2'=>Y) - // date2 => array( 'field1' =>X2, 'field2'=>Y2) - - $arrayValues = array(); - foreach ($queries as $table => $aIds) { - $inIds = implode(', ', array_filter($aIds)); - if (empty($inIds)) { - // Probable timezone configuration error, i.e., mismatch between PHP and MySQL server. - continue; - } - - $sql = "SELECT value, name, date1 as startDate - FROM $table - WHERE idarchive IN ( $inIds ) - AND name IN ( $inNames ) - ORDER BY date1, name"; - $values = $db->fetchAll($sql, $fields); - foreach ($values as $value) { - $timestamp = Piwik_Date::factory($value['startDate'])->getTimestamp(); - $arrayValues[$timestamp][$value['name']] = $this->formatNumericValue($value['value']); - } - } - - $contentArray = array(); - // we add empty tables so that every requested date has an entry, even if there is nothing - // example: <result date="2007-01-01" /> - $archiveByTimestamp = array(); - foreach ($this->archives as $archive) { - $timestamp = $archive->getTimestampStartDate(); - $archiveByTimestamp[$timestamp] = $archive; - $contentArray[$timestamp]['table'] = $archive->makeDataTable($simple = true); - $contentArray[$timestamp]['prettyDate'] = $archive->getPrettyDate(); - } - - foreach ($arrayValues as $timestamp => $aNameValues) { - // undefined in some edge/unknown cases see http://dev.piwik.org/trac/ticket/2578 - if (isset($contentArray[$timestamp]['table'])) { - $contentArray[$timestamp]['table']->addRowsFromArray($aNameValues); - } - } - - $tableArray = $this->getNewDataTableArray(); - foreach ($contentArray as $timestamp => $aData) { - $tableArray->addTable($aData['table'], $aData['prettyDate']); - } - return $tableArray; - } -} diff --git a/core/Archive/Array/IndexedBySite.php b/core/Archive/Array/IndexedBySite.php deleted file mode 100644 index 1ac9ff682a..0000000000 --- a/core/Archive/Array/IndexedBySite.php +++ /dev/null @@ -1,266 +0,0 @@ -<?php -/** - * Piwik - Open source web analytics - * - * @link http://piwik.org - * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later - * - * @category Piwik - * @package Piwik - */ - -/** - * @package Piwik - * @subpackage Piwik_Archive - */ -class Piwik_Archive_Array_IndexedBySite extends Piwik_Archive_Array -{ - /** - * Used to cache the name of the table that holds the data this archive. - * - * This will only be used if the archives held by this instance are instances of - * Piwik_Archive_Single. - */ - private $tableName = null; - - /** - * @param array $sites array of siteIds - * @param string $strPeriod eg. 'day' 'week' etc. - * @param string $strDate A date range, eg. 'last10', 'previous5' or 'YYYY-MM-DD,YYYY-MM-DD' - * @param Piwik_Segment $segment - * @param string $_restrictSitesToLogin - */ - function __construct($sites, $strPeriod, $strDate, Piwik_Segment $segment, $_restrictSitesToLogin) - { - foreach ($sites as $idSite) { - $archive = Piwik_Archive::build($idSite, $strPeriod, $strDate, $segment, $_restrictSitesToLogin); - $this->archives[$idSite] = $archive; - } - ksort($this->archives); - } - - /** - * @return string - */ - protected function getIndexName() - { - return 'idSite'; - } - - /** - * @param Piwik_Archive $archive - * @return mixed - */ - protected function getDataTableLabelValue($archive) - { - return $archive->getIdSite(); - } - - /** - * Given a list of fields defining numeric values, it will return a Piwik_DataTable_Array - * ordered by idsite - * - * @param array|string $fields array( fieldName1, fieldName2, ...) Names of the mysql table fields to load - * @return Piwik_DataTable_Array - */ - public function getDataTableFromNumeric($fields) - { - $tableArray = $this->getNewDataTableArray(); - if ($this->getFirstArchive() instanceof Piwik_Archive_Single) { - $values = $this->getValues($fields); - foreach ($this->archives as $idSite => $archive) { - $table = $archive->makeDataTable($isSimple = true); - if (array_key_exists($idSite, $values)) { - $table->addRowsFromArray($values[$idSite]); - } - $tableArray->addTable($table, $idSite); - } - } elseif ($this->getFirstArchive() instanceof Piwik_Archive_Array) { - foreach ($this->archives as $idSite => $archive) { - $tableArray->addTable($archive->getDataTableFromNumeric($fields), $idSite); - } - } - - return $tableArray; - } - - /** - * Returns the values of the requested fields - * - * @param array $fields - * @return array - */ - private function getValues($fields) - { - // Creating the default array, to ensure consistent order - $defaultValues = array(); - foreach ($fields as $field) { - $defaultValues[$field] = null; - } - - $arrayValues = array(); - foreach ($this->loadValuesFromDB($fields) as $value) { - if (!isset($arrayValues[$value['idsite']])) { - $arrayValues[$value['idsite']] = $defaultValues; - } - $arrayValues[$value['idsite']][$value['name']] = $this->formatNumericValue($value['value']); - } - return $arrayValues; - } - - /** - * @param $fields - * @return array|array (one row in the array per row fetched in the DB) - */ - private function loadValuesFromDB($fields) - { - $requestedMetrics = is_string($fields) ? array($fields) : $fields; - $inNames = Piwik_Common::getSqlStringFieldsArray($fields); - - // get the archive ids - if (!$this->getFirstArchive()->isArchivingDisabled()) { - $archiveIds = $this->getArchiveIdsAfterLaunching($requestedMetrics); - } else { - $archiveIds = $this->getArchiveIdsWithoutLaunching($requestedMetrics); - } - - $archiveIds = implode(', ', array_filter($archiveIds)); - - // if no archive ids are found, avoid executing any SQL queries - if (empty($archiveIds)) { - return array(); - } - - // select archive data - $sql = "SELECT value, name, idarchive, idsite - FROM {$this->getNumericTableName()} - WHERE idarchive IN ( $archiveIds ) - AND name IN ( $inNames )"; - - return Piwik_FetchAll($sql, $fields); - } - - /** - * Returns the first archive in the list - * - * @return Piwik_Archive - */ - private function getFirstArchive() - { - return reset($this->archives); - } - - /** - * Gets the archive id of every Single archive this archive holds. This method - * will launch the archiving process if appropriate. - * - * @param array $metrics The requested archive metrics. - * @throws Exception - * @return array - */ - private function getArchiveIdsAfterLaunching($metrics) - { - // collect the individual report names for the requested metrics - $reports = array(); - foreach ($metrics as $metric) { - $report = Piwik_Archive_Single::getRequestedReportFor($metric); - $reports[$report] = $metric; - } - - // process archives for each individual report - $archiveIds = array(); - foreach ($reports as $report => $metric) { - // prepare archives (this will launch archiving when appropriate) - foreach ($this->archives as $archive) { - // NOTE: Piwik_Archive_Single expects a metric here, not a report - $archive->setRequestedReport($metric); - $archive->prepareArchive(); - } - - // collect archive ids for archives that have visits - foreach ($this->archives as $archive) { - if (!$archive->isThereSomeVisits) { - continue; - } - - $archiveIds[] = $archive->getIdArchive(); - - if ($this->getNumericTableName() != $archive->archiveProcessing->getTableArchiveNumericName()) { - throw new Exception("Piwik_Archive_Array_IndexedBySite::getDataTableFromNumeric() algorithm won't work if data is stored in different tables"); - } - } - } - - return $archiveIds; - } - - /** - * Gets the archive id of every Single archive this archive holds. This method - * will not launch the archiving process. - * - * @param array $metrics The requested archive metrics. - * @return array - */ - private function getArchiveIdsWithoutLaunching($metrics) - { - $firstArchive = $this->getFirstArchive(); - $segment = $firstArchive->getSegment(); - $period = $firstArchive->getPeriod(); - - // the flags used to tell how the archiving process for a specific archive was completed, - // if it was completed - $doneFlags = array(); - foreach ($metrics as $metric) { - $done = Piwik_ArchiveProcessing::getDoneStringFlagFor($segment, $period, $metric); - $donePlugins = Piwik_ArchiveProcessing::getDoneStringFlagFor($segment, $period, $metric, true); - - $doneFlags[$done] = $done; - $doneFlags[$donePlugins] = $donePlugins; - } - - $allDoneFlags = "'" . implode("','", $doneFlags) . "'"; - - // create the SQL to query every archive ID - $nameCondition = "(name IN ($allDoneFlags)) AND - (value = '" . Piwik_ArchiveProcessing::DONE_OK . "' OR - value = '" . Piwik_ArchiveProcessing::DONE_OK_TEMPORARY . "')"; - - $sql = "SELECT idsite, - MAX(idarchive) AS idarchive - FROM " . $this->getNumericTableName() . " - WHERE date1 = ? - AND date2 = ? - AND period = ? - AND $nameCondition - AND idsite IN (" . implode(',', array_keys($this->archives)) . ") - GROUP BY idsite"; - - $bind = array($period->getDateStart()->toString('Y-m-d'), - $period->getDateEnd()->toString('Y-m-d'), - $period->getId()); - - // execute the query and process the results. - $archiveIds = array(); - foreach (Piwik_FetchAll($sql, $bind) as $row) { - $archiveIds[] = $row['idarchive']; - } - - return $archiveIds; - } - - /** - * Gets the name of the database table that holds the numeric archive data for - * this archive. - * - * @return string - */ - private function getNumericTableName() - { - if (is_null($this->tableName)) { - $table = Piwik_ArchiveProcessing::makeNumericArchiveTable($this->getFirstArchive()->getPeriod()); - $this->tableName = $table->getTableName(); - } - - return $this->tableName; - } -} diff --git a/core/Archive/DataCollection.php b/core/Archive/DataCollection.php new file mode 100644 index 0000000000..4313fadd1c --- /dev/null +++ b/core/Archive/DataCollection.php @@ -0,0 +1,326 @@ +<?php +/** + * Piwik - Open source web analytics + * + * @link http://piwik.org + * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later + * + * @category Piwik + * @package Piwik + */ + +/** + * This class is used to hold and transform archive data for the Piwik_Archive class. + * + * Archive data is loaded into an instance of this type, can be indexed by archive + * metadata (such as the site ID, period string, etc.), and can be transformed into + * Piwik_DataTable and Piwik_DataTable_Array instances. + */ +class Piwik_Archive_DataCollection +{ + /** + * The archive data, indexed first by site ID and then by period date range. Eg, + * + * array( + * '0' => array( + * array( + * '2012-01-01,2012-01-01' => array(...), + * '2012-01-02,2012-01-02' => array(...), + * ) + * ), + * '1' => array( + * array( + * '2012-01-01,2012-01-01' => array(...), + * ) + * ) + * ) + * + * Archive data can be either a numeric value or a serialized string blob. Every + * piece of archive data is associated by it's archive name. For example, + * the array(...) above could look like: + * + * array( + * 'nb_visits' => 1, + * 'nb_actions' => 2 + * ) + * + * There is a special element '_metadata' in data rows that holds values treated + * as DataTable metadata. + */ + private $data = array(); + + /** + * The whole list of metric/record names that were used in the archive query. + * + * @var array + */ + private $dataNames; + + /** + * The type of data that was queried for (ie, "blob" or "numeric"). + * + * @var string + */ + private $dataType; + + /** + * The default values to use for each metric/record name that's being queried + * for. + * + * @var array + */ + private $defaultRow; + + /** + * The list of all site IDs that were queried for. + * + * @var array + */ + private $sitesId; + + /** + * The list of all periods that were queried for. Each period is associated with + * the period's range string. Eg, + * + * array( + * '2012-01-01,2012-01-31' => new Piwik_Period(...), + * '2012-02-01,2012-02-28' => new Piwik_Period(...), + * ) + * + * @var array + */ + private $periods; + + /** + * Constructor. + * + * @param array $dataNames @see $this->dataNames + * @param string $dataType @see $this->dataType + * @param array $sitesId @see $this->sitesId + * @param array $periods @see $this->periods + * @param array $defaultRow @see $this->defaultRow + */ + public function __construct($dataNames, $dataType, $sitesId, $periods, $defaultRow = null) + { + $this->dataNames = $dataNames; + $this->dataType = $dataType; + + if ($defaultRow === null) { + $defaultRow = array_fill_keys($dataNames, 0); + } + $this->sitesId = $sitesId; + $this->periods = $periods; + $this->defaultRow = $defaultRow; + } + + /** + * Returns a reference to the data for a specific site & period. If there is + * no data for the given site ID & period, it is set to the default row. + * + * @param int $idSite + * @param string $period eg, '2012-01-01,2012-01-31' + */ + public function &get($idSite, $period) + { + if (!isset($this->data[$idSite][$period])) { + $this->data[$idSite][$period] = $this->defaultRow; + } + return $this->data[$idSite][$period]; + } + + /** + * Adds a new metadata to the data for specific site & period. If there is no + * data for the given site ID & period, it is set to the default row. + * + * Note: Site ID and period range string are two special types of metadata. Since + * the data stored in this class is indexed by site & period, this metadata is not + * stored in individual data rows. + * + * @param int $idSite + * @param string $period eg, '2012-01-01,2012-01-31' + * @param string $name The metadata name. + * @param mixed $value The metadata name. + */ + public function addMetadata($idSite, $period, $name, $value) + { + $row = &$this->get($idSite, $period); + $row['_metadata'][$name] = $value; + } + + /** + * Returns archive data as an array indexed by metadata. + * + * @param array $resultIndices An array mapping metadata names to pretty labels + * for them. Each archive data row will be indexed + * by the metadata specified here. + * + * Eg, array('site' => 'idSite', 'period' => 'Date') + * @return array + */ + public function getArray($resultIndices) + { + $indexKeys = array_keys($resultIndices); + + $result = $this->createEmptyIndex($indexKeys); + foreach ($this->data as $idSite => $rowsByPeriod) { + foreach ($rowsByPeriod as $period => $row) { + $indexRowKeys = $this->getRowKeys($indexKeys, $row, $idSite, $period); + + $this->setIndexRow($result, $indexRowKeys, $row); + } + } + return $result; + } + + /** + * Returns archive data as a DataTable indexed by metadata. Indexed data will + * be represented by Piwik_DataTable_Array instances. + * + * @param array $resultIndices An array mapping metadata names to pretty labels + * for them. Each archive data row will be indexed + * by the metadata specified here. + * + * Eg, array('site' => 'idSite', 'period' => 'Date') + * @return Piwik_DataTable|Piwik_DataTable_Array + */ + public function getDataTable($resultIndices) + { + $dataTableFactory = new Piwik_Archive_DataTableFactory( + $this->dataNames, $this->dataType, $this->sitesId, $this->periods, $this->defaultRow); + + $index = $this->getArray($resultIndices); + return $dataTableFactory->make($index, $resultIndices); + } + + /** + * Returns archive data as a DataTable indexed by metadata. Indexed data will + * be represented by Piwik_DataTable_Array instances. Each DataTable will have + * its subtable IDs set. + * + * This function will only work if blob data was loaded and only one record + * was loaded (not including subtables of the record). + * + * @param array $resultIndices An array mapping metadata names to pretty labels + * for them. Each archive data row will be indexed + * by the metadata specified here. + * + * Eg, array('site' => 'idSite', 'period' => 'Date') + * @param int $idSubtable The subtable to return. + * @param bool $addMetadataSubtableId Whether to add the DB subtable ID as metadata + * to each datatable, or not. + */ + public function getExpandedDataTable($resultIndices, $idSubtable = null, $addMetadataSubtableId = false) + { + if ($this->dataType != 'blob') { + throw new Exception("Piwik_Archive_DataCollection: cannot call getExpandedDataTable with " + . "{$this->dataType} data types. Only works with blob data."); + } + + if (count($this->dataNames) !== 1) { + throw new Exception("Piwik_Archive_DataCollection: cannot call getExpandedDataTable with " + . "more than one record."); + } + + $dataTableFactory = new Piwik_Archive_DataTableFactory( + $this->dataNames, 'blob', $this->sitesId, $this->periods, $this->defaultRow); + $dataTableFactory->expandDataTable($addMetadataSubtableId); + $dataTableFactory->useSubtable($idSubtable); + + $index = $this->getArray($resultIndices); + return $dataTableFactory->make($index, $resultIndices); + } + + /** + * Returns metadata for a data row. + * + * @param array $data The data row. + */ + public static function getDataRowMetadata($data) + { + if (isset($data['_metadata'])) { + return $data['_metadata']; + } else { + return array(); + } + } + + /** + * Removes all table metadata from a data row. + * + * @param array $data The data row. + */ + public static function removeMetadataFromDataRow(&$data) + { + unset($data['_metadata']); + } + + /** + * Creates an empty index using a list of metadata names. If the 'site' and/or + * 'period' metadata names are supplied, empty rows are added for every site/period + * that was queried for. + * + * @param array $indexKeys List of metadata names to index archive data by. + * @return array + */ + private function createEmptyIndex($indexKeys) + { + $result = array(); + + if (!empty($indexKeys)) { + $index = array_shift($indexKeys); + if ($index == 'site') { + foreach ($this->sitesId as $idSite) { + $result[$idSite] = $this->createEmptyIndex($indexKeys); + } + } else if ($index == 'period') { + foreach ($this->periods as $period => $periodObject) { + $result[$period] = $this->createEmptyIndex($indexKeys); + } + } + } + + return $result; + } + + /** + * Sets a row in an index by the index keys of the row. + */ + private function setIndexRow(&$result, $keys, $row) + { + $keyCount = count($keys); + + if ($keyCount > 1) { + $firstKey = array_shift($keys); + $this->setIndexRow($result[$firstKey], $keys, $row); + } else if ($keyCount == 1) { + $result[reset($keys)] = $row; + } else { + $result = $row; + } + } + + /** + * Returns the index keys for a row based on a set of metadata names. + * + * @param array $metadataNames + * @param array $row + * @param int $idSite The site ID for the row (needed since site ID is not + * stored as metadata). + * @param string $period eg, '2012-01-01,2012-01-31'. The period for the + * row (needed since period is not stored as metadata). + */ + private function getRowKeys($metadataNames, $row, $idSite, $period) + { + $result = array(); + foreach ($metadataNames as $name) { + if ($name == 'site') { + $result['site'] = $idSite; + } else if ($name == 'period') { + $result['period'] = $period; + } else if (isset($row['_metadata'][$name])) { + $result[$name] = $row['_metadata'][$name]; + } + } + return $result; + } +} diff --git a/core/Archive/DataTableFactory.php b/core/Archive/DataTableFactory.php new file mode 100644 index 0000000000..6333545380 --- /dev/null +++ b/core/Archive/DataTableFactory.php @@ -0,0 +1,363 @@ +<?php +/** + * Piwik - Open source web analytics + * + * @link http://piwik.org + * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later + * + * @category Piwik + * @package Piwik + */ + +/** + * Creates a Piwik_DataTable or Piwik_DataTable_Array instance based on an array + * index created by Piwik_Archive_DataCollection. + * + * This class is only used by Piwik_Archive_DataCollection. + */ +class Piwik_Archive_DataTableFactory +{ + /** + * @see Piwik_Archive_DataCollection::$dataNames. + */ + private $dataNames; + + /** + * @see Piwik_Archive_DataCollection::$dataType. + */ + private $dataType; + + /** + * Whether to expand the DataTables that're created or not. Expanding a DataTable + * means creating DataTables using subtable blobs and correctly setting the subtable + * IDs of all DataTables. + * + * @var bool + */ + private $expandDataTable = false; + + /** + * Whether to add the subtable ID used in the database to the in-memory DataTables + * as metadata or not. + * + * @var bool + */ + private $addMetadataSubtableId = false; + + /** + * @see Piwik_Archive_DataCollection::$sitesId. + */ + private $sitesId; + + /** + * @see Piwik_Archive_DataCollection::$periods. + */ + private $periods; + + /** + * The ID of the subtable to create a DataTable for. Only relevant for blob data. + * + * @var int|null + */ + private $idSubtable = null; + + /** + * @see Piwik_Archive_DataCollection::$defaultRow. + */ + private $defaultRow; + + /** + * Constructor. + */ + public function __construct($dataNames, $dataType, $sitesId, $periods, $defaultRow) + { + $this->dataNames = $dataNames; + $this->dataType = $dataType; + $this->sitesId = $sitesId; + $this->periods = $periods; + $this->defaultRow = $defaultRow; + } + + /** + * Tells the factory instance to expand the DataTables that are created by + * creating subtables and setting the subtable IDs of rows w/ subtables correctly. + * + * @param bool $addMetadataSubtableId Whether to add the subtable ID used in the + * database to the in-memory DataTables as + * metadata or not. + */ + public function expandDataTable($addMetadataSubtableId = false) + { + $this->expandDataTable = true; + $this->addMetadataSubtableId = $addMetadataSubtableId; + } + + /** + * Tells the factory instance to create a DataTable using a blob with the + * supplied subtable ID. + * + * @param int $idSubtable An in-database subtable ID. + */ + public function useSubtable($idSubtable) + { + if (count($this->dataNames) !== 1) { + throw new Exception("Piwik_Archive_DataTableFactory: Getting subtables for multiple records in one" + . " archive query is not currently supported."); + } + + $this->idSubtable = $idSubtable; + } + + /** + * Creates a Piwik_DataTable|Piwik_DataTable_Array instance using an index of + * archive data. + * + * @param array $index @see Piwik_Archive_DataCollection + * @param array $resultIndices an array mapping metadata names with pretty metadata + * labels. + * @return Piwik_DataTable|Piwik_DataTable_Array + */ + public function make($index, $resultIndices) + { + if (empty($resultIndices)) { + // for numeric data, if there's no index (and thus only 1 site & period in the query), + // we want to display every queried metric name + if (empty($index) + && $this->dataType == 'numeric' + ) { + $index = $this->defaultRow; + } + + $dataTable = $this->createDataTable($index, $keyMetadata = array()); + } else { + $dataTable = $this->createDataTableArrayFromIndex($index, $resultIndices); + } + + $this->transformMetadata($dataTable); + return $dataTable; + } + + /** + * Creates a Piwik_DataTable|Piwik_DataTable_Array instance using an array + * of blobs. + * + * If only one record is being queried, a single DataTable will + * be returned. Otherwise, a DataTable_Array is returned that indexes + * DataTables by record name. + * + * If expandDataTable was called, and only one record is being queried, + * the created DataTable's subtables will be expanded. + * + * @param array $blobRow + * @return Piwik_DataTable|Piwik_DataTable_Array + */ + private function makeFromBlobRow($blobRow) + { + if ($blobRow === false) { + return new Piwik_DataTable(); + } + + if (count($this->dataNames) === 1) { + return $this->makeDataTableFromSingleBlob($blobRow); + } else { + return $this->makeIndexedByRecordNameDataTable($blobRow); + } + } + + /** + * Creates a DataTable for one record from an archive data row. + * + * @see makeFromBlobRow + * + * @param array $blobRow + * @return Piwik_DataTable + */ + private function makeDataTableFromSingleBlob($blobRow) + { + $recordName = reset($this->dataNames); + if ($this->idSubtable !== null) { + $recordName .= '_' . $this->idSubtable; + } + + if (!empty($blobRow[$recordName])) { + $table = Piwik_DataTable::fromSerializedArray($blobRow[$recordName]); + } else { + $table = new Piwik_DataTable(); + } + + // set table metadata + $table->metadata = Piwik_Archive_DataCollection::getDataRowMetadata($blobRow); + + if ($this->expandDataTable) { + $table->enableRecursiveFilters(); + $this->setSubtables($table, $blobRow); + } + + return $table; + } + + /** + * Creates a DataTable for every record in an archive data row and puts them + * in a DataTable_Array instance. + * + * @param array $blobRow + * @return Piwik_DataTable_Array + */ + private function makeIndexedByRecordNameDataTable($blobRow) + { + $table = new Piwik_DataTable_Array(); + $table->setKeyName('recordName'); + + $tableMetadata = Piwik_Archive_DataCollection::getDataRowMetadata($blobRow); + + foreach ($blobRow as $name => $blob) { + $newTable = Piwik_DataTable::fromSerializedArray($blob); + $newTable->metadata = $tableMetadata; + + $table->addTable($newTable, $name); + } + + return $table; + } + + /** + * Creates a Piwik_DataTable_Array from an array index. + * + * @param array $index @see Piwik_Archive_DataCollection + * @param array $resultIndices @see make + * @param array $keyMetadata The metadata to add to the table when it's created. + */ + private function createDataTableArrayFromIndex($index, $resultIndices, $keyMetadata = array()) + { + $resultIndexLabel = reset($resultIndices); + $resultIndex = key($resultIndices); + + array_shift($resultIndices); + + $result = new Piwik_DataTable_Array(); + $result->setKeyName($resultIndexLabel); + + foreach ($index as $label => $value) { + $keyMetadata[$resultIndex] = $label; + + if (empty($resultIndices)) { + $newTable = $this->createDataTable($value, $keyMetadata); + } else { + $newTable = $this->createDataTableArrayFromIndex($value, $resultIndices, $keyMetadata); + } + + $result->addTable($newTable, $this->prettifyIndexLabel($resultIndex, $label)); + } + + return $result; + } + + /** + * Creates a Piwik_DataTable instance from an index row. + * + * @param array|false $data An archive data row. + * @param array $keyMetadata The metadata to add to the table(s) when created. + * @return Piwik_DataTable|Piwik_DataTable_Array + */ + private function createDataTable($data, $keyMetadata) + { + if ($this->dataType == 'blob') { + $result = $this->makeFromBlobRow($data); + } else { + $table = new Piwik_DataTable_Simple(); + + if (!empty($data)) { + $table->metadata = Piwik_Archive_DataCollection::getDataRowMetadata($data); + + Piwik_Archive_DataCollection::removeMetadataFromDataRow($data); + + $table->addRow(new Piwik_DataTable_Row(array(Piwik_DataTable_Row::COLUMNS => $data))); + } + + $result = $table; + } + + if (!isset($keyMetadata['site'])) { + $keyMetadata['site'] = reset($this->sitesId); + } + + if (!isset($keyMetadata['period'])) { + reset($this->periods); + $keyMetadata['period'] = key($this->periods); + } + + // Note: $result can be a DataTable_Array + $result->filter(function ($table) use ($keyMetadata) { + foreach ($keyMetadata as $name => $value) { + $table->setMetadata($name, $value); + } + }); + + return $result; + } + + /** + * Creates DataTables from $dataTable's subtable blobs (stored in $blobRow) and sets + * the subtable IDs of each DataTable row. + * + * @param Piwik_DataTable $dataTable + * @param array $blobRow An array associating record names (w/ subtable if applicable) + * with blob values. This should hold every subtable blob for + * the loaded DataTable. + */ + private function setSubtables($dataTable, $blobRow) + { + $dataName = reset($this->dataNames); + + foreach ($dataTable->getRows() as $row) { + $sid = $row->getIdSubDataTable(); + if ($sid === null) { + continue; + } + + $blobName = $dataName."_".$sid; + if (isset($blobRow[$blobName])) { + $subtable = Piwik_DataTable::fromSerializedArray($blobRow[$blobName]); + $this->setSubtables($subtable, $blobRow); + + // we edit the subtable ID so that it matches the newly table created in memory + // NB: we dont overwrite the datatableid in the case we are displaying the table expanded. + if ($this->addMetadataSubtableId) { + // this will be written back to the column 'idsubdatatable' just before rendering, + // see Renderer/Php.php + $row->addMetadata('idsubdatatable_in_db', $row->getIdSubDataTable()); + } + + $row->setSubtable($subtable); + } + } + } + + /** + * Converts site IDs and period string ranges into Piwik_Site instances and + * Piwik_Period instances in DataTable metadata. + */ + private function transformMetadata($table) + { + $periods = $this->periods; + $table->filter(function ($table) use($periods) { + $table->metadata['site'] = new Piwik_Site($table->metadata['site']); + $table->metadata['period'] = $periods[$table->metadata['period']]; + }); + } + + /** + * Returns the pretty version of an index label. + * + * @param string $labelType eg, 'site', 'period', etc. + * @param string $label eg, '0', '1', '2012-01-01,2012-01-31', etc. + * @return string + */ + private function prettifyIndexLabel($labelType, $label) + { + if ($labelType == 'period') { // prettify period labels + return $this->periods[$label]->getPrettyString(); + } + return $label; + } +} diff --git a/core/Archive/Single.php b/core/Archive/Single.php deleted file mode 100644 index 3e805d4a98..0000000000 --- a/core/Archive/Single.php +++ /dev/null @@ -1,632 +0,0 @@ -<?php -/** - * Piwik - Open source web analytics - * - * @link http://piwik.org - * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later - * - * - * @category Piwik - * @package Piwik - */ - -/** - * Piwik_Archive_Single is used to store the data of a single archive, - * for example the statistics for the 'day' '2008-02-21' for the website idSite '2' - * - * @package Piwik - * @subpackage Piwik_Archive - */ -class Piwik_Archive_Single extends Piwik_Archive -{ - /** - * The Piwik_ArchiveProcessing object used to check that the archive is available - * and launch the processing if the archive was not yet processed - * - * @var Piwik_ArchiveProcessing - */ - public $archiveProcessing = null; - - /** - * @var bool Set to true if the archive has at least 1 visit - */ - public $isThereSomeVisits = null; - - /** - * Period of this Archive - * - * @var Piwik_Period - */ - protected $period = null; - - /** - * Set to true will activate numeric value caching for this archive. - * - * @var bool - */ - protected $cacheEnabledForNumeric = true; - - /** - * Array of cached numeric values, used to make requests faster - * when requesting the same value again and again - * - * @var array of numeric - */ - protected $numericCached = array(); - - /** - * Array of cached blob, used to make requests faster when requesting the same blob again and again - * - * @var array of mixed - */ - protected $blobCached = array(); - - /** - * idarchive of this Archive in the database - * - * @var int - */ - protected $idArchive = null; - - /** - * name of requested report - * - * @var string - */ - protected $requestedReport = null; - - /** - * Flag set to true once the archive has been checked (when we make sure it is archived) - * - * @var bool - */ - protected $alreadyChecked = array(); - - protected function clearCache() - { - foreach ($this->blobCached as $name => $blob) { - $this->freeBlob($name); - } - $this->blobCached = array(); - } - - public function __destruct() - { - $this->clearCache(); - } - - /** - * Returns the blob cache. For testing. - * - * @return array - */ - public function getBlobCache() - { - return $this->blobCached; - } - - /** - * Returns the pretty date of this Archive, eg. 'Thursday 20th March 2008' - * - * @return string - */ - public function getPrettyDate() - { - return $this->period->getPrettyString(); - } - - /** - * Returns the idarchive of this Archive used to index this archive in the DB - * - * @throws Exception - * @return int - */ - public function getIdArchive() - { - if (is_null($this->idArchive)) { - throw new Exception("idArchive is null"); - } - return $this->idArchive; - } - - /** - * Set the period - * - * @param Piwik_Period $period - */ - public function setPeriod(Piwik_Period $period) - { - $this->period = $period; - } - - public function getPeriod() - { - return $this->period; - } - - /** - * Returns the timestamp of the first date in the period for this Archive. - * This is used to sort archives by date when working on a Archive_Array - * - * @return int Unix timestamp - */ - public function getTimestampStartDate() - { - if (!is_null($this->archiveProcessing)) { - $timestamp = $this->archiveProcessing->getTimestampStartDate(); - if (!empty($timestamp)) { - return $timestamp; - } - } - return $this->period->getDateStart()->getTimestamp(); - } - - /** - * Prepares the archive. Gets the idarchive from the ArchiveProcessing. - * - * This will possibly launch the archiving process if the archive was not available. - * @return bool - */ - public function prepareArchive() - { - $archiveJustProcessed = false; - - - $periodString = $this->period->getLabel(); - $plugin = Piwik_ArchiveProcessing::getPluginBeingProcessed($this->getRequestedReport()); - - $cacheKey = 'all'; - if ($periodString == 'range') { - $cacheKey = $plugin; - } - if (!isset($this->alreadyChecked[$cacheKey])) { - $this->isThereSomeVisits = false; - $this->alreadyChecked[$cacheKey] = true; - $dayString = $this->period->getPrettyString(); - $logMessage = sprintf("%s (%s), plugin %s", $periodString, $dayString, $plugin); - // if the END of the period is BEFORE the website creation date - // we already know there are no stats for this period - // we add one day to make sure we don't miss the day of the website creation - if ($this->period->getDateEnd()->addDay(2)->isEarlier($this->site->getCreationDate())) { - Piwik::log(sprintf("Archive %s skipped, archive is before the website was created.", $logMessage)); - return; - } - - // if the starting date is in the future we know there is no visit - if ($this->period->getDateStart()->subDay(2)->isLater(Piwik_Date::today())) { - Piwik::log(sprintf("Archive %s skipped, archive is after today.", $logMessage)); - return; - } - - // we make sure the archive is available for the given date - $periodLabel = $this->period->getLabel(); - $this->archiveProcessing = Piwik_ArchiveProcessing::factory($periodLabel); - $this->archiveProcessing->setSite($this->site); - $this->archiveProcessing->setPeriod($this->period); - $this->archiveProcessing->setSegment($this->segment); - - $this->archiveProcessing->init(); - - $this->archiveProcessing->setRequestedReport($this->getRequestedReport()); - - $archivingDisabledArchiveNotProcessed = false; - $idArchive = $this->archiveProcessing->loadArchive(); - if (empty($idArchive)) { - if ($this->archiveProcessing->isArchivingDisabled()) { - $archivingDisabledArchiveNotProcessed = true; - $logMessage = sprintf("Archiving disabled, for %s", $logMessage); - } else { - Piwik::log(sprintf("Processing %s, not archived yet...", $logMessage)); - $archiveJustProcessed = true; - - // Process the reports - $this->archiveProcessing->launchArchiving(); - - $idArchive = $this->archiveProcessing->getIdArchive(); - $logMessage = sprintf("Processed %d, for %s", $idArchive, $logMessage); - } - } else { - $logMessage = sprintf("Already processed, fetching idArchive = %d (idSite=%d), for %s", $idArchive, $this->site->getId(), $logMessage); - } - Piwik::log(sprintf("%s, Visits = %d", $logMessage, $this->archiveProcessing->getNumberOfVisits())); - $this->isThereSomeVisits = !$archivingDisabledArchiveNotProcessed - && $this->archiveProcessing->isThereSomeVisits(); - $this->idArchive = $idArchive; - } - return $archiveJustProcessed; - } - - /** - * Returns a value from the current archive with the name = $name - * Method used by getNumeric or getBlob - * - * @param string $name - * @param string $typeValue numeric|blob - * @param string|bool $archivedDate Value to store date of archive info in. If false, not stored. - * @return mixed|bool false if no result - */ - protected function get($name, $typeValue = 'numeric', &$archivedDate = false) - { - $this->setRequestedReport($name); - $this->prepareArchive(); - - // values previously "get" and now cached - if ($typeValue == 'numeric' - && $this->cacheEnabledForNumeric - && isset($this->numericCached[$name]) - ) { - return $this->numericCached[$name]; - } - - // During archiving we prefetch the blobs recursively - // and we get them faster from memory after - if ($typeValue == 'blob' - && isset($this->blobCached[$name]) - ) { - return $this->blobCached[$name]; - } - - if ($name == 'idarchive') { - return $this->idArchive; - } - - if (!$this->isThereSomeVisits) { - return false; - } - - // select the table to use depending on the type of the data requested - switch ($typeValue) { - case 'blob': - $table = $this->archiveProcessing->getTableArchiveBlobName(); - break; - - case 'numeric': - default: - $table = $this->archiveProcessing->getTableArchiveNumericName(); - break; - } - - $db = Zend_Registry::get('db'); - $row = $db->fetchRow("SELECT value, ts_archived - FROM $table - WHERE idarchive = ? AND name = ?", - array($this->idArchive, $name) - ); - - $value = $tsArchived = false; - if (is_array($row)) { - $value = $row['value']; - $tsArchived = $row['ts_archived']; - } - - if ($archivedDate !== false) { - $archivedDate = $tsArchived; - } - - if ($value === false) { - if ($typeValue == 'numeric' - && $this->cacheEnabledForNumeric - ) { - $this->numericCached[$name] = false; - } - return $value; - } - - // uncompress when selecting from the BLOB table - if ($typeValue == 'blob' && $db->hasBlobDataType()) { - $value = $this->uncompress($value); - } - - if ($typeValue == 'numeric' - && $this->cacheEnabledForNumeric - ) { - $this->numericCached[$name] = $value; - } - return $value; - } - - - /** - * This method loads in memory all the subtables for the main table called $name. - * You have to give it the parent table $dataTableToLoad so we can lookup the sub tables ids to load. - * - * If $addMetadataSubtableId set to true, it will add for each row a 'metadata' called 'databaseSubtableId' - * containing the child ID of the subtable associated to this row. - * - * @param string $name - * @param Piwik_DataTable $dataTableToLoad - * @param bool $addMetadataSubtableId - */ - public function loadSubDataTables($name, Piwik_DataTable $dataTableToLoad, $addMetadataSubtableId = false) - { - // we have to recursively load all the subtables associated to this table's rows - // and update the subtableID so that it matches the newly instanciated table - foreach ($dataTableToLoad->getRows() as $row) { - $subTableID = $row->getIdSubDataTable(); - - if ($subTableID !== null) { - $subDataTableLoaded = $this->getDataTable($name, $subTableID); - - $row->setSubtable($subDataTableLoaded); - - $this->loadSubDataTables($name, $subDataTableLoaded, $addMetadataSubtableId); - - // we edit the subtable ID so that it matches the newly table created in memory - // NB: we dont overwrite the datatableid in the case we are displaying the table expanded. - if ($addMetadataSubtableId) { - // this will be written back to the column 'idsubdatatable' just before rendering, see Renderer/Php.php - $row->addMetadata('idsubdatatable_in_db', $row->getIdSubDataTable()); - } - } - } - } - - - /** - * Free the blob cache memory array - * @param $name - */ - public function freeBlob($name) - { - unset($this->blobCached[$name]); - $this->blobCached[$name] = null; - } - - protected function uncompress($data) - { - return @gzuncompress($data); - } - - /** - * Fetches all blob fields name_* at once for the current archive for performance reasons. - * - * @param string $name - * - * @return void - */ - public function preFetchBlob($name) - { - $this->setRequestedReport($name); - $this->prepareArchive(); - if (!$this->isThereSomeVisits) { - return; - } - - $tableBlob = $this->archiveProcessing->getTableArchiveBlobName(); - - $db = Zend_Registry::get('db'); - $hasBlobs = $db->hasBlobDataType(); - - // select blobs w/ name like "$name_[0-9]+" w/o using RLIKE - $nameEnd = strlen($name) + 2; - $query = $db->query("SELECT value, name - FROM $tableBlob - WHERE idarchive = ? - AND (name = ? OR - (name LIKE ? AND SUBSTRING(name, $nameEnd, 1) >= '0' - AND SUBSTRING(name, $nameEnd, 1) <= '9') )", - array($this->idArchive, $name, $name . '%') - ); - - while ($row = $query->fetch()) { - $value = $row['value']; - $name = $row['name']; - - if ($hasBlobs) { - $this->blobCached[$name] = $this->uncompress($value); - if ($this->blobCached[$name] === false) { - //throw new Exception("Error gzuncompress $name "); - } - } else { - $this->blobCached[$name] = $value; - } - } - } - - /** - * Returns a numeric value from this Archive, with the name '$name' - * - * @param string $name - * @return int|float - */ - public function getNumeric($name) - { - return $this->formatNumericValue($this->get($name, 'numeric')); - } - - - /** - * Returns a blob value from this Archive, with the name '$name' - * Blob values are all values except int and float. - * - * @param string $name - * @return mixed - */ - public function getBlob($name) - { - return $this->get($name, 'blob'); - } - - /** - * Given a list of fields defining numeric values, it will return a Piwik_DataTable_Simple - * containing one row per field name. - * - * For example $fields = array( 'max_actions', - * 'nb_uniq_visitors', - * 'nb_visits', - * 'nb_actions', - * 'sum_visit_length', - * 'bounce_count', - * 'nb_visits_converted' - * ); - * - * @param string|array $fields Name or array of names of Archive fields - * - * @return Piwik_DataTable_Simple - */ - public function getDataTableFromNumeric($fields) - { - if (!is_array($fields)) { - $fields = array($fields); - } - - $values = array(); - foreach ($fields as $field) { - $values[$field] = $this->getNumeric($field); - } - - $table = new Piwik_DataTable_Simple(); - $table->addRowsFromArray($values); - return $table; - } - - /** - * Returns a DataTable that has the name '$name' from the current Archive. - * If $idSubTable is specified, returns the subDataTable called '$name_$idSubTable' - * - * @param string $name - * @param int $idSubTable optional id SubDataTable - * @return Piwik_DataTable - */ - public function getDataTable($name, $idSubTable = null) - { - if (!is_null($idSubTable)) { - $name .= sprintf("_%s", $idSubTable); - } - - $this->setRequestedReport($name); - - $data = $this->get($name, 'blob', $tsArchived); - - $table = $this->makeDataTable(); - - if ($data !== false) { - $table->addRowsFromSerializedArray($data); - $table->setMetadata(Piwik_DataTable::ARCHIVED_DATE_METADATA_NAME, $tsArchived); - } - if ($data === false - && $idSubTable !== null - ) { - // This is not expected, but somehow happens in some unknown cases and very rarely. - // Do not throw error in this case - //throw new Exception("not expected"); - return new Piwik_DataTable(); - } - - return $table; - } - - /** - * Creates a new DataTable with some metadata set. Sets the following metadata: - * - 'site' => Piwik_Site instance for this archive - * - 'period' => Piwik_Period instance for this archive - * - 'timestamp' => the timestamp of the first date in this archive - * - * @param bool $isSimple Whether the DataTable should be a DataTable_Simple - * instance or not. - * @return Piwik_DataTable - */ - public function makeDataTable($isSimple = false) - { - if ($isSimple) { - $result = new Piwik_DataTable_Simple(); - } else { - $result = new Piwik_DataTable(); - } - - $result->setMetadata('site', $this->getSite()); - $result->setMetadata('period', $this->getPeriod()); - $result->setMetadata('timestamp', $this->getTimestampStartDate()); - - return $result; - } - - public function setRequestedReport($requestedReport) - { - $this->requestedReport = $requestedReport; - } - - /** - * Returns the report (the named collection of metrics) this Archive instance is - * currently going to query/process. - * - * @return string - */ - protected function getRequestedReport() - { - return self::getRequestedReportFor($this->requestedReport); - } - - /** - * Returns the name of the report (the named collection of metrics) that contains the - * specified metric. - * - * @param string $metric The metric whose report is being requested. If this does - * not belong to a known report, its assumed to be the report - * itself. - * @return string - */ - public static function getRequestedReportFor($metric) - { - // Core metrics are always processed in Core, for the requested date/period/segment - if (in_array($metric, Piwik_ArchiveProcessing::getCoreMetrics()) - || $metric == 'max_actions' - ) { - return 'VisitsSummary_CoreMetrics'; - } - // VisitFrequency metrics don't follow the same naming convention (HACK) - if (strpos($metric, '_returning') > 0 - // ignore Goal_visitor_returning_1_1_nb_conversions - && strpos($metric, 'Goal_') === false - ) { - return 'VisitFrequency_Metrics'; - } - // Goal_* metrics are processed by the Goals plugin (HACK) - if (strpos($metric, 'Goal_') === 0) { - return 'Goals_Metrics'; - } - // Actions metrics are processed by the Actions plugin (HACK) (3RD HACK IN FACT) (YES, THIS IS TOO MUCH HACKING) - // (FIXME PLEASE). - if (in_array($metric, Piwik_Archive::$actionsMetrics)) { - return 'Actions_Metrics'; - } - return $metric; - } - - /** - * Returns a DataTable that has the name '$name' from the current Archive. - * Also loads in memory all subDataTable for this DataTable. - * - * For example, if $name = 'Referers_keywordBySearchEngine' it will load all DataTable - * named 'Referers_keywordBySearchEngine_*' and they will be set as subDataTable to the - * rows. You can then go through the rows - * $rows = DataTable->getRows(); - * and for each row request the subDataTable (in this case the DataTable of the keywords for each search engines) - * $idSubTable = $row->getIdSubDataTable(); - * $subTable = Piwik_DataTable_Manager::getInstance()->getTable($idSubTable); - * - * @param string $name - * @param int $idSubTable Optional subDataTable to load instead of loading the parent DataTable - * @return Piwik_DataTable - */ - public function getDataTableExpanded($name, $idSubTable = null) - { - $this->preFetchBlob($name); - $dataTableToLoad = $this->getDataTable($name, $idSubTable); - $this->loadSubDataTables($name, $dataTableToLoad, $addMetadataSubtableId = true); - $dataTableToLoad->enableRecursiveFilters(); - $this->freeBlob($name); - return $dataTableToLoad; - } - - /** - * Returns true if Piwik can launch the archiving process for this archive, - * false if otherwise. - * - * @return bool - */ - public function isArchivingDisabled() - { - return Piwik_ArchiveProcessing::isArchivingDisabledFor($this->segment, $this->period); - } -} diff --git a/core/ArchiveProcessing.php b/core/ArchiveProcessing.php index 18acd71b1a..31af7e1047 100644 --- a/core/ArchiveProcessing.php +++ b/core/ArchiveProcessing.php @@ -507,7 +507,7 @@ abstract class Piwik_ArchiveProcessing public function getDoneStringFlag($flagArchiveAsAllPlugins = false) { return self::getDoneStringFlagFor( - $this->getSegment(), $this->period, $this->getRequestedReport(), $flagArchiveAsAllPlugins); + $this->getSegment(), $this->period->getLabel(), $this->getRequestedReport(), $flagArchiveAsAllPlugins); } /** @@ -515,15 +515,15 @@ abstract class Piwik_ArchiveProcessing * whether the archive was created successfully or not). * * @param Piwik_Segment $segment - * @param Piwik_Period $period + * @param string $periodLabel * @param string $requestedReport * @param bool $flagArchiveAsAllPlugins * @return string */ - public static function getDoneStringFlagFor($segment, $period, $requestedReport, $flagArchiveAsAllPlugins = false) + public static function getDoneStringFlagFor($segment, $periodLabel, $requestedReport, $flagArchiveAsAllPlugins = false) { $segmentHash = $segment->getHash(); - if (!self::shouldProcessReportsAllPluginsFor($segment, $period)) { + if (!self::shouldProcessReportsAllPluginsFor($segment, $periodLabel)) { $pluginProcessed = self::getPluginBeingProcessed($requestedReport); if (!Piwik_PluginsManager::getInstance()->isPluginLoaded($pluginProcessed) || $flagArchiveAsAllPlugins @@ -583,10 +583,10 @@ abstract class Piwik_ArchiveProcessing } $this->insertNumericRecord($done, $flag); } - + /** * Returns the name of the numeric table where the archive numeric values are stored - * + * * @return string */ public function getTableArchiveNumericName() @@ -639,7 +639,7 @@ abstract class Piwik_ArchiveProcessing $this->requestedReport = $requestedReport; } - protected function getRequestedReport() + public function getRequestedReport() { return $this->requestedReport; } @@ -957,15 +957,15 @@ abstract class Piwik_ArchiveProcessing */ public function isArchivingDisabled() { - return self::isArchivingDisabledFor($this->getSegment(), $this->period); + return self::isArchivingDisabledFor($this->getSegment(), $this->period->getLabel()); } - public static function isArchivingDisabledFor($segment, $period) + public static function isArchivingDisabledFor($segment, $periodLabel) { - if ($period->getLabel() == 'range') { + if ($periodLabel == 'range') { return false; } - $processOneReportOnly = !self::shouldProcessReportsAllPluginsFor($segment, $period); + $processOneReportOnly = !self::shouldProcessReportsAllPluginsFor($segment, $periodLabel); $isArchivingDisabled = !self::isRequestAuthorizedToArchive(); if ($processOneReportOnly) { @@ -1003,17 +1003,17 @@ abstract class Piwik_ArchiveProcessing */ protected function shouldProcessReportsAllPlugins($segment, $period) { - return self::shouldProcessReportsAllPluginsFor($segment, $period); + return self::shouldProcessReportsAllPluginsFor($segment, $period->getLabel()); } /** * @param Piwik_Segment $segment - * @param Piwik_Period $period + * @param string $period * @return bool */ - protected static function shouldProcessReportsAllPluginsFor($segment, $period) + protected static function shouldProcessReportsAllPluginsFor($segment, $periodLabel) { - if ($segment->isEmpty() && $period->getLabel() != 'range') { + if ($segment->isEmpty() && $periodLabel != 'range') { return true; } diff --git a/core/ArchiveProcessing/Day.php b/core/ArchiveProcessing/Day.php index 75b48bde5c..96a630a93a 100644 --- a/core/ArchiveProcessing/Day.php +++ b/core/ArchiveProcessing/Day.php @@ -126,11 +126,7 @@ class Piwik_ArchiveProcessing_Day extends Piwik_ArchiveProcessing */ private function redirectRequestToVisitsSummary() { - $archive = new Piwik_Archive_Single(); - $archive->setSite($this->site); - $archive->setPeriod($this->period); - $archive->setSegment($this->getSegment()); - $archive->setRequestedReport('VisitsSummary'); + $archive = new Piwik_Archive($this->site->getId(), $this->period, $this->getSegment()); $nbVisits = $archive->getNumeric('nb_visits'); $this->isThereSomeVisits = $nbVisits > 0; diff --git a/core/ArchiveProcessing/Period.php b/core/ArchiveProcessing/Period.php index 6d3de2e6f6..840f5b5c97 100644 --- a/core/ArchiveProcessing/Period.php +++ b/core/ArchiveProcessing/Period.php @@ -34,7 +34,42 @@ class Piwik_ArchiveProcessing_Period extends Piwik_ArchiveProcessing /** * @var Piwik_Archive_Single[] */ - public $archives = array(); + //public $archives = array(); + + public $archive = null; + + /** + * Set the period + * + * @param Piwik_Period $period + */ + public function setPeriod( Piwik_Period $period ) + { + parent::setPeriod($period); + $this->resetSubperiodArchiveQuery(); + } + + /** + * Sets the segment. + * + * @param Piwik_Segment $segment + */ + public function setSegment( Piwik_Segment $segment) + { + parent::setSegment($segment); + $this->resetSubperiodArchiveQuery(); + } + + /** + * Set the site + * + * @param Piwik_Site $site + */ + public function setSite( Piwik_Site $site ) + { + parent::setSite($site); + $this->resetSubperiodArchiveQuery(); + } /** * Sums all values for the given field names $aNames over the period @@ -78,52 +113,72 @@ class Piwik_ArchiveProcessing_Period extends Piwik_ArchiveProcessing $aNames = array($aNames); } - // fetch the numeric values and apply the operation on them - $results = array(); - foreach ($this->archives as $id => $archive) { - foreach ($aNames as $name) { - if (!isset($results[$name])) { - $results[$name] = 0; - } - if ($name == 'nb_uniq_visitors') continue; - - $valueToSum = $archive->getNumeric($name); - - if ($valueToSum !== false) { - switch ($operationToApply) { - case 'sum': - $results[$name] += $valueToSum; - break; - case 'max': - $results[$name] = max($results[$name], $valueToSum); - break; - case 'min': - $results[$name] = min($results[$name], $valueToSum); - break; - default: - throw new Exception("Operation not applicable."); - break; - } + // remove nb_uniq_visitors if present + foreach ($aNames as $i => $name) { + if ($name == 'nb_uniq_visitors') { + $results['nb_uniq_visitors'] = 0; + unset($aNames[$i]); + + break; + } + } + + // data will be array mapping each period w/ result row for period + $data = $this->archive->getNumeric($aNames); + foreach ($data as $dateRange => $row) { + foreach ($row as $name => $value) { + switch ($operationToApply) { + case 'sum': + if (!isset($results[$name])) { + $results[$name] = 0; + } + + $results[$name] += $value; + break; + case 'max': + if (!isset($results[$name])) { + $results[$name] = 0; + } + + $results[$name] = max($results[$name], $value); + break; + case 'min': + if (!isset($results[$name])) { + $results[$name] = $value; + } + + $results[$name] = min($results[$name], $value); + break; + default: + throw new Exception("Operation not applicable."); + break; } } } - + + // set default for metrics that weren't found + foreach ($aNames as $name) { + if (!isset($results[$name])) { + $results[$name] = 0; + } + } + if (!Piwik::isUniqueVisitorsEnabled($this->period->getLabel())) { unset($results['nb_uniq_visitors']); } - - foreach ($results as $name => $value) { + + foreach($results as $name => $value) { if ($name == 'nb_uniq_visitors') { - $value = (float)$this->computeNbUniqVisitors(); + $value = (float) $this->computeNbUniqVisitors(); } $this->insertRecord($name, $value); } - + // if asked for only one field to sum if (count($results) == 1) { - return $results[$name]; + return reset($results); } - + // returns the array of records once summed return $results; } @@ -200,23 +255,24 @@ class Piwik_ArchiveProcessing_Period extends Piwik_ArchiveProcessing protected function getRecordDataTableSum($name, $invalidSummedColumnNameToRenamedName, &$columnAggregationOperations = null) { $table = new Piwik_DataTable(); - - if (is_array($columnAggregationOperations)) { + if ($columnAggregationOperations !== null) { $table->setColumnAggregationOperations($columnAggregationOperations); } - foreach ($this->archives as $archive) { - $archive->preFetchBlob($name); - $datatableToSum = $archive->getDataTable($name); - $archive->loadSubDataTables($name, $datatableToSum); + $data = $this->archive->getDataTableExpanded($name, $idSubTable = null, $addMetadataSubtableId = false); + foreach ($data->getArray() as $dateRange => $datatableToSum) + { $table->addDataTable($datatableToSum); - $archive->freeBlob($name); } - - if (is_null($invalidSummedColumnNameToRenamedName)) { + + unset($data); + + if(is_null($invalidSummedColumnNameToRenamedName)) + { $invalidSummedColumnNameToRenamedName = self::$invalidSummedColumnNameToRenamedName; } - foreach ($invalidSummedColumnNameToRenamedName as $oldName => $newName) { + foreach($invalidSummedColumnNameToRenamedName as $oldName => $newName) + { $table->renameColumn($oldName, $newName); } return $table; @@ -228,25 +284,20 @@ class Piwik_ArchiveProcessing_Period extends Piwik_ArchiveProcessing } /** - * Returns the ID of the archived subperiods. - * - * @return array Array of the idArchive of the subperiods + * Returns an archive instance that can be used to query for data in each + * subperiod of the period we're archiving data for. + * + * @return Piwik_Archive */ protected function loadSubperiodsArchive() { - $periods = array(); - - // we first compute every subperiod of the archive - foreach ($this->period->getSubperiods() as $period) { - $archivePeriod = new Piwik_Archive_Single(); - $archivePeriod->setSite($this->site); - $archivePeriod->setPeriod($period); - $archivePeriod->setSegment($this->getSegment()); - $archivePeriod->setRequestedReport($this->getRequestedReport()); - - $periods[] = $archivePeriod; - } - return $periods; + return new Piwik_Archive( + array($this->site->getId()), + $this->period->getSubperiods(), + $this->getSegment(), + $forceIndexedBySite = false, + $forceIndexedByDate = true + ); } /** @@ -266,8 +317,9 @@ class Piwik_ArchiveProcessing_Period extends Piwik_ArchiveProcessing protected function loadSubPeriods() { - if (empty($this->archives)) { - $this->archives = $this->loadSubperiodsArchive(); + if(is_null($this->archive)) + { + $this->archive = $this->loadSubperiodsArchive(); } } @@ -290,18 +342,21 @@ class Piwik_ArchiveProcessing_Period extends Piwik_ArchiveProcessing $record = $this->archiveNumericValuesSum($toSum); $this->archiveNumericValuesMax('max_actions'); - $nbVisitsConverted = $record['nb_visits_converted']; - $nbVisits = $record['nb_visits']; + if (!isset($record['nb_visits'])) { + $nbVisits = $nbVisitsConverted = 0; + } else { + $nbVisitsConverted = $record['nb_visits_converted']; + $nbVisits = $record['nb_visits']; + } } else { - $archive = new Piwik_Archive_Single(); - $archive->setSite($this->site); - $archive->setPeriod($this->period); - $archive->setSegment($this->getSegment()); - - $nbVisits = $archive->getNumeric('nb_visits'); - $nbVisitsConverted = 0; - if ($nbVisits > 0) { - $nbVisitsConverted = $archive->getNumeric('nb_visits_converted'); + $archive = new Piwik_Archive($this->site->getId(), $this->period, $this->getSegment()); + + $metrics = $archive->getNumeric(array('nb_visits', 'nb_visits_converted')); + if (!isset($metrics['nb_visits'])) { + $nbVisits = $nbVisitsConverted = 0; + } else { + $nbVisits = $metrics['nb_visits']; + $nbVisitsConverted = $metrics['nb_visits_converted']; } } @@ -324,8 +379,8 @@ class Piwik_ArchiveProcessing_Period extends Piwik_ArchiveProcessing $select = "count(distinct log_visit.idvisitor) as nb_uniq_visitors"; $from = "log_visit"; $where = "log_visit.visit_last_action_time >= ? - AND log_visit.visit_last_action_time <= ? - AND log_visit.idsite = ?"; + AND log_visit.visit_last_action_time <= ? + AND log_visit.idsite = ?"; $bind = array($this->getStartDatetimeUTC(), $this->getEndDatetimeUTC(), $this->idsite); @@ -344,14 +399,8 @@ class Piwik_ArchiveProcessing_Period extends Piwik_ArchiveProcessing $numericTable = $this->tableArchiveNumeric->getTableName(); self::doPurgeOutdatedArchives($numericTable, $this->isArchiveTemporary()); - - if (!isset($this->archives)) { - return; - } - foreach ($this->archives as $archive) { - destroy($archive); - } - $this->archives = array(); + + $this->resetSubperiodArchiveQuery(); } const FLAG_TABLE_PURGED = 'lastPurge_'; @@ -397,12 +446,12 @@ class Piwik_ArchiveProcessing_Period extends Piwik_ArchiveProcessing $purgeArchivesOlderThan = Piwik_Date::factory('today')->getDateTime(); } $result = Piwik_FetchAll(" - SELECT idarchive - FROM $numericTable - WHERE name LIKE 'done%' - AND (( value = " . Piwik_ArchiveProcessing::DONE_OK_TEMPORARY . " - AND ts_archived < ?) - OR value = " . Piwik_ArchiveProcessing::DONE_ERROR . ")", + SELECT idarchive + FROM $numericTable + WHERE name LIKE 'done%' + AND (( value = " . Piwik_ArchiveProcessing::DONE_OK_TEMPORARY . " + AND ts_archived < ?) + OR value = " . Piwik_ArchiveProcessing::DONE_ERROR . ")", array($purgeArchivesOlderThan) ); @@ -412,9 +461,9 @@ class Piwik_ArchiveProcessing_Period extends Piwik_ArchiveProcessing $idArchivesToDelete[] = $row['idarchive']; } $query = "DELETE - FROM %s - WHERE idarchive IN (" . implode(',', $idArchivesToDelete) . ") - "; + FROM %s + WHERE idarchive IN (" . implode(',', $idArchivesToDelete) . ") + "; Piwik_Query(sprintf($query, $numericTable)); @@ -430,9 +479,9 @@ class Piwik_ArchiveProcessing_Period extends Piwik_ArchiveProcessing // and would take up unecessary space $yesterday = Piwik_Date::factory('yesterday')->getDateTime(); $query = "DELETE - FROM %s - WHERE period = ? - AND ts_archived < ?"; + FROM %s + WHERE period = ? + AND ts_archived < ?"; $bind = array(Piwik::$idPeriods['range'], $yesterday); Piwik::log("Purging Custom Range archives: done [ purged archives older than $yesterday from $blobTable and $numericTable ]"); @@ -449,4 +498,12 @@ class Piwik_ArchiveProcessing_Period extends Piwik_ArchiveProcessing Piwik::log("Purging temporary archives: skipped."); } } -}
\ No newline at end of file + + private function resetSubperiodArchiveQuery() + { + if ($this->archive !== null) { + destroy($this->archive); + $this->archive = null; + } + } +} diff --git a/core/DataAccess/ArchiveQuery.php b/core/DataAccess/ArchiveQuery.php new file mode 100644 index 0000000000..ca3fbbd0e5 --- /dev/null +++ b/core/DataAccess/ArchiveQuery.php @@ -0,0 +1,189 @@ +<?php +/** + * Piwik - Open source web analytics + * + * @link http://piwik.org + * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later + * + * @category Piwik + * @package Piwik + */ + +/** + * Data Access object used to query archive data. + */ +class Piwik_DataAccess_ArchiveQuery +{ + /** + * Queries and returns archive IDs for a set of sites, periods, and a segment. + * + * @param array $siteIds + * @param array $periods + * @param Piwik_Segment|null $segment + * @param array $requestedReports + * @return array Archive IDs are grouped by archive name and period range, ie, + * array( + * 'VisitsSummary.done' => array( + * '2010-01-01' => array(1,2,3) + * ) + * ) + */ + public function getArchiveIds($siteIds, $periods, $segment, $requestedReports) + { + $periodType = reset($periods)->getLabel(); + + $getArchiveIdsSql = "SELECT idsite, name, date1, date2, MAX(idarchive) as idarchive + FROM %s + WHERE period = ? + AND %s + AND ".$this->getNameCondition($requestedReports, $segment, $periodType)." + AND idsite IN (".implode(',', $siteIds).") + GROUP BY idsite, date1, date2"; + + // for every month within the archive query, select from numeric table + $result = array(); + foreach ($this->getPeriodsByTableMonth($periods) as $tableMonth => $periods) { + $firstPeriod = reset($periods); + $table = Piwik_Common::prefixTable("archive_numeric_$tableMonth"); + + Piwik_TablePartitioning_Monthly::createArchiveTablesIfAbsent($firstPeriod); + + // if looking for a range archive. NOTE: we assume there's only one period if its a range. + $bind = array($firstPeriod->getId()); + if ($firstPeriod instanceof Piwik_Period_Range) { + $dateCondition = "date1 = ? AND date2 = ?"; + $bind[] = $firstPeriod->getDateStart()->toString('Y-m-d'); + $bind[] = $firstPeriod->getDateEnd()->toString('Y-m-d'); + } else { // if looking for a normal period + $dateStrs = array(); + foreach ($periods as $period) { + $dateStrs[] = $period->getDateStart()->toString('Y-m-d'); + } + + $dateCondition = "date1 IN ('".implode("','", $dateStrs)."')"; + } + + $sql = sprintf($getArchiveIdsSql, $table, $dateCondition); + + // get the archive IDs + foreach (Piwik_FetchAll($sql, $bind) as $row) { + $archiveName = $row['name']; + $dateStr = $row['date1'].",".$row['date2']; + + $result[$archiveName][$dateStr][] = $row['idarchive']; + } + } + + return $result; + } + + /** + * Queries and returns archive data using a set of archive IDs. + * + * @param array $archiveIds The IDs of the archives to get data from. + * @param array $archiveNames The names of the data to retrieve (ie, nb_visits, + * nb_actions, etc.) + * @param string $archiveDataType The archive data type (either, 'blob' or 'numeric'). + * @param string|null $idSubtable The subtable to retrieve ('all' for all subtables). + */ + public function getArchiveData($archiveIds, $archiveNames, $archiveDataType, $idSubtable) + { + $archiveTableType = 'archive_'.$archiveDataType; + + // create the SQL to select archive data + $inNames = Piwik_Common::getSqlStringFieldsArray($archiveNames); + if ($idSubtable == 'all') { + $name = reset($archiveNames); + + // select blobs w/ name like "$name_[0-9]+" w/o using RLIKE + $nameEnd = strlen($name) + 2; + $getValuesSql = "SELECT value, name, idsite, date1, date2, ts_archived + FROM %s + WHERE idarchive IN (%s) + AND (name = ? OR + (name LIKE ? AND SUBSTRING(name, $nameEnd, 1) >= '0' + AND SUBSTRING(name, $nameEnd, 1) <= '9') )"; + $bind = array($name, $name.'%'); + } else { + $getValuesSql = "SELECT name, value, idsite, date1, date2, ts_archived + FROM %s + WHERE idarchive IN (%s) + AND name IN ($inNames)"; + $bind = array_values($archiveNames); + } + + // get data from every table we're querying + $rows = array(); + foreach ($archiveIds as $period => $ids) { + $tableMonth = $this->getTableMonthFromDateRange($period); + + $table = Piwik_Common::prefixTable($archiveTableType."_".$tableMonth); + $sql = sprintf($getValuesSql, $table, implode(',', $ids)); + + foreach (Piwik_FetchAll($sql, $bind) as $row) { + $rows[] = $row; + } + } + + return $rows; + } + + /** + * Returns the SQL condition used to find successfully completed archives that + * this instance is querying for. + * + * @param array $requestedReports @see getRequestedReport + * @return string + */ + private function getNameCondition($requestedReports, $segment, $periodType) + { + // the flags used to tell how the archiving process for a specific archive was completed, + // if it was completed + $doneFlags = array(); + foreach ($requestedReports as $report) { + $done = Piwik_ArchiveProcessing::getDoneStringFlagFor($segment, $periodType, $report); + $donePlugins = Piwik_ArchiveProcessing::getDoneStringFlagFor($segment, $periodType, $report, true); + + $doneFlags[$done] = $done; + $doneFlags[$donePlugins] = $donePlugins; + } + + $allDoneFlags = "'".implode("','", $doneFlags)."'"; + + // create the SQL to find archives that are DONE + return "(name IN ($allDoneFlags)) AND + (value = '".Piwik_ArchiveProcessing::DONE_OK."' OR + value = '".Piwik_ArchiveProcessing::DONE_OK_TEMPORARY."')"; + } + + /** + * Returns the periods of the archives this instance is querying for grouped by + * by year & month. + * + * @return array The result will be an array of Piwik_Period instances, where each + * instance is associated w/ a string describing the year and month, + * eg, 2012_01. The format is the same format used in archive database + * table names. + */ + private function getPeriodsByTableMonth($periods) + { + $result = array(); + foreach ($periods as $period) { + $tableMonth = $period->getDateStart()->toString('Y_m'); + $result[$tableMonth][] = $period; + } + return $result; + } + + /** + * Returns the table & month that an archive for a specific date range is stored + * in. + * + * @param string $dateRange eg, "2012-01-01,2012-01-02" + * @return string eg, "2012_01" + */ + private function getTableMonthFromDateRange($dateRange) + { + return str_replace('-', '_', substr($dateRange, 0, 7)); + } +} diff --git a/core/DataTable.php b/core/DataTable.php index b5e2064980..ca38047bd9 100644 --- a/core/DataTable.php +++ b/core/DataTable.php @@ -369,11 +369,17 @@ class Piwik_DataTable /** * Apply a filter to this datatable * - * @param string $className Class name, eg. "Sort" or "Piwik_DataTable_Filter_Sort" + * @param string|Closure $className Class name, eg. "Sort" or "Piwik_DataTable_Filter_Sort". + * If this variable is a closure, it will get executed immediately. * @param array $parameters Array of parameters to the filter, eg. array('nb_visits', 'asc') */ public function filter($className, $parameters = array()) { + if ($className instanceof Closure) { + $className($this); + return; + } + if (!class_exists($className, false)) { $className = "Piwik_DataTable_Filter_" . $className; } @@ -1491,4 +1497,16 @@ class Piwik_DataTable return $this->columnAggregationOperations; } + /** + * Creates a new DataTable instance from a serialize()'d array of rows. + * + * @param string $data + * @return Piwik_DataTable + */ + public static function fromSerializedArray($data) + { + $result = new Piwik_DataTable(); + $result->addRowsFromSerializedArray($data); + return $result; + } } diff --git a/core/DataTable/Manager.php b/core/DataTable/Manager.php index 3fff1bcc2d..b6baa899d6 100644 --- a/core/DataTable/Manager.php +++ b/core/DataTable/Manager.php @@ -39,7 +39,7 @@ class Piwik_DataTable_Manager * * @var array */ - protected $tables = array(); + private $tables = array(); /** * Id of the next inserted table id in the Manager diff --git a/core/DataTable/Renderer/Rss.php b/core/DataTable/Renderer/Rss.php index 60f11ca13e..a9e6376455 100644 --- a/core/DataTable/Renderer/Rss.php +++ b/core/DataTable/Renderer/Rss.php @@ -66,7 +66,7 @@ class Piwik_DataTable_Renderer_Rss extends Piwik_DataTable_Renderer $out = ""; $moreRecentFirst = array_reverse($table->getArray(), true); foreach ($moreRecentFirst as $date => $subtable) { - $timestamp = $subtable->getMetadata('timestamp'); + $timestamp = $subtable->getMetadata('period')->getDateStart()->getTimestamp(); $site = $subtable->getMetadata('site'); $pudDate = date('r', $timestamp); diff --git a/core/Period.php b/core/Period.php index 6e6d4cf4d6..b004f1731a 100644 --- a/core/Period.php +++ b/core/Period.php @@ -265,4 +265,9 @@ abstract class Piwik_Period abstract public function getLocalizedShortString(); abstract public function getLocalizedLongString(); + + public function getRangeString() + { + return $this->getDateStart()->toString("Y-m-d").",".$this->getDateEnd()->toString("Y-m-d"); + } } diff --git a/core/Site.php b/core/Site.php index ae62ae0519..53a939256b 100644 --- a/core/Site.php +++ b/core/Site.php @@ -204,12 +204,13 @@ class Piwik_Site * Checks the given string for valid site ids and returns them as an array * * @param string $ids comma separated idSite list + * @param false|string $_restrictSitesToLogin Used only when running as a scheduled task. * @return array of valid integer */ - static public function getIdSitesFromIdSitesString($ids) + static public function getIdSitesFromIdSitesString($ids, $_restrictSitesToLogin = false) { if ($ids === 'all') { - return Piwik_SitesManager_API::getInstance()->getSitesIdWithAtLeastViewAccess(); + return Piwik_SitesManager_API::getInstance()->getSitesIdWithAtLeastViewAccess($_restrictSitesToLogin); } if (!is_array($ids)) { diff --git a/core/TablePartitioning.php b/core/TablePartitioning.php index 40bcec40d0..b3341940ba 100644 --- a/core/TablePartitioning.php +++ b/core/TablePartitioning.php @@ -97,6 +97,9 @@ abstract class Piwik_TablePartitioning */ class Piwik_TablePartitioning_Monthly extends Piwik_TablePartitioning { + private static $blobArchiveTable = null; + private static $numericArchiveTable = null; + public function __construct($tableName) { parent::__construct($tableName); @@ -107,24 +110,29 @@ class Piwik_TablePartitioning_Monthly extends Piwik_TablePartitioning $config = Piwik_Config::getInstance(); return $config->database['tables_prefix'] . $this->tableName . "_" . date("Y_m", $this->timestamp); } - -} - -/** - * - * @package Piwik - * @subpackage Piwik_TablePartitioning - */ -class Piwik_TablePartitioning_Daily extends Piwik_TablePartitioning -{ - public function __construct($tableName) + + /** + * Creates archive_blob & archive_numeric tables for a period if they don't + * already exist. + * + * @param Piwik_Period $periodInMonth + */ + public static function createArchiveTablesIfAbsent($periodInMonth) { - parent::__construct($tableName); + $timestamp = $periodInMonth->getDateStart()->getTimestamp(); + + self::$blobArchiveTable->setTimestamp($timestamp); + self::$blobArchiveTable->getTableName(); + + self::$numericArchiveTable->setTimestamp($timestamp); + self::$numericArchiveTable->getTableName(); } - - protected function generateTableName() + + public static function init() { - $config = Piwik_Config::getInstance(); - return $config->database['tables_prefix'] . $this->tableName . "_" . date("Y_m_d", $this->timestamp); + self::$blobArchiveTable = new Piwik_TablePartitioning_Monthly('archive_blob'); + self::$numericArchiveTable = new Piwik_TablePartitioning_Monthly('archive_numeric'); } } + +Piwik_TablePartitioning_Monthly::init(); |