diff options
| author | diosmosis <diosmosis@users.noreply.github.com> | 2020-03-10 03:21:49 +0300 |
|---|---|---|
| committer | GitHub <noreply@github.com> | 2020-03-10 03:21:49 +0300 |
| commit | 22cde646ec8247a068f75e673b69a51b97c825c2 (patch) | |
| tree | 9625275ddbcc4b4252f3ab8dc42f449b3cf6bb3c | |
| parent | d6df56cb86f74b4fd3406b2802945470449b4b0a (diff) | |
Move Archive.php archive invalidation to Loader… (#15616)3.13.4-b1
* Move Archive.php archive invalidation to Loader so we only invalidate when about to launch archiving.
* Attempt to handle more cases when invalidating before launching archiving.
* fix possible sql error
* fix possible error
* fixing some tests
* remove test code
* Only invalidate specific archive being requested.
* Do not invalidate on today in tracker and avoid existing valid archive check in CronArchive.
* more test fixes
* Attempt to fix more tests.
* Fixing last tests.
* another test fix
* Invalidate in scheduled task if browser triggered archiving is enabled.
* deal with TODO
* Get ArchiveSelectorTest to pass.
* applying review feedback including new tests
* apply review feedback & fix tests
* fix couple more tests
Co-authored-by: Thomas Steur <tsteur@users.noreply.github.com>
25 files changed, 879 insertions, 650 deletions
diff --git a/core/Archive.php b/core/Archive.php index 1f56ef6062..141d477af4 100644 --- a/core/Archive.php +++ b/core/Archive.php @@ -12,7 +12,6 @@ use Piwik\Archive\ArchiveQuery; use Piwik\Archive\ArchiveQueryFactory; use Piwik\Archive\Parameters; use Piwik\ArchiveProcessor\Rules; -use Piwik\Archive\ArchiveInvalidator; use Piwik\Container\StaticContainer; use Piwik\DataAccess\ArchiveSelector; @@ -168,11 +167,6 @@ class Archive implements ArchiveQuery private static $cache; /** - * @var ArchiveInvalidator - */ - private $invalidator; - - /** * @param Parameters $params * @param bool $forceIndexedBySite Whether to force index the result of a query by site ID. * @param bool $forceIndexedByDate Whether to force index the result of a query by period. @@ -183,8 +177,6 @@ class Archive implements ArchiveQuery $this->params = $params; $this->forceIndexedBySite = $forceIndexedBySite; $this->forceIndexedByDate = $forceIndexedByDate; - - $this->invalidator = StaticContainer::get('Piwik\Archive\ArchiveInvalidator'); } /** @@ -453,67 +445,6 @@ class Archive implements ArchiveQuery return $dataTable; } - private function getSiteIdsThatAreRequestedInThisArchiveButWereNotInvalidatedYet() - { - if (is_null(self::$cache)) { - self::$cache = Cache::getTransientCache(); - } - - $id = 'Archive.SiteIdsOfRememberedReportsInvalidated'; - - if (!self::$cache->contains($id)) { - self::$cache->save($id, array()); - } - - $siteIdsAlreadyHandled = self::$cache->fetch($id); - $siteIdsRequested = $this->params->getIdSites(); - - foreach ($siteIdsRequested as $index => $siteIdRequested) { - $siteIdRequested = (int) $siteIdRequested; - - if (in_array($siteIdRequested, $siteIdsAlreadyHandled)) { - unset($siteIdsRequested[$index]); // was already handled previously, do not do it again - } else { - $siteIdsAlreadyHandled[] = $siteIdRequested; // we will handle this id this time - } - } - - self::$cache->save($id, $siteIdsAlreadyHandled); - - return $siteIdsRequested; - } - - private function invalidatedReportsIfNeeded() - { - $siteIdsRequested = $this->getSiteIdsThatAreRequestedInThisArchiveButWereNotInvalidatedYet(); - - if (empty($siteIdsRequested)) { - return; // all requested site ids were already handled - } - - $sitesPerDays = $this->invalidator->getRememberedArchivedReportsThatShouldBeInvalidated(); - - foreach ($sitesPerDays as $date => $siteIds) { - if (empty($siteIds)) { - continue; - } - - $siteIdsToActuallyInvalidate = array_intersect($siteIds, $siteIdsRequested); - - if (empty($siteIdsToActuallyInvalidate)) { - continue; // all site ids that should be handled are already handled - } - - try { - $this->invalidator->markArchivesAsInvalidated($siteIdsToActuallyInvalidate, array(Date::factory($date)), false); - } catch (\Exception $e) { - Site::clearCache(); - throw $e; - } - } - - Site::clearCache(); - } /** * Queries archive tables for data and returns the result. @@ -638,8 +569,6 @@ class Archive implements ArchiveQuery */ private function cacheArchiveIdsAfterLaunching($archiveGroups, $plugins) { - $this->invalidatedReportsIfNeeded(); - $today = Date::today(); foreach ($this->params->getPeriods() as $period) { @@ -856,8 +785,11 @@ class Archive implements ArchiveQuery */ private function prepareArchive(array $archiveGroups, Site $site, Period $period) { + // if cron archiving is running, we will invalidate in CronArchive, not here + $invalidateBeforeArchiving = !SettingsServer::isArchivePhpTriggered(); + $parameters = new ArchiveProcessor\Parameters($site, $period, $this->params->getSegment()); - $archiveLoader = new ArchiveProcessor\Loader($parameters); + $archiveLoader = new ArchiveProcessor\Loader($parameters, $invalidateBeforeArchiving); $periodString = $period->getRangeString(); diff --git a/core/Archive/ArchiveInvalidator.php b/core/Archive/ArchiveInvalidator.php index 68274c9857..ad449921f2 100644 --- a/core/Archive/ArchiveInvalidator.php +++ b/core/Archive/ArchiveInvalidator.php @@ -147,6 +147,10 @@ class ArchiveInvalidator $siteId = (int) $report[0]; $date = $report[1]; + if (empty($siteId)) { + continue; + } + if (empty($sitesPerDay[$date])) { $sitesPerDay[$date] = array(); } diff --git a/core/ArchiveProcessor/Loader.php b/core/ArchiveProcessor/Loader.php index 4564146bd9..20627e98fd 100644 --- a/core/ArchiveProcessor/Loader.php +++ b/core/ArchiveProcessor/Loader.php @@ -8,13 +8,18 @@ */ namespace Piwik\ArchiveProcessor; +use Piwik\Archive\ArchiveInvalidator; use Piwik\Cache; use Piwik\Config; use Piwik\Container\StaticContainer; use Piwik\Context; use Piwik\DataAccess\ArchiveSelector; +use Piwik\DataAccess\ArchiveTableCreator; use Piwik\Date; +use Piwik\Db; use Piwik\Piwik; +use Piwik\Site; +use Psr\Log\LoggerInterface; /** * This class uses PluginsArchiver class to trigger data aggregation and create archives. @@ -33,9 +38,28 @@ class Loader */ protected $params; - public function __construct(Parameters $params) + /** + * @var ArchiveInvalidator + */ + private $invalidator; + + /** + * @var \Piwik\Cache\Cache + */ + private $cache; + + /** + * @var LoggerInterface + */ + private $logger; + + public function __construct(Parameters $params, $invalidateBeforeArchiving = false) { $this->params = $params; + $this->invalidateBeforeArchiving = $invalidateBeforeArchiving; + $this->invalidator = StaticContainer::get(ArchiveInvalidator::class); + $this->cache = Cache::getTransientCache(); + $this->logger = StaticContainer::get(LoggerInterface::class); } /** @@ -65,11 +89,19 @@ class Loader { $this->params->setRequestedPlugin($pluginName); - list($idArchive, $visits, $visitsConverted) = $this->loadExistingArchiveIdFromDb(); - if (!empty($idArchive)) { + list($idArchive, $visits, $visitsConverted, $isAnyArchiveExists) = $this->loadExistingArchiveIdFromDb(); + if (!empty($idArchive)) { // we have a usable idarchive (it's not invalidated and it's new enough) return $idArchive; } + // if there is an archive, but we can't use it for some reason, invalidate existing archives before + // we start archiving. if the archive is made invalid, we will correctly re-archive below. + if ($this->invalidateBeforeArchiving + && $isAnyArchiveExists + ) { + $this->invalidatedReportsIfNeeded(); + } + /** @var ArchivingStatus $archivingStatus */ $archivingStatus = StaticContainer::get(ArchivingStatus::class); $archivingStatus->archiveStarted($this->params); @@ -170,20 +202,17 @@ class Loader */ public function loadExistingArchiveIdFromDb() { - $noArchiveFound = array(false, false, false); - if ($this->isArchivingForcedToTrigger()) { - return $noArchiveFound; - } + $this->logger->debug("Archiving forced to trigger for {$this->params}."); - $minDatetimeArchiveProcessedUTC = $this->getMinTimeArchiveProcessed(); - $idAndVisits = ArchiveSelector::getArchiveIdAndVisits($this->params, $minDatetimeArchiveProcessedUTC); - - if (!$idAndVisits) { - return $noArchiveFound; + // return no usable archive found, and no existing archive. this will skip invalidation, which should + // be fine since we just force archiving. + return [false, false, false, false]; } - return $idAndVisits; + $minDatetimeArchiveProcessedUTC = $this->getMinTimeArchiveProcessed(); + $result = ArchiveSelector::getArchiveIdAndVisits($this->params, $minDatetimeArchiveProcessedUTC); + return $result; } /** @@ -242,4 +271,46 @@ class Loader return $cache->fetch($cacheKey); } + + // public for tests + public function getReportsToInvalidate() + { + $sitesPerDays = $this->invalidator->getRememberedArchivedReportsThatShouldBeInvalidated(); + + foreach ($sitesPerDays as $dateStr => $siteIds) { + if (empty($siteIds) + || !in_array($this->params->getSite()->getId(), $siteIds) + ) { + unset($sitesPerDays[$dateStr]); + } + + $date = Date::factory($dateStr); + if ($date->isEarlier($this->params->getPeriod()->getDateStart()) + || $date->isLater($this->params->getPeriod()->getDateEnd()) + ) { // date in list is not the current date, so ignore it + unset($sitesPerDays[$dateStr]); + } + } + + return $sitesPerDays; + } + + private function invalidatedReportsIfNeeded() + { + $sitesPerDays = $this->getReportsToInvalidate(); + if (empty($sitesPerDays)) { + return; + } + + foreach ($sitesPerDays as $date => $siteIds) { + try { + $this->invalidator->markArchivesAsInvalidated([$this->params->getSite()->getId()], array(Date::factory($date)), false, $this->params->getSegment()); + } catch (\Exception $e) { + Site::clearCache(); + throw $e; + } + } + + Site::clearCache(); + } } diff --git a/core/ArchiveProcessor/Parameters.php b/core/ArchiveProcessor/Parameters.php index e765787920..954be0a87d 100644 --- a/core/ArchiveProcessor/Parameters.php +++ b/core/ArchiveProcessor/Parameters.php @@ -258,4 +258,9 @@ class Parameters { $this->isRootArchiveRequest = $isRootArchiveRequest; } + + public function __toString() + { + return "[idSite = {$this->getSite()->getId()}, period = {$this->getPeriod()->getLabel()} {$this->getPeriod()->getRangeString()}, segment = {$this->getSegment()->getString()}]"; + } } diff --git a/core/CronArchive.php b/core/CronArchive.php index 99bd257c43..9d8801fcbd 100644 --- a/core/CronArchive.php +++ b/core/CronArchive.php @@ -898,6 +898,8 @@ class CronArchive $visitsLastDays = 0; + $this->invalidateArchivedReportsForSitesThatNeedToBeArchivedAgain(); + list($isThereArchive, $newDate) = $this->isThereAValidArchiveForPeriod($idSite, 'day', $date, $segment = ''); if ($isThereArchive) { $visitsToday = Archive::build($idSite, 'day', $date)->getNumeric('nb_visits'); @@ -972,7 +974,8 @@ class CronArchive return $dayArchiveWasSuccessful; } - private function isThereAValidArchiveForPeriod($idSite, $period, $date, $segment = '') + // public for tests + public function isThereAValidArchiveForPeriod($idSite, $period, $date, $segment = '') { if (Range::isMultiplePeriod($date, $period)) { $rangePeriod = Factory::build($period, $date, Site::getTimezoneFor($idSite)); @@ -981,9 +984,17 @@ class CronArchive $periodsToCheck = [Factory::build($period, $date, Site::getTimezoneFor($idSite))]; } - $periodsToCheckRanges = array_map(function (Period $p) { return $p->getRangeString(); }, $periodsToCheck); + $isTodayIncluded = $this->isTodayIncludedInPeriod($idSite, $periodsToCheck); + $isLast = preg_match('/^last([0-9]+)/', $date, $matches); - $this->invalidateArchivedReportsForSitesThatNeedToBeArchivedAgain(); + // don't do this check for a single period that includes today + if ($isTodayIncluded + && !$isLast + ) { + return [false, null]; + } + + $periodsToCheckRanges = array_map(function (Period $p) { return $p->getRangeString(); }, $periodsToCheck); $archiveIds = ArchiveSelector::getArchiveIds( [$idSite], $periodsToCheck, new Segment($segment, [$idSite]), $plugins = [], // empty plugins param since we only check for an 'all' archive @@ -1003,37 +1014,60 @@ class CronArchive // if there is an invalidated archive within the range, find out the oldest one and how far it is from today, // and change the lastN $date to be value so it is correctly re-processed. $newDate = $date; - if (!$isThereArchiveForAllPeriods - && preg_match('/^last([0-9]+)/', $date, $matches) - ) { - $lastNValue = (int) $matches[1]; - - usort($diff, function ($lhs, $rhs) { - $lhsDate = explode(',', $lhs)[0]; - $rhsDate = explode(',', $rhs)[0]; - - if ($lhsDate == $rhsDate) { - return 1; - } else if (Date::factory($lhsDate)->isEarlier(Date::factory($rhsDate))) { - return -1; - } else { - return 1; - } - }); + if ($isLast) { + if (!$isThereArchiveForAllPeriods) { + $lastNValue = (int)$matches[1]; + + usort($diff, function ($lhs, $rhs) { + $lhsDate = explode(',', $lhs)[0]; + $rhsDate = explode(',', $rhs)[0]; + + if ($lhsDate == $rhsDate) { + return 1; + } else if (Date::factory($lhsDate)->isEarlier(Date::factory($rhsDate))) { + return -1; + } else { + return 1; + } + }); - $oldestDateWithoutArchive = explode(',', reset($diff))[0]; - $todayInTimezone = Date::factoryInTimezone('today', Site::getTimezoneFor($idSite)); + $oldestDateWithoutArchive = explode(',', reset($diff))[0]; + $todayInTimezone = Date::factoryInTimezone('today', Site::getTimezoneFor($idSite)); - /** @var Range $newRangePeriod */ - $newRangePeriod = PeriodFactory::build($period, $oldestDateWithoutArchive . ',' . $todayInTimezone); + /** @var Range $newRangePeriod */ + $newRangePeriod = PeriodFactory::build($period, $oldestDateWithoutArchive . ',' . $todayInTimezone); - $newDate = 'last' . min($lastNValue, $newRangePeriod->getNumberOfSubperiods()); + $newDate = 'last' . max(min($lastNValue, $newRangePeriod->getNumberOfSubperiods()), 2); + } else if ($isTodayIncluded) { + $isThereArchiveForAllPeriods = false; + $newDate = 'last2'; + } } return [$isThereArchiveForAllPeriods, $newDate]; } /** + * @param int $idSite + * @param Period[] $periods + * @return bool + * @throws Exception + */ + private function isTodayIncludedInPeriod($idSite, $periods) + { + $timezone = Site::getTimezoneFor($idSite); + $today = Date::factoryInTimezone('today', $timezone); + + foreach ($periods as $period) { + if ($period->isDateInPeriod($today)) { + return true; + } + } + + return false; + } + + /** * @param $idSite * @return array */ @@ -1102,6 +1136,8 @@ class CronArchive return Request::ABORT; } + $this->invalidateArchivedReportsForSitesThatNeedToBeArchivedAgain(); + list($isThereArchive, $newDate) = $this->isThereAValidArchiveForPeriod($idSite, $period, $date, $segment); if ($isThereArchive) { $this->logArchiveWebsiteSkippedValidArchiveExists($idSite, $period, $date); @@ -1979,6 +2015,8 @@ class CronArchive return Request::ABORT; } + $this->invalidateArchivedReportsForSitesThatNeedToBeArchivedAgain(); + list($isThereArchive, $newDate) = $this->isThereAValidArchiveForPeriod($idSite, $period, $date, $segment); if ($isThereArchive) { $this->logArchiveWebsiteSkippedValidArchiveExists($idSite, $period, $date, $segment); diff --git a/core/DataAccess/ArchiveSelector.php b/core/DataAccess/ArchiveSelector.php index 4232404141..daaeb7b2a6 100644 --- a/core/DataAccess/ArchiveSelector.php +++ b/core/DataAccess/ArchiveSelector.php @@ -48,7 +48,12 @@ class ArchiveSelector /** * @param ArchiveProcessor\Parameters $params * @param bool $minDatetimeArchiveProcessedUTC deprecated. will be removed in Matomo 4. - * @return array|bool + * @return array An array with four values: \ + * - the latest archive ID or false if none + * - the latest visits value for the latest archive, regardless of whether the archive is invalidated or not + * - the latest visits converted value for the latest archive, regardless of whether the archive is invalidated or not + * - whether there is an archive that exists or not. if this is true and the latest archive is false, it means + * the archive found was not usable (for example, it was invalidated and we are not looking for invalidated archives) * @throws Exception */ public static function getArchiveIdAndVisits(ArchiveProcessor\Parameters $params, $minDatetimeArchiveProcessedUTC = false, $includeInvalidated = true) @@ -61,80 +66,41 @@ class ArchiveSelector $numericTable = ArchiveTableCreator::getNumericTable($dateStart); - $minDatetimeIsoArchiveProcessedUTC = null; - if ($minDatetimeArchiveProcessedUTC) { - $minDatetimeIsoArchiveProcessedUTC = Date::factory($minDatetimeArchiveProcessedUTC)->getDatetime(); - } - $requestedPlugin = $params->getRequestedPlugin(); $segment = $params->getSegment(); $plugins = array("VisitsSummary", $requestedPlugin); $doneFlags = Rules::getDoneFlags($plugins, $segment); + $requestedPluginDoneFlags = Rules::getDoneFlags([$requestedPlugin], $segment); $doneFlagValues = Rules::getSelectableDoneFlagValues($includeInvalidated, $params); - $results = self::getModel()->getArchiveIdAndVisits($numericTable, $idSite, $period, $dateStartIso, $dateEndIso, $minDatetimeIsoArchiveProcessedUTC, $doneFlags, $doneFlagValues); - - if (empty($results)) { - return false; - } - - $idArchive = self::getMostRecentIdArchiveFromResults($segment, $requestedPlugin, $results); - - $idArchiveVisitsSummary = self::getMostRecentIdArchiveFromResults($segment, "VisitsSummary", $results); - - list($visits, $visitsConverted) = self::getVisitsMetricsFromResults($idArchive, $idArchiveVisitsSummary, $results); - - if (false === $visits && false === $idArchive) { - return false; + $results = self::getModel()->getArchiveIdAndVisits($numericTable, $idSite, $period, $dateStartIso, $dateEndIso, null, $doneFlags); + if (empty($results)) { // no archive found + return [false, false, false, false]; } - return array($idArchive, $visits, $visitsConverted); - } + $result = self::findArchiveDataWithLatestTsArchived($results, $requestedPluginDoneFlags); - protected static function getVisitsMetricsFromResults($idArchive, $idArchiveVisitsSummary, $results) - { - $visits = $visitsConverted = false; - $archiveWithVisitsMetricsWasFound = ($idArchiveVisitsSummary !== false); + $visits = isset($result['nb_visits']) ? $result['nb_visits'] : false; + $visitsConverted = isset($result['nb_visits_converted']) ? $result['nb_visits_converted'] : false; - if ($archiveWithVisitsMetricsWasFound) { - $visits = $visitsConverted = 0; + if (isset($result['value']) + && !in_array($result['value'], $doneFlagValues) + ) { // the archive cannot be considered valid for this request (has wrong done flag value) + return [false, $visits, $visitsConverted, true]; } - foreach ($results as $result) { - if (in_array($result['idarchive'], array($idArchive, $idArchiveVisitsSummary))) { - $value = (int)$result['value']; - if (empty($visits) - && $result['name'] == self::NB_VISITS_RECORD_LOOKED_UP - ) { - $visits = $value; - } - if (empty($visitsConverted) - && $result['name'] == self::NB_VISITS_CONVERTED_RECORD_LOOKED_UP - ) { - $visitsConverted = $value; - } - } + // the archive is too old + if ($minDatetimeArchiveProcessedUTC + && isset($result['idarchive']) + && Date::factory($result['ts_archived'])->isEarlier(Date::factory($minDatetimeArchiveProcessedUTC)) + ) { + return [false, $visits, $visitsConverted, true]; } - return array($visits, $visitsConverted); - } - - protected static function getMostRecentIdArchiveFromResults(Segment $segment, $requestedPlugin, $results) - { - $idArchive = false; - $namesRequestedPlugin = Rules::getDoneFlags(array($requestedPlugin), $segment); - - foreach ($results as $result) { - if ($idArchive === false - && in_array($result['name'], $namesRequestedPlugin) - ) { - $idArchive = $result['idarchive']; - break; - } - } + $idArchive = isset($result['idarchive']) ? $result['idarchive'] : false; - return $idArchive; + return array($idArchive, $visits, $visitsConverted, true); } /** @@ -213,7 +179,6 @@ class ArchiveSelector $sql = sprintf($getArchiveIdsSql, $table, $dateCondition); - $archiveIds = $db->fetchAll($sql, $bind); // get the archive IDs @@ -381,4 +346,71 @@ class ArchiveSelector // create the SQL to find archives that are DONE return "((name IN ($allDoneFlags)) AND (value IN (" . implode(',', $possibleValues) . ")))"; } + + /** + * This method takes the output of Model::getArchiveIdAndVisits() and selects data from the + * latest archives. + * + * This includes: + * - the idarchive with the latest ts_archived ($results will be ordered by ts_archived desc) + * - the visits/converted visits of the latest archive, which includes archives for VisitsSummary alone + * ($requestedPluginDoneFlags will have the done flag for the overall archive plus a done flag for + * VisitsSummary by itself) + * - the ts_archived for the latest idarchive + * - the doneFlag value for the latest archive + * + * @param $results + * @param $requestedPluginDoneFlags + * @return array + */ + private static function findArchiveDataWithLatestTsArchived($results, $requestedPluginDoneFlags) + { + // find latest idarchive for each done flag + $idArchives = []; + foreach ($results as $row) { + $doneFlag = $row['name']; + if (preg_match('/^done/', $doneFlag) + && !isset($idArchives[$doneFlag]) + ) { + $idArchives[$doneFlag] = $row['idarchive']; + } + } + + $archiveData = []; + + // gather the latest visits/visits_converted metrics + foreach ($results as $row) { + $name = $row['name']; + if (!isset($archiveData[$name]) + && in_array($name, [self::NB_VISITS_RECORD_LOOKED_UP, self::NB_VISITS_CONVERTED_RECORD_LOOKED_UP]) + && in_array($row['idarchive'], $idArchives) + ) { + $archiveData[$name] = $row['value']; + } + } + + // if an archive is found, but the metric data isn't found, we set the value to 0, + // so it won't get returned as false. this is here because the code used to do this before this change + // and we didn't want to introduce any side effects. it may be removable in the future. + foreach ([self::NB_VISITS_RECORD_LOOKED_UP, self::NB_VISITS_CONVERTED_RECORD_LOOKED_UP] as $metric) { + if (!empty($idArchives) + && !isset($archiveData[$metric]) + ) { + $archiveData[$metric] = 0; + } + } + + // set the idarchive & ts_archived for the archive we're looking for + foreach ($results as $row) { + $name = $row['name']; + if (in_array($name, $requestedPluginDoneFlags)) { + $archiveData['idarchive'] = $row['idarchive']; + $archiveData['ts_archived'] = $row['ts_archived']; + $archiveData['value'] = $row['value']; + break; + } + } + + return $archiveData; + } } diff --git a/core/DataAccess/Model.php b/core/DataAccess/Model.php index ab32487f8d..4e72698269 100644 --- a/core/DataAccess/Model.php +++ b/core/DataAccess/Model.php @@ -215,7 +215,8 @@ class Model return $deletedRows; } - public function getArchiveIdAndVisits($numericTable, $idSite, $period, $dateStartIso, $dateEndIso, $minDatetimeIsoArchiveProcessedUTC, $doneFlags, $doneFlagValues) + public function getArchiveIdAndVisits($numericTable, $idSite, $period, $dateStartIso, $dateEndIso, $minDatetimeIsoArchiveProcessedUTC, + $doneFlags, $doneFlagValues = null) { $bindSQL = array($idSite, $dateStartIso, @@ -231,7 +232,8 @@ class Model $bindSQL[] = $minDatetimeIsoArchiveProcessedUTC; } - $sqlQuery = "SELECT idarchive, value, name, date1 as startDate FROM $numericTable + // NOTE: we can't predict how many segments there will be so there could be lots of nb_visits/nb_visits_converted rows... have to select everything. + $sqlQuery = "SELECT idarchive, value, name, ts_archived, date1 as startDate FROM $numericTable WHERE idsite = ? AND date1 = ? AND date2 = ? @@ -240,7 +242,7 @@ class Model OR name = '" . ArchiveSelector::NB_VISITS_RECORD_LOOKED_UP . "' OR name = '" . ArchiveSelector::NB_VISITS_CONVERTED_RECORD_LOOKED_UP . "') $timeStampWhere - ORDER BY idarchive DESC"; + ORDER BY ts_archived DESC, idarchive DESC"; $results = Db::fetchAll($sqlQuery, $bindSQL); return $results; @@ -428,7 +430,14 @@ class Model $allDoneFlags = "'" . implode("','", $doneFlags) . "'"; // create the SQL to find archives that are DONE - return "((name IN ($allDoneFlags)) AND (value IN (" . implode(',', $possibleValues) . ")))"; + $result = "((name IN ($allDoneFlags))"; + + if (!empty($possibleValues)) { + $result .= " AND (value IN (" . implode(',', $possibleValues) . ")))"; + } + $result .= ')'; + + return $result; } } diff --git a/core/Period.php b/core/Period.php index 6e3ef0f327..e717995f28 100644 --- a/core/Period.php +++ b/core/Period.php @@ -248,6 +248,21 @@ abstract class Period } /** + * Returns whether the date `$date` is within the current period or not. + * + * Note: the time component of the period's dates and `$date` is ignored. + * + * @param Date $today + * @return bool + */ + public function isDateInPeriod(Date $date) + { + $ts = $date->getStartOfDay()->getTimestamp(); + return $ts >= $this->getDateStart()->getStartOfDay()->getTimestamp() + && $ts < $this->getDateEnd()->addDay(1)->getStartOfDay()->getTimestamp(); + } + + /** * Add a date to the period. * * Protected because adding periods after initialization is not supported. diff --git a/core/Tracker/Visit.php b/core/Tracker/Visit.php index 9c76967adc..d3de0015ca 100644 --- a/core/Tracker/Visit.php +++ b/core/Tracker/Visit.php @@ -572,10 +572,10 @@ class Visit implements VisitInterface $date = Date::factory((int)$time, $timezone); // $date->isToday() is buggy when server and website timezones don't match - so we'll do our own checking - $startOfTomorrow = Date::factoryInTimezone('today', $timezone)->addDay(1); - $isLaterThanToday = $date->getTimestamp() >= $startOfTomorrow->getTimestamp(); - if ($isLaterThanToday) { - return; + $startOfToday = Date::factoryInTimezone('yesterday', $timezone)->addDay(1); + $isLaterThanYesterday = $date->getTimestamp() >= $startOfToday->getTimestamp(); + if ($isLaterThanYesterday) { + return; // don't try to invalidate archives for today or later } $this->invalidator->rememberToInvalidateArchivedReportsLater($idSite, $date); diff --git a/core/Twig.php b/core/Twig.php index afb6a1cdc2..687e67be75 100644 --- a/core/Twig.php +++ b/core/Twig.php @@ -362,7 +362,7 @@ class Twig if (!empty($options['raw'])) { $template .= piwik_fix_lbrace($message); } else { - $template .= twig_escape_filter($twigEnv, $message, 'html'); + $template .= piwik_escape_filter($twigEnv, $message, 'html'); } $template .= '</div>'; diff --git a/plugins/CoreAdminHome/Tasks.php b/plugins/CoreAdminHome/Tasks.php index d94979c9c3..1e3933f972 100644 --- a/plugins/CoreAdminHome/Tasks.php +++ b/plugins/CoreAdminHome/Tasks.php @@ -9,11 +9,14 @@ namespace Piwik\Plugins\CoreAdminHome; use Piwik\API\Request; +use Piwik\Archive; +use Piwik\Archive\ArchiveInvalidator; use Piwik\ArchiveProcessor\Rules; use Piwik\Archive\ArchivePurger; use Piwik\Common; use Piwik\Config; use Piwik\Container\StaticContainer; +use Piwik\CronArchive; use Piwik\DataAccess\ArchiveTableCreator; use Piwik\Date; use Piwik\Db; @@ -60,6 +63,10 @@ class Tasks extends \Piwik\Plugin\Tasks public function schedule() { + // for browser triggered archiving, make sure we invalidate archives once a day just to make + // sure all archives that need to be invalidated get invalidated + $this->daily('invalidateOutdatedArchives', null, self::HIGH_PRIORITY); + // general data purge on older archive tables, executed daily $this->daily('purgeOutdatedArchives', null, self::HIGH_PRIORITY); @@ -81,6 +88,17 @@ class Tasks extends \Piwik\Plugin\Tasks $this->scheduleTrackingCodeReminderChecks(); } + public function invalidateOutdatedArchives() + { + if (!Rules::isBrowserTriggerEnabled()) { + $this->logger->info("Browser triggered archiving disabled, archives will be invalidated during core:archive."); + return; + } + + $cronArchive = new CronArchive(); + $cronArchive->invalidateArchivedReportsForSitesThatNeedToBeArchivedAgain(); + } + private function scheduleTrackingCodeReminderChecks() { $daysToTrackedVisitsCheck = (int) Config::getInstance()->General['num_days_before_tracking_code_reminder']; diff --git a/plugins/CoreAdminHome/tests/Integration/TasksTest.php b/plugins/CoreAdminHome/tests/Integration/TasksTest.php index a23d368de4..89c62252d5 100644 --- a/plugins/CoreAdminHome/tests/Integration/TasksTest.php +++ b/plugins/CoreAdminHome/tests/Integration/TasksTest.php @@ -132,6 +132,7 @@ class TasksTest extends IntegrationTestCase $tasks = array_map(function (Task $task) { return $task->getMethodName() . '.' . $task->getMethodParameter(); }, $tasks); $expected = [ + 'invalidateOutdatedArchives.', 'purgeOutdatedArchives.', 'purgeInvalidatedArchives.', 'purgeOrphanedArchives.', diff --git a/plugins/Monolog/Processor/ExceptionToTextProcessor.php b/plugins/Monolog/Processor/ExceptionToTextProcessor.php index ab828dc820..0089b27d94 100644 --- a/plugins/Monolog/Processor/ExceptionToTextProcessor.php +++ b/plugins/Monolog/Processor/ExceptionToTextProcessor.php @@ -85,6 +85,10 @@ class ExceptionToTextProcessor public static function getWholeBacktrace(\Exception $exception, $shouldPrintBacktrace = true) { + if (!$shouldPrintBacktrace) { + return $exception->getMessage(); + } + $message = ""; $e = $exception; diff --git a/plugins/PrivacyManager/tests/Integration/DataPurgingTest.php b/plugins/PrivacyManager/tests/Integration/DataPurgingTest.php index fd1bd4b823..a077bbb230 100644 --- a/plugins/PrivacyManager/tests/Integration/DataPurgingTest.php +++ b/plugins/PrivacyManager/tests/Integration/DataPurgingTest.php @@ -10,6 +10,7 @@ namespace Piwik\Plugins\PrivacyManager\tests\Integration; use Piwik\Archive; use Piwik\Common; use Piwik\Config; +use Piwik\Container\StaticContainer; use Piwik\DataAccess\RawLogDao; use Piwik\Date; use Piwik\Db; @@ -740,10 +741,14 @@ class DataPurgingTest extends IntegrationTestCase $range = $rangeStart->toString('Y-m-d') . "," . $rangeEnd->toString('Y-m-d'); $rangeArchive = Archive::build(self::$idSite, 'range', $range); - $rangeArchive->getNumeric('nb_visits', 'nb_hits'); + $rangeArchive->getNumeric(['nb_visits']); APIVisitorInterest::getInstance()->getNumberOfVisitsPerVisitDuration(self::$idSite, 'range', $range); + // remove invalidated + StaticContainer::get(Archive\ArchivePurger::class)->purgeInvalidatedArchivesFrom(Date::factory('2012-01-01')); + StaticContainer::get(Archive\ArchivePurger::class)->purgeInvalidatedArchivesFrom(Date::factory('2012-02-01')); + // when archiving is initiated, the archive metrics & reports for EVERY loaded plugin // are archived. don't want this test to depend on every possible metric, so get rid of // the unwanted archive data now. diff --git a/tests/PHPUnit/Integration/ArchiveProcessor/LoaderTest.php b/tests/PHPUnit/Integration/ArchiveProcessor/LoaderTest.php index e99d495931..e354a2b988 100644 --- a/tests/PHPUnit/Integration/ArchiveProcessor/LoaderTest.php +++ b/tests/PHPUnit/Integration/ArchiveProcessor/LoaderTest.php @@ -10,10 +10,12 @@ namespace Piwik\Tests\Integration\ArchiveProcessor; +use Piwik\Archive\ArchiveInvalidator; use Piwik\ArchiveProcessor\Parameters; use Piwik\ArchiveProcessor\Loader; use Piwik\Common; use Piwik\Config; +use Piwik\Container\StaticContainer; use Piwik\DataAccess\ArchiveTableCreator; use Piwik\DataAccess\ArchiveWriter; use Piwik\Date; @@ -31,6 +33,7 @@ class LoaderTest extends IntegrationTestCase parent::beforeTableDataCached(); Fixture::createWebsite('2012-02-03 00:00:00'); + Fixture::createWebsite('2012-02-03 00:00:00'); } public function test_loadExistingArchiveIdFromDb_returnsFalsesIfNoArchiveFound() @@ -40,7 +43,7 @@ class LoaderTest extends IntegrationTestCase $archiveInfo = $loader->loadExistingArchiveIdFromDb(); - $this->assertEquals([false, false, false], $archiveInfo); + $this->assertEquals([false, false, false, false], $archiveInfo); } /** @@ -55,12 +58,12 @@ class LoaderTest extends IntegrationTestCase $loader = new Loader($params); $archiveInfo = $loader->loadExistingArchiveIdFromDb(); - $this->assertNotEquals([false, false, false], $archiveInfo); + $this->assertNotEquals([false, false, false, false], $archiveInfo); Config::getInstance()->Debug[$configSetting] = 1; $archiveInfo = $loader->loadExistingArchiveIdFromDb(); - $this->assertEquals([false, false, false], $archiveInfo); + $this->assertEquals([false, false, false, false], $archiveInfo); } public function getTestDataForLoadExistingArchiveIdFromDbDebugConfig() @@ -82,7 +85,7 @@ class LoaderTest extends IntegrationTestCase $loader = new Loader($params); $archiveInfo = $loader->loadExistingArchiveIdFromDb(); - $this->assertEquals(['1', '10', '0'], $archiveInfo); + $this->assertEquals(['1', '10', '0', true], $archiveInfo); } public function test_loadExistingArchiveIdFromDb_returnsArchiveIfForACurrentPeriod_AndNewEnough() @@ -93,7 +96,7 @@ class LoaderTest extends IntegrationTestCase $loader = new Loader($params); $archiveInfo = $loader->loadExistingArchiveIdFromDb(); - $this->assertEquals(['1', '10', '0'], $archiveInfo); + $this->assertEquals(['1', '10', '0', true], $archiveInfo); } public function test_loadExistingArchiveIdFromDb_returnsNoArchiveIfForACurrentPeriod_AndNoneAreNewEnough() @@ -104,7 +107,103 @@ class LoaderTest extends IntegrationTestCase $loader = new Loader($params); $archiveInfo = $loader->loadExistingArchiveIdFromDb(); - $this->assertEquals([false, false, false], $archiveInfo); + $this->assertEquals([false, '10', '0', true], $archiveInfo); // visits are still returned as this was the original behavior + } + + /** + * @dataProvider getTestDataForGetReportsToInvalidate + */ + public function test_getReportsToInvalidate_returnsCorrectReportsToInvalidate($rememberedReports, $idSite, $period, $date, $segment, $expected) + { + $invalidator = StaticContainer::get(ArchiveInvalidator::class); + foreach ($rememberedReports as $entry) { + $invalidator->rememberToInvalidateArchivedReportsLater($entry['idSite'], Date::factory($entry['date'])); + } + + $params = new Parameters(new Site($idSite), Factory::build($period, $date), new Segment($segment, [$idSite])); + $loader = new Loader($params); + + $reportsToInvalidate = $loader->getReportsToInvalidate(); + $this->assertEquals($expected, $reportsToInvalidate); + } + + public function getTestDataForGetReportsToInvalidate() + { + return [ + // two dates for one site + [ + [ + ['idSite' => 1, 'date' => '2013-04-05'], + ['idSite' => 1, 'date' => '2013-03-05'], + ['idSite' => 2, 'date' => '2013-05-05'], + ], + 1, + 'day', + '2013-04-05', + '', + [ + '2013-04-05' => [1], + ], + ], + + // no dates for a site + [ + [ + ['idSite' => '', 'date' => '2013-04-05'], + ['idSite' => '', 'date' => '2013-04-06'], + ['idSite' => 2, 'date' => '2013-05-05'], + ], + 1, + 'day', + '2013-04-05', + 'browserCode==ff', + [], + ], + + // day period not within range + [ + [ + ['idSite' => 1, 'date' => '2014-03-04'], + ['idSite' => 1, 'date' => '2014-03-06'], + ], + 1, + 'day', + '2013-03-05', + '', + [], + ], + + // non-day periods + [ + [ + ['idSite' => 1, 'date' => '2014-03-01'], + ['idSite' => 1, 'date' => '2014-03-06'], + ['idSite' => 2, 'date' => '2014-03-01'], + ], + 1, + 'week', + '2014-03-01', + '', + [ + '2014-03-01' => [2, 1], + ], + ], + [ + [ + ['idSite' => 1, 'date' => '2014-02-01'], + ['idSite' => 1, 'date' => '2014-03-06'], + ['idSite' => 2, 'date' => '2014-03-05'], + ['idSite' => 2, 'date' => '2014-03-06'], + ], + 1, + 'month', + '2014-03-01', + '', + [ + '2014-03-06' => [2, 1], + ], + ], + ]; } private function insertArchive(Parameters $params, $tsArchived = null, $visits = 10) diff --git a/tests/PHPUnit/Integration/ArchiveTest.php b/tests/PHPUnit/Integration/ArchiveTest.php deleted file mode 100644 index bd2e2d8246..0000000000 --- a/tests/PHPUnit/Integration/ArchiveTest.php +++ /dev/null @@ -1,441 +0,0 @@ -<?php -/** - * Piwik - free/libre analytics platform - * - * @link https://matomo.org - * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later - */ -namespace Piwik\Tests\Integration; - -use Piwik\API\Proxy; -use Piwik\API\Request; -use Piwik\Archive as PiwikArchive; -use Piwik\ArchiveProcessor; -use Piwik\ArchiveProcessor\Parameters; -use Piwik\ArchiveProcessor\Rules; -use Piwik\Common; -use Piwik\Config; -use Piwik\DataAccess\ArchiveSelector; -use Piwik\DataAccess\ArchiveTableCreator; -use Piwik\DataAccess\ArchiveWriter; -use Piwik\DataAccess\LogAggregator; -use Piwik\Date; -use Piwik\Db; -use Piwik\Piwik; -use Piwik\Plugins\UserLanguage; -use Piwik\Segment; -use Piwik\Site; -use Piwik\Tests\Fixtures\OneVisitorTwoVisits; -use Piwik\Tests\Framework\Fixture; -use Piwik\Tests\Framework\TestCase\IntegrationTestCase; -use Piwik\Period\Factory as PeriodFactory; -use Piwik\Archive\Chunk; - -class Archive extends PiwikArchive -{ - public function get($archiveNames, $archiveDataType, $idSubtable = null) - { - return parent::get($archiveNames, $archiveDataType, $idSubtable); - } -} - -class CustomArchiveQueryFactory extends PiwikArchive\ArchiveQueryFactory -{ - public function newInstance(\Piwik\Archive\Parameters $params, $forceIndexedBySite, $forceIndexedByDate) - { - return new Archive($params, $forceIndexedBySite, $forceIndexedByDate); - } -} - -/** - * @group Core - */ -class ArchiveTest extends IntegrationTestCase -{ - /** - * @var OneVisitorTwoVisits - */ - public static $fixture; - - public function tearDown() - { - parent::tearDown(); - - unset($_GET['trigger']); - } - - protected static function configureFixture($fixture) - { - $fixture->createSuperUser = true; - } - - protected static function beforeTableDataCached() - { - $date = Date::factory('2010-03-01'); - - $archiveTableCreator = new ArchiveTableCreator(); - $archiveTableCreator->getBlobTable($date); - $archiveTableCreator->getNumericTable($date); - } - - public function getForceOptionsForForceArchivingOnBrowserRequest() - { - return array( - array(1), - array(null) - ); - } - - /** - * @dataProvider getForceOptionsForForceArchivingOnBrowserRequest - */ - public function test_ArchivingIsLaunchedForRanges_WhenForceOnBrowserRequest_IsTruthy($optionValue) - { - $this->archiveDataForIndividualDays(); - - Config::getInstance()->General['archiving_range_force_on_browser_request'] = $optionValue; - Rules::setBrowserTriggerArchiving(false); - - $data = $this->initiateArchivingForRange(); - - $this->assertNotEmpty($data); - $this->assertArchiveTablesAreNotEmpty('2010_03'); - } - - public function test_ArchivingIsNotLaunchedForRanges_WhenForceOnBrowserRequest_IsFalse() - { - $this->archiveDataForIndividualDays(); - - Config::getInstance()->General['archiving_range_force_on_browser_request'] = 0; - Rules::setBrowserTriggerArchiving(false); - - $data = $this->initiateArchivingForRange(); - - $this->assertEmpty($data); - $this->assertArchiveTablesAreEmpty('2010_03'); - } - - public function test_ArchiveIsLaunched_WhenForceOnBrowserRequest_IsFalse_AndArchivePhpTriggered() - { - $this->archiveDataForIndividualDays(); - - Config::getInstance()->General['archiving_range_force_on_browser_request'] = 0; - $_GET['trigger'] = 'archivephp'; - Rules::setBrowserTriggerArchiving(false); - - $data = $this->initiateArchivingForRange(); - - $this->assertNotEmpty($data); - $this->assertArchiveTablesAreNotEmpty('2010_03'); - } - - public function test_ArchiveBlob_ShouldBeAbleToLoadFirstLevelDataArrays() - { - $this->createManyDifferentArchiveBlobs(); - - $archive = $this->getArchive('day', '2013-01-01,2013-01-05'); - $dataArrays = $archive->get(array('Actions_Actionsurl'), 'blob'); - - $this->assertArchiveBlob($dataArrays, '2013-01-01', array('Actions_Actionsurl' => 'test01')); - $this->assertArchiveBlob($dataArrays, '2013-01-02', array('Actions_Actionsurl' => 'test02')); - $this->assertArchiveBlob($dataArrays, '2013-01-03', array('Actions_Actionsurl' => 'test03')); - $this->assertArchiveBlob($dataArrays, '2013-01-04', array('Actions_Actionsurl' => 'test04')); - $this->assertArchiveBlob($dataArrays, '2013-01-05', array('Actions_Actionsurl' => 0)); - } - - public function test_ArchiveBlob_ShouldBeAbleToLoadOneSubtable_NoMatterWhetherTheyAreStoredSeparatelyOrInACombinedSubtableEntry() - { - $this->createManyDifferentArchiveBlobs(); - - $archive = $this->getArchive('day', '2013-01-01,2013-01-05'); - $dataArrays = $archive->get(array('Actions_Actionsurl'), 'blob', 2); - - $this->assertArchiveBlob($dataArrays, '2013-01-01', array('Actions_Actionsurl_2' => 0)); - $this->assertArchiveBlob($dataArrays, '2013-01-02', array('Actions_Actionsurl_2' => 'test2')); - $this->assertArchiveBlob($dataArrays, '2013-01-03', array('Actions_Actionsurl_2' => 'subtable2')); - $this->assertArchiveBlob($dataArrays, '2013-01-04', array('Actions_Actionsurl_2' => 0)); - $this->assertArchiveBlob($dataArrays, '2013-01-05', array('Actions_Actionsurl_2' => 0)); - - // test another one - $dataArrays = $archive->get(array('Actions_Actionsurl'), 'blob', 5); - - $this->assertArchiveBlob($dataArrays, '2013-01-01', array('Actions_Actionsurl_5' => 0)); - $this->assertArchiveBlob($dataArrays, '2013-01-02', array('Actions_Actionsurl_5' => 0)); - $this->assertArchiveBlob($dataArrays, '2013-01-03', array('Actions_Actionsurl_5' => 'subtable5')); - $this->assertArchiveBlob($dataArrays, '2013-01-04', array('Actions_Actionsurl_5' => 'subtable45')); - $this->assertArchiveBlob($dataArrays, '2013-01-05', array('Actions_Actionsurl_5' => 0)); - - // test one that does not exist - $dataArrays = $archive->get(array('Actions_Actionsurl'), 'blob', 999); - - $this->assertArchiveBlob($dataArrays, '2013-01-01', array('Actions_Actionsurl_999' => 0)); - $this->assertArchiveBlob($dataArrays, '2013-01-02', array('Actions_Actionsurl_999' => 0)); - $this->assertArchiveBlob($dataArrays, '2013-01-03', array('Actions_Actionsurl_999' => 0)); - $this->assertArchiveBlob($dataArrays, '2013-01-04', array('Actions_Actionsurl_999' => 0)); - $this->assertArchiveBlob($dataArrays, '2013-01-05', array('Actions_Actionsurl_999' => 0)); - } - - public function test_ArchiveBlob_ShouldBeAbleToLoadAllSubtables_NoMatterWhetherTheyAreStoredSeparatelyOrInACombinedSubtableEntry() - { - $this->createManyDifferentArchiveBlobs(); - - $archive = $this->getArchive('day', '2013-01-01,2013-01-06'); - $dataArrays = $archive->get(array('Actions_Actionsurl'), 'blob', Archive::ID_SUBTABLE_LOAD_ALL_SUBTABLES); - - $this->assertArchiveBlob($dataArrays, '2013-01-01', array('Actions_Actionsurl' => 'test01')); - $this->assertArchiveBlob($dataArrays, '2013-01-02', array('Actions_Actionsurl' => 'test02', 'Actions_Actionsurl_1' => 'test1', 'Actions_Actionsurl_2' => 'test2')); - $this->assertArchiveBlob($dataArrays, '2013-01-03', array('Actions_Actionsurl' => 'test03', 'Actions_Actionsurl_1' => 'subtable1', 'Actions_Actionsurl_2' => 'subtable2', 'Actions_Actionsurl_5' => 'subtable5')); - $this->assertArchiveBlob($dataArrays, '2013-01-04', array('Actions_Actionsurl' => 'test04', 'Actions_Actionsurl_5' => 'subtable45', 'Actions_Actionsurl_6' => 'subtable6')); - $this->assertArchiveBlob($dataArrays, '2013-01-05', array('Actions_Actionsurl' => 0)); - $this->assertArchiveBlob($dataArrays, '2013-01-06', array('Actions_Actionsurl' => 'test06')); - } - - public function test_ArchiveBlob_ShouldBeAbleToLoadDifferentArchives_NoMatterWhetherTheyAreStoredSeparatelyOrInACombinedSubtableEntry() - { - $this->createManyDifferentArchiveBlobs(); - - $archive = $this->getArchive('day', '2013-01-01,2013-01-06'); - $dataArrays = $archive->get(array('Actions_Actionsurl', 'Actions_Actions'), 'blob', 2); - - $this->assertArchiveBlob($dataArrays, '2013-01-01', array('Actions_Actionsurl_2' => 0, 'Actions_Actions_2' => 0)); - $this->assertArchiveBlob($dataArrays, '2013-01-02', array('Actions_Actions_2' => 'actionsSubtable2', 'Actions_Actionsurl_2' => 'test2')); - $this->assertArchiveBlob($dataArrays, '2013-01-03', array('Actions_Actions_2' => 'actionsTest2', 'Actions_Actionsurl_2' => 'subtable2')); - $this->assertArchiveBlob($dataArrays, '2013-01-04', array('Actions_Actionsurl_2' => 0, 'Actions_Actions_2' => 0)); - $this->assertArchiveBlob($dataArrays, '2013-01-05', array('Actions_Actionsurl_2' => 0, 'Actions_Actions_2' => 0)); - $this->assertArchiveBlob($dataArrays, '2013-01-06', array('Actions_Actionsurl_2' => 0, 'Actions_Actions_2' => 0)); - } - - /** - * @dataProvider findBlobsWithinDifferentChunksDataProvider - */ - public function test_ArchiveBlob_ShouldBeFindBlobs_WithinDifferentChunks($idSubtable, $expectedBlob) - { - $recordName = 'Actions_Actions'; - - $chunk = new Chunk(); - $chunk5 = $chunk->getRecordNameForTableId($recordName, $subtableId = 5); - $chunk152 = $chunk->getRecordNameForTableId($recordName, $subtableId = 152); - $chunk399 = $chunk->getRecordNameForTableId($recordName, $subtableId = 399); - - $this->createArchiveBlobEntry('2013-01-02', array( - $recordName => 'actions_02', - $chunk5 => serialize(array(1 => 'actionsSubtable1', 2 => 'actionsSubtable2', 5 => 'actionsSubtable5')), - $chunk152 => serialize(array(151 => 'actionsSubtable151', 152 => 'actionsSubtable152')), - $chunk399 => serialize(array(399 => 'actionsSubtable399')) - )); - - $archive = $this->getArchive('day', '2013-01-02,2013-01-02'); - - $dataArrays = $archive->get(array('Actions_Actions'), 'blob', $idSubtable); - $this->assertArchiveBlob($dataArrays, '2013-01-02', $expectedBlob); - } - - public function findBlobsWithinDifferentChunksDataProvider() - { - return array( - array($idSubtable = 2, $expectedBlobs = array('Actions_Actions_2' => 'actionsSubtable2')), - array(5, array('Actions_Actions_5' => 'actionsSubtable5')), - array(151, array('Actions_Actions_151' => 'actionsSubtable151')), - array(152, array('Actions_Actions_152' => 'actionsSubtable152')), - array(399, array('Actions_Actions_399' => 'actionsSubtable399')), - // this one does not exist - array(404, array('Actions_Actions_404' => 0)), - ); - } - - public function testExistingArchivesAreReplaced() - { - $date = self::$fixture->dateTime; - $period = PeriodFactory::makePeriodFromQueryParams('UTC', 'day', $date); - - // request an report to trigger archiving - $userLanguageReport = Proxy::getInstance()->call('\\Piwik\\Plugins\\UserLanguage\\API', 'getLanguage', array( - 'idSite' => 1, - 'period' => 'day', - 'date' => $date - )); - - $this->assertEquals(1, $userLanguageReport->getRowsCount()); - $this->assertEquals('UserLanguage_LanguageCode fr', $userLanguageReport->getFirstRow()->getColumn('label')); - $this->assertEquals('UserLanguage_LanguageCode fr', $userLanguageReport->getLastRow()->getColumn('label')); - - $parameters = new Parameters(new Site(1), $period, new Segment('', [])); - $parameters->setRequestedPlugin('UserLanguage'); - - $result = ArchiveSelector::getArchiveIdAndVisits($parameters, $period->getDateStart()->getDateStartUTC()); - $idArchive = $result ? array_shift($result) : null; - - if (empty($idArchive)) { - $this->fail('Archive should be available'); - } - - // track a new visits now - $t = Fixture::getTracker(1, $date, $defaultInit = true); - $t->setForceVisitDateTime(Date::factory($date)->addHour(1)->getDatetime()); - $t->setUrl('http://site.com/index.htm'); - $t->setBrowserLanguage('pt-br'); - Fixture::checkResponse($t->doTrackPageView('my_site')); - - $archiveWriter = new ArchiveWriter($parameters, !!$idArchive); - $archiveWriter->idArchive = $idArchive; - - $archiveProcessor = new ArchiveProcessor($parameters, $archiveWriter, - new LogAggregator($parameters)); - - $archiveProcessor->setNumberOfVisits(1, 1); - - // directly trigger specific archiver for existing archive - $archiver = new UserLanguage\Archiver($archiveProcessor); - $archiver->aggregateDayReport(); - $archiveWriter->finalizeArchive(); - - // report should be updated - $userLanguageReport = Proxy::getInstance()->call('\\Piwik\\Plugins\\UserLanguage\\API', 'getLanguage', array( - 'idSite' => 1, - 'period' => 'day', - 'date' => $date - )); - - $this->assertEquals(2, $userLanguageReport->getRowsCount()); - $this->assertEquals('UserLanguage_LanguageCode fr', $userLanguageReport->getFirstRow()->getColumn('label')); - $this->assertEquals('UserLanguage_LanguageCode pt', $userLanguageReport->getLastRow()->getColumn('label')); - } - - public function testNoFutureArchiveTablesAreCreatedWithoutArchiving() - { - $dateTime = '2066-01-01'; - Config::getInstance()->General['enable_browser_archiving_triggering'] = 0; - - // request API which should not trigger archiving due to config and shouldn't create any archive tables - Request::processRequest('VisitsSummary.get', array('idSite' => 1, 'period' => 'day', 'date' => $dateTime)); - - $numericTables = Db::get()->fetchAll('SHOW TABLES like "%archive_numeric_2066_%"'); - - $this->assertEmpty($numericTables, 'Archive table for future date found'); - } - - public function testNoFutureArchiveTablesAreCreatedWithArchiving() - { - $dateTime = '2066-01-01'; - Config::getInstance()->General['enable_browser_archiving_triggering'] = 1; - - // request API which should not trigger archiving due to config and shouldn't create any archive tables - Request::processRequest('VisitsSummary.get', array('idSite' => 1, 'period' => 'day', 'date' => $dateTime)); - - $numericTables = Db::get()->fetchAll('SHOW TABLES like "%archive_numeric_2066_%"'); - - $this->assertEmpty($numericTables, 'Archive table for future date found'); - } - - private function createManyDifferentArchiveBlobs() - { - $recordName1 = 'Actions_Actions'; - $recordName2 = 'Actions_Actionsurl'; - - $chunk = new Chunk(); - $chunk0_1 = $chunk->getRecordNameForTableId($recordName1, 0); - $chunk0_2 = $chunk->getRecordNameForTableId($recordName2, 0); - - $this->createArchiveBlobEntry('2013-01-01', array( - $recordName2 => 'test01' - )); - $this->createArchiveBlobEntry('2013-01-02', array( - $recordName2 => 'test02', - $recordName2 . '_1' => 'test1', // testing BC where each subtable was stored seperately - $recordName2 . '_2' => 'test2', // testing BC - $recordName1 => 'actions_02', - $chunk0_1 => serialize(array(1 => 'actionsSubtable1', 2 => 'actionsSubtable2', 5 => 'actionsSubtable5')) - )); - $this->createArchiveBlobEntry('2013-01-03', array( - $recordName2 => 'test03', - $chunk0_2 => serialize(array(1 => 'subtable1', 2 => 'subtable2', 5 => 'subtable5')), - $recordName1 => 'actions_03', - $recordName1 . '_1' => 'actionsTest1', - $recordName1 . '_2' => 'actionsTest2' - )); - $this->createArchiveBlobEntry('2013-01-04', array( - $recordName2 => 'test04', - $recordName2 . '_5' => 'subtable45', - $recordName2 . '_6' => 'subtable6' - )); - $this->createArchiveBlobEntry('2013-01-06', array( - $recordName2 => 'test06', - $chunk0_2 => serialize(array()) - )); - } - - private function assertArchiveBlob(PiwikArchive\DataCollection $dataCollection, $date, $expectedBlob) - { - $dateIndex = $date . ',' . $date; - $dataArrays = $dataCollection->get(1, $dateIndex); - - if (!empty($expectedBlob) && 0 !== reset($expectedBlob)) { - $this->assertNotEmpty($dataArrays['_metadata']['ts_archived']); - $dataArrays['_metadata']['ts_archived'] = true; - unset($dataArrays['_metadata']); - } - - $this->assertEquals($expectedBlob, $dataArrays); - } - - private function createArchiveBlobEntry($date, $blobs) - { - $oPeriod = PeriodFactory::makePeriodFromQueryParams('UTC', 'day', $date); - - $segment = new Segment(false, array(1)); - $params = new Parameters(new Site(1), $oPeriod, $segment); - $writer = new ArchiveWriter($params, false); - $writer->initNewArchive(); - foreach ($blobs as $name => $blob) { - $writer->insertBlobRecord($name, $blob); - } - $writer->finalizeArchive(); - } - - private function assertArchiveTablesAreNotEmpty($tableMonth) - { - $this->assertNotEquals(0, $this->getRangeArchiveTableCount('archive_numeric', $tableMonth)); - } - - private function assertArchiveTablesAreEmpty($tableMonth) - { - $this->assertEquals(0, $this->getRangeArchiveTableCount('archive_numeric', $tableMonth)); - $this->assertEquals(0, $this->getRangeArchiveTableCount('archive_blob', $tableMonth)); - } - - private function getRangeArchiveTableCount($tableType, $tableMonth) - { - $table = Common::prefixTable($tableType . '_' . $tableMonth); - return Db::fetchOne("SELECT COUNT(*) FROM $table WHERE period = " . Piwik::$idPeriods['range']); - } - - private function initiateArchivingForRange() - { - $archive = $this->getArchive('range'); - return $archive->getNumeric('nb_visits'); - } - - private function archiveDataForIndividualDays() - { - $archive = $this->getArchive('day'); - return $archive->getNumeric('nb_visits'); - } - - /** - * @return Archive - */ - private function getArchive($period, $day = '2010-03-04,2010-03-07') - { - /** @noinspection PhpIncompatibleReturnTypeInspection */ - return Archive::build(self::$fixture->idSite, $period, $day); - } - - public function provideContainerConfig() - { - return [ - PiwikArchive\ArchiveQueryFactory::class => \DI\object(CustomArchiveQueryFactory::class), - ]; - } -} - -ArchiveTest::$fixture = new OneVisitorTwoVisits();
\ No newline at end of file diff --git a/tests/PHPUnit/Integration/CronArchiveTest.php b/tests/PHPUnit/Integration/CronArchiveTest.php index fa568d6b9b..a9b9ee7b93 100644 --- a/tests/PHPUnit/Integration/CronArchiveTest.php +++ b/tests/PHPUnit/Integration/CronArchiveTest.php @@ -10,7 +10,9 @@ namespace Piwik\Tests\Integration; use Piwik\Container\StaticContainer; use Piwik\CronArchive; +use Piwik\DataAccess\ArchiveTableCreator; use Piwik\Date; +use Piwik\Db; use Piwik\Plugins\CoreAdminHome\tests\Framework\Mock\API; use Piwik\Plugins\SegmentEditor\Model; use Piwik\Tests\Framework\Fixture; @@ -104,7 +106,7 @@ class CronArchiveTest extends IntegrationTestCase \Piwik\Tests\Framework\Mock\FakeCliMulti::$specifiedResults = array( '/method=API.get/' => serialize(array(array('nb_visits' => 1))) ); - + Fixture::createWebsite('2014-12-12 00:01:02'); SegmentAPI::getInstance()->add('foo', 'actions>=1', 1, true, true); $id = SegmentAPI::getInstance()->add('barb', 'actions>=2', 1, true, true); @@ -230,12 +232,198 @@ LOG; $this->assertContains($expected, $logger->output); } + /** + * @dataProvider getTestDataForIsThereAValidArchiveForPeriod + */ + public function test_isThereAValidArchiveForPeriod_modifiesLastNCorrectly($archiveRows, $idSite, $period, $date, $expected) + { + Date::$now = strtotime('2015-03-04 05:05:05'); + + Fixture::createWebsite('2014-12-12 00:01:02'); + + $this->insertArchiveData($archiveRows); + + $cronArchive = new CronArchive(); + $result = $cronArchive->isThereAValidArchiveForPeriod($idSite, $period, $date); + + $this->assertEquals($expected, $result); + } + + public function getTestDataForIsThereAValidArchiveForPeriod() + { + return [ + // single periods that include today (we don't perform the check and just archive since we assume today is invalid) + [ + [ + ['idarchive' => 1, 'idsite' => 1, 'period' => 1, 'date1' => '2015-03-04', 'date2' => '2015-03-04', 'name' => 'done', 'value' => 1], + ], + 1, + 'day', + '2015-03-04', + [false, null], + ], + [ + [], + 1, + 'week', + '2015-03-04', + [false, null], + ], + [ + [], + 1, + 'month', + '2015-03-04', + [false, null], + ], + [ + [], + 1, + 'year', + '2015-03-04', + [false, null], + ], + + // single periods that do not include today + [ + [ + ['idarchive' => 1, 'idsite' => 1, 'period' => 1, 'date1' => '2015-03-02', 'date2' => '2015-03-02', 'name' => 'done', 'value' => 1], + ], + 1, + 'day', + '2015-03-02', + [true, '2015-03-02'], + ], + [ + [ + ['idarchive' => 1, 'idsite' => 1, 'period' => 1, 'date1' => '2015-03-01', 'date2' => '2015-03-01', 'name' => 'done', 'value' => 1], + ], + 1, + 'day', + '2015-03-02', + [false, '2015-03-02'], + ], + [ + [ + ['idarchive' => 1, 'idsite' => 1, 'period' => 2, 'date1' => '2015-02-16', 'date2' => '2015-02-22', 'name' => 'done', 'value' => 1], + ], + 1, + 'week', + '2015-02-17', + [true, '2015-02-17'], + ], + [ + [ + ['idarchive' => 1, 'idsite' => 1, 'period' => 2, 'date1' => '2015-02-15', 'date2' => '2015-02-21', 'name' => 'done', 'value' => 4], + ], + 1, + 'week', + '2015-02-17', + [false, '2015-02-17'], + ], + + // lastN periods + [ // last day invalid, some valid in between + [ + ['idarchive' => 1, 'idsite' => 1, 'period' => 1, 'date1' => '2015-03-04', 'date2' => '2015-03-04', 'name' => 'done', 'value' => 4], + ['idarchive' => 2, 'idsite' => 1, 'period' => 1, 'date1' => '2015-03-03', 'date2' => '2015-03-04', 'name' => 'done', 'value' => 4], + ['idarchive' => 3, 'idsite' => 1, 'period' => 1, 'date1' => '2015-03-02', 'date2' => '2015-03-04', 'name' => 'done', 'value' => 1], + ['idarchive' => 4, 'idsite' => 1, 'period' => 1, 'date1' => '2015-03-01', 'date2' => '2015-03-04', 'name' => 'done', 'value' => 1], + ['idarchive' => 5, 'idsite' => 1, 'period' => 1, 'date1' => '2015-02-28', 'date2' => '2015-03-04', 'name' => 'done', 'value' => 4], + ], + 1, + 'day', + 'last5', + [false, 'last5'], + ], + [ // last two invalid, rest valid + [ + ['idarchive' => 1, 'idsite' => 1, 'period' => 1, 'date1' => '2015-03-04', 'date2' => '2015-03-04', 'name' => 'done', 'value' => 4], + ['idarchive' => 2, 'idsite' => 1, 'period' => 1, 'date1' => '2015-03-03', 'date2' => '2015-03-03', 'name' => 'done', 'value' => 4], + ['idarchive' => 3, 'idsite' => 1, 'period' => 1, 'date1' => '2015-03-02', 'date2' => '2015-03-02', 'name' => 'done', 'value' => 1], + ['idarchive' => 4, 'idsite' => 1, 'period' => 1, 'date1' => '2015-03-01', 'date2' => '2015-03-01', 'name' => 'done', 'value' => 1], + ['idarchive' => 5, 'idsite' => 1, 'period' => 1, 'date1' => '2015-02-28', 'date2' => '2015-02-28', 'name' => 'done', 'value' => 1], + ], + 1, + 'day', + 'last5', + [false, 'last2'], + ], + [ // all valid + [ + ['idarchive' => 1, 'idsite' => 1, 'period' => 1, 'date1' => '2015-03-04', 'date2' => '2015-03-04', 'name' => 'done', 'value' => 1], + ['idarchive' => 2, 'idsite' => 1, 'period' => 1, 'date1' => '2015-03-03', 'date2' => '2015-03-03', 'name' => 'done', 'value' => 1], + ['idarchive' => 3, 'idsite' => 1, 'period' => 1, 'date1' => '2015-03-02', 'date2' => '2015-03-02', 'name' => 'done', 'value' => 1], + ['idarchive' => 4, 'idsite' => 1, 'period' => 1, 'date1' => '2015-03-01', 'date2' => '2015-03-01', 'name' => 'done', 'value' => 1], + ['idarchive' => 5, 'idsite' => 1, 'period' => 1, 'date1' => '2015-02-28', 'date2' => '2015-02-28', 'name' => 'done', 'value' => 1], + ], + 1, + 'day', + 'last5', + [false, 'last2'], + ], + [ // month w/ last 3 invalid, today valid + [ + ['idarchive' => 1, 'idsite' => 1, 'period' => 3, 'date1' => '2015-03-01', 'date2' => '2015-03-31', 'name' => 'done', 'value' => 1], + ['idarchive' => 2, 'idsite' => 1, 'period' => 3, 'date1' => '2015-02-01', 'date2' => '2015-02-28', 'name' => 'done', 'value' => 4], + ['idarchive' => 3, 'idsite' => 1, 'period' => 3, 'date1' => '2015-01-01', 'date2' => '2015-01-31', 'name' => 'done', 'value' => 4], + ['idarchive' => 4, 'idsite' => 1, 'period' => 3, 'date1' => '2014-12-01', 'date2' => '2014-12-31', 'name' => 'done', 'value' => 1], + ['idarchive' => 5, 'idsite' => 1, 'period' => 3, 'date1' => '2014-11-01', 'date2' => '2014-11-30', 'name' => 'done', 'value' => 1], + ], + 1, + 'month', + 'last5', + [false, 'last3'], + ], + + // range periods + [ // includes today + [ + ['idarchive' => 5, 'idsite' => 1, 'period' => 5, 'date1' => '2015-03-02', 'date2' => '2015-03-04', 'name' => 'done', 'value' => 1], + ], + 1, + 'range', + '2015-03-02,2015-03-04', + [false, null], + ], + [ // does not include today, invalid + [ + ['idarchive' => 1, 'idsite' => 1, 'period' => 5, 'date1' => '2015-03-01', 'date2' => '2015-03-03', 'name' => 'done', 'value' => 4], + ], + 1, + 'range', + '2015-03-01,2015-03-03', + [false, '2015-03-01,2015-03-03'], + ], + [ // does not include today, valid + [ + ['idarchive' => 1, 'idsite' => 1, 'period' => 5, 'date1' => '2015-03-01', 'date2' => '2015-03-03', 'name' => 'done', 'value' => 1], + ], + 1, + 'range', + '2015-03-01,2015-03-03', + [true, '2015-03-01,2015-03-03'], + ], + ]; + } + public function provideContainerConfig() { return array( 'Piwik\CliMulti' => \DI\object('Piwik\Tests\Framework\Mock\FakeCliMulti') ); } + + private function insertArchiveData($archiveRows) + { + foreach ($archiveRows as $row) { + $table = ArchiveTableCreator::getNumericTable(Date::factory($row['date1'])); + + $tsArchived = isset($row['ts_archived']) ? $row['ts_archived'] : Date::now()->getDatetime(); + Db::query("INSERT INTO `$table` (idarchive, idsite, period, date1, date2, `name`, `value`, ts_archived) VALUES (?, ?, ?, ?, ?, ?, ?, ?)", + [$row['idarchive'], $row['idsite'], $row['period'], $row['date1'], $row['date2'], $row['name'], $row['value'], $tsArchived]); + } + } } class TestCronArchive extends CronArchive diff --git a/tests/PHPUnit/Integration/DataAccess/ArchiveSelectorTest.php b/tests/PHPUnit/Integration/DataAccess/ArchiveSelectorTest.php new file mode 100644 index 0000000000..6143534a87 --- /dev/null +++ b/tests/PHPUnit/Integration/DataAccess/ArchiveSelectorTest.php @@ -0,0 +1,212 @@ +<?php +/** + * Piwik - free/libre analytics platform + * + * @link https://matomo.org + * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later + * + */ + +namespace Piwik\Tests\Integration\DataAccess; + + +use Piwik\ArchiveProcessor\Parameters; +use Piwik\DataAccess\ArchiveSelector; +use Piwik\DataAccess\ArchiveTableCreator; +use Piwik\Date; +use Piwik\Db; +use Piwik\Period\Factory; +use Piwik\Segment; +use Piwik\Site; +use Piwik\Tests\Framework\Fixture; +use Piwik\Tests\Framework\TestCase\IntegrationTestCase; + +class ArchiveSelectorTest extends IntegrationTestCase +{ + protected static function configureFixture($fixture) + { + parent::configureFixture($fixture); + $fixture->createSuperUser = true; + } + + /** + * @dataProvider getTestDataForGetArchiveIdAndVisits + */ + public function test_getArchiveIdAndVisits_returnsCorrectResult($archiveRows, $segment, $minDateProcessed, $includeInvalidated, $expected) + { + Fixture::createWebsite('2010-02-02 00:00:00'); + + $this->insertArchiveData($archiveRows); + + $params = new Parameters(new Site(1), Factory::build('day', '2019-10-05'), new Segment($segment, [1])); + $result = ArchiveSelector::getArchiveIdAndVisits($params, $minDateProcessed, $includeInvalidated); + $this->assertEquals($expected, $result); + } + + private function insertArchiveData($archiveRows) + { + $table = ArchiveTableCreator::getNumericTable(Date::factory('2019-10-01 12:13:14')); + foreach ($archiveRows as $row) { + $tsArchived = isset($row['ts_archived']) ? $row['ts_archived'] : Date::now()->getDatetime(); + Db::query("INSERT INTO `$table` (idarchive, idsite, period, date1, date2, `name`, `value`, ts_archived) VALUES (?, ?, ?, ?, ?, ?, ?, ?)", + [$row['idarchive'], $row['idsite'], $row['period'], $row['date1'], $row['date2'], $row['name'], $row['value'], $tsArchived]); + } + } + + public function getTestDataForGetArchiveIdAndVisits() + { + $minDateProcessed = Date::factory('2020-03-04 00:00:00')->subSeconds(900)->getDatetime(); + return [ + // no archive data found + [ // nothing in the db + [], + '', + $minDateProcessed, + true, + [false, false, false, false], + ], + [ + [ + ['idarchive' => 1, 'idsite' => 1, 'period' => 2, 'date1' => '2019-10-05', 'date2' => '2019-10-05', 'name' => 'done', 'value' => 1], + ['idarchive' => 2, 'idsite' => 1, 'period' => 1, 'date1' => '2019-10-06', 'date2' => '2019-10-06', 'name' => 'done', 'value' => 1], + ['idarchive' => 3, 'idsite' => 2, 'period' => 1, 'date1' => '2019-10-05', 'date2' => '2019-10-05', 'name' => 'done', 'value' => 1], + ], + '', + $minDateProcessed, + true, + [false, false, false, false], + ], + + // value is not valid + [ + [ + ['idarchive' => 1, 'idsite' => 1, 'period' => 1, 'date1' => '2019-10-05', 'date2' => '2019-10-05', 'name' => 'done', 'value' => 4], + ['idarchive' => 2, 'idsite' => 1, 'period' => 1, 'date1' => '2019-10-05', 'date2' => '2019-10-05', 'name' => 'done', 'value' => 2], + ['idarchive' => 3, 'idsite' => 1, 'period' => 1, 'date1' => '2019-10-05', 'date2' => '2019-10-05', 'name' => 'done', 'value' => 3], + ['idarchive' => 4, 'idsite' => 1, 'period' => 1, 'date1' => '2019-10-05', 'date2' => '2019-10-05', 'name' => 'done', 'value' => 99], + ], + '', + $minDateProcessed, + false, + [false, false, false, true], + ], + [ + [ + ['idarchive' => 1, 'idsite' => 1, 'period' => 1, 'date1' => '2019-10-05', 'date2' => '2019-10-05', 'name' => 'done', 'value' => 4], + ['idarchive' => 1, 'idsite' => 1, 'period' => 1, 'date1' => '2019-10-05', 'date2' => '2019-10-05', 'name' => 'nb_visits', 'value' => 20], + ['idarchive' => 1, 'idsite' => 1, 'period' => 1, 'date1' => '2019-10-05', 'date2' => '2019-10-05', 'name' => 'nb_visits_converted', 'value' => 40], + ['idarchive' => 2, 'idsite' => 1, 'period' => 1, 'date1' => '2019-10-05', 'date2' => '2019-10-05', 'name' => 'done', 'value' => 2], + ['idarchive' => 3, 'idsite' => 1, 'period' => 1, 'date1' => '2019-10-05', 'date2' => '2019-10-05', 'name' => 'done', 'value' => 3], + ['idarchive' => 4, 'idsite' => 1, 'period' => 1, 'date1' => '2019-10-05', 'date2' => '2019-10-05', 'name' => 'done', 'value' => 99], + ], + '', + $minDateProcessed, + false, + [false, 0, 0, true], + ], + [ + [ + ['idarchive' => 1, 'idsite' => 1, 'period' => 1, 'date1' => '2019-10-05', 'date2' => '2019-10-05', 'name' => 'done.VisitsSummary', 'value' => 4], + ['idarchive' => 1, 'idsite' => 1, 'period' => 1, 'date1' => '2019-10-05', 'date2' => '2019-10-05', 'name' => 'nb_visits', 'value' => 20], + ['idarchive' => 1, 'idsite' => 1, 'period' => 1, 'date1' => '2019-10-05', 'date2' => '2019-10-05', 'name' => 'nb_visits_converted', 'value' => 40], + ], + '', + $minDateProcessed, + false, + [false, 20, 40, true], + ], + [ + [ + ['idarchive' => 1, 'idsite' => 1, 'period' => 1, 'date1' => '2019-10-05', 'date2' => '2019-10-05', 'name' => 'done.VisitsSummary', 'value' => 1], + ['idarchive' => 1, 'idsite' => 1, 'period' => 1, 'date1' => '2019-10-05', 'date2' => '2019-10-05', 'name' => 'nb_visits', 'value' => 30], + ['idarchive' => 1, 'idsite' => 1, 'period' => 1, 'date1' => '2019-10-05', 'date2' => '2019-10-05', 'name' => 'nb_visits_converted', 'value' => 50], + ], + '', + $minDateProcessed, + false, + [false, 30, 50, true], + ], + [ + [ + ['idarchive' => 1, 'idsite' => 1, 'period' => 1, 'date1' => '2019-10-05', 'date2' => '2019-10-05', 'name' => 'done.VisitsSummary', 'value' => 1], + ['idarchive' => 1, 'idsite' => 1, 'period' => 1, 'date1' => '2019-10-05', 'date2' => '2019-10-05', 'name' => 'nb_visits', 'value' => 30], + ['idarchive' => 1, 'idsite' => 1, 'period' => 1, 'date1' => '2019-10-05', 'date2' => '2019-10-05', 'name' => 'nb_visits_converted', 'value' => 50], + ['idarchive' => 2, 'idsite' => 1, 'period' => 1, 'date1' => '2019-10-05', 'date2' => '2019-10-05', 'name' => 'done.VisitsSummary', 'value' => 4], + ], + '', + $minDateProcessed, + false, + [false, 0, 0, true], + ], + + // archive is too old + [ + [ + ['idarchive' => 1, 'idsite' => 1, 'period' => 1, 'date1' => '2019-10-05', 'date2' => '2019-10-05', 'name' => 'done', 'value' => 1, + 'ts_archived' => Date::factory($minDateProcessed)->subSeconds(1)->getDatetime()], + ], + '', + $minDateProcessed, + false, + [false, false, false, true], + ], + [ + [ + ['idarchive' => 1, 'idsite' => 1, 'period' => 1, 'date1' => '2019-10-05', 'date2' => '2019-10-05', 'name' => 'done', 'value' => 1, + 'ts_archived' => Date::factory($minDateProcessed)->subSeconds(1)->getDatetime()], + ['idarchive' => 1, 'idsite' => 1, 'period' => 1, 'date1' => '2019-10-05', 'date2' => '2019-10-05', 'name' => 'nb_visits', 'value' => 1, + 'ts_archived' => Date::factory($minDateProcessed)->subSeconds(1)->getDatetime()], + ], + '', + $minDateProcessed, + false, + [false, 1, false, true], + ], + + // no archive done flags, but metric + [ + [ + ['idarchive' => 1, 'idsite' => 1, 'period' => 1, 'date1' => '2019-10-05', 'date2' => '2019-10-05', 'name' => 'nb_visits_converted', 'value' => 10], + ['idarchive' => 1, 'idsite' => 1, 'period' => 1, 'date1' => '2019-10-05', 'date2' => '2019-10-05', 'name' => 'nb_visits', 'value' => 1], + ], + '', + $minDateProcessed, + false, + [false, false, false, true], + ], + [ + [ + ['idarchive' => 1, 'idsite' => 1, 'period' => 1, 'date1' => '2019-10-05', 'date2' => '2019-10-05', 'name' => 'nb_visits_converted', 'value' => 10], + ['idarchive' => 1, 'idsite' => 1, 'period' => 1, 'date1' => '2019-10-05', 'date2' => '2019-10-05', 'name' => 'nb_visits', 'value' => 3], + ['idarchive' => 2, 'idsite' => 1, 'period' => 1, 'date1' => '2019-10-05', 'date2' => '2019-10-05', 'name' => 'nb_visits', 'value' => 5], + ], + '', + $minDateProcessed, + false, + [false, false, false, true], + ], + + // archive exists and is usable + [ + [ + ['idarchive' => 1, 'idsite' => 1, 'period' => 1, 'date1' => '2019-10-05', 'date2' => '2019-10-05', 'name' => 'done', 'value' => 1], + ], + '', + $minDateProcessed, + false, + [1, false, false, true], + ], + [ + [ + ['idarchive' => 1, 'idsite' => 1, 'period' => 1, 'date1' => '2019-10-05', 'date2' => '2019-10-05', 'name' => 'done', 'value' => 1], + ['idarchive' => 1, 'idsite' => 1, 'period' => 1, 'date1' => '2019-10-05', 'date2' => '2019-10-05', 'name' => 'nb_visits', 'value' => 5], + ['idarchive' => 1, 'idsite' => 1, 'period' => 1, 'date1' => '2019-10-05', 'date2' => '2019-10-05', 'name' => 'nb_visits_converted', 'value' => 10], + ], + '', + $minDateProcessed, + false, + [1, 5, 10, true], + ], + ]; + } +}
\ No newline at end of file diff --git a/tests/PHPUnit/Integration/Tracker/VisitTest.php b/tests/PHPUnit/Integration/Tracker/VisitTest.php index 0ccc7f4953..ccd73ce15f 100644 --- a/tests/PHPUnit/Integration/Tracker/VisitTest.php +++ b/tests/PHPUnit/Integration/Tracker/VisitTest.php @@ -392,7 +392,7 @@ class VisitTest extends IntegrationTestCase $currentActionTime = Date::today()->getDatetime(); $idsite = API::getInstance()->addSite('name', 'http://piwik.net/'); - $expectedRemembered = array(Date::today()->toString() => [1]); + $expectedRemembered = array(); $this->assertRememberedArchivedReportsThatShouldBeInvalidated($idsite, $currentActionTime, $expectedRemembered); } @@ -414,7 +414,6 @@ class VisitTest extends IntegrationTestCase // The double-handling below is needed to work around weird behaviour when UTC and UTC+5 are different dates // Example: 4:32am on 1 April in UTC+5 is 11:32pm on 31 March in UTC $midnight = Date::factoryInTimezone('today', 'UTC+5')->setTimezone('UTC+5'); - $today = Date::factoryInTimezone('today', 'UTC+5'); $oneHourAfterMidnight = $midnight->addHour(1)->getDatetime(); $oneHourBeforeMidnight = $midnight->subHour(1)->getDatetime(); @@ -428,11 +427,10 @@ class VisitTest extends IntegrationTestCase $expectedRemembered = array( substr($oneHourAfterMidnight, 0, 10) => array($idsite), - $today->toString() => [$idsite], ); // if website timezone was von considered both would be today (expected = array()) - $this->assertRememberedArchivedReportsThatShouldBeInvalidated($idsite, $oneHourAfterMidnight, array($today->toString() => [$idsite])); + $this->assertRememberedArchivedReportsThatShouldBeInvalidated($idsite, $oneHourAfterMidnight, array()); $this->assertRememberedArchivedReportsThatShouldBeInvalidated($idsite, $oneHourBeforeMidnight, $expectedRemembered); } diff --git a/tests/PHPUnit/Integration/TrackerTest.php b/tests/PHPUnit/Integration/TrackerTest.php index 574aa3312c..94f2feefa0 100644 --- a/tests/PHPUnit/Integration/TrackerTest.php +++ b/tests/PHPUnit/Integration/TrackerTest.php @@ -369,7 +369,7 @@ class TrackerTest extends IntegrationTestCase public function test_archiveInvalidation_differentServerAndWebsiteTimezones() { // Server timezone is UTC - ini_set('date.timezone', 'America/New_York'); + ini_set('date.timezone', 'UTC'); // Website timezone is New York $idSite = Fixture::createWebsite('2014-01-01 00:00:00', 0, false, false, @@ -385,12 +385,8 @@ class TrackerTest extends IntegrationTestCase $this->request->setCurrentTimestamp(Date::$now); $this->tracker->trackRequest($this->request); - $keys = Option::getLike('%report_to_invalidate_2_2019-04-02%'); - $this->assertCount(1, $keys); - $key = key($keys); - $this->assertStringEndsWith('report_to_invalidate_2_2019-04-02_' . getmypid(), $key); - // make sure today archives are also invalidated - $this->assertEquals([$key => '1'], $keys); + // make sure today archives are not invalidated + $this->assertEquals([], Option::getLike('report_to_invalidate_2_2019-04-02%')); } public function test_TrackingNewVisitOfKnownVisitor() diff --git a/tests/PHPUnit/System/OneVisitorOneWebsiteSeveralDaysDateRangeArchivingTest.php b/tests/PHPUnit/System/OneVisitorOneWebsiteSeveralDaysDateRangeArchivingTest.php index 2a005ab980..9268944755 100644 --- a/tests/PHPUnit/System/OneVisitorOneWebsiteSeveralDaysDateRangeArchivingTest.php +++ b/tests/PHPUnit/System/OneVisitorOneWebsiteSeveralDaysDateRangeArchivingTest.php @@ -7,8 +7,11 @@ */ namespace Piwik\Tests\System; +use Piwik\Archive\ArchivePurger; use Piwik\Archive\Chunk; use Piwik\Common; +use Piwik\Container\StaticContainer; +use Piwik\Date; use Piwik\Db; use Piwik\Piwik; use Piwik\Tests\Framework\TestCase\SystemTestCase; @@ -23,6 +26,9 @@ use Piwik\Tests\Fixtures\VisitsOverSeveralDays; */ class OneVisitorOneWebsiteSeveralDaysDateRangeArchivingTest extends SystemTestCase { + /** + * @var VisitsOverSeveralDays + */ public static $fixture = null; // initialized below test definition public static function getOutputPrefix() @@ -101,6 +107,11 @@ class OneVisitorOneWebsiteSeveralDaysDateRangeArchivingTest extends SystemTestCa */ public function test_checkArchiveRecords_whenPeriodIsRange() { + $archivePurger = StaticContainer::get(ArchivePurger::class); + foreach (self::$fixture->dateTimes as $date) { + $archivePurger->purgeInvalidatedArchivesFrom(Date::factory($date)); + } + // we expect 5 blobs for Actions plugins, because flat=1 or expanded=1 was not set // so we only archived the parent table $expectedActionsBlobs = 5; diff --git a/tests/PHPUnit/System/OneVisitorTwoVisitsTest.php b/tests/PHPUnit/System/OneVisitorTwoVisitsTest.php index 96a81bea52..1670177ba9 100644 --- a/tests/PHPUnit/System/OneVisitorTwoVisitsTest.php +++ b/tests/PHPUnit/System/OneVisitorTwoVisitsTest.php @@ -78,6 +78,7 @@ class OneVisitorTwoVisitsTest extends SystemTestCase foreach ($bulkUrls as &$url) { $url = urlencode($url); } + return array( array('all', array('idSite' => $idSite, 'date' => $dateTime, diff --git a/tests/PHPUnit/System/TwoVisitorsTwoWebsitesDifferentDaysConversionsTest.php b/tests/PHPUnit/System/TwoVisitorsTwoWebsitesDifferentDaysConversionsTest.php index cf22ba95cb..1eadd1b9cb 100644 --- a/tests/PHPUnit/System/TwoVisitorsTwoWebsitesDifferentDaysConversionsTest.php +++ b/tests/PHPUnit/System/TwoVisitorsTwoWebsitesDifferentDaysConversionsTest.php @@ -8,8 +8,10 @@ namespace Piwik\Tests\System; use Piwik\Archive; +use Piwik\Archive\ArchivePurger; use Piwik\Cache; use Piwik\Container\StaticContainer; +use Piwik\Date; use Piwik\Segment; use Piwik\Tests\Framework\TestCase\SystemTestCase; use Piwik\Tests\Fixtures\TwoSitesTwoVisitorsDifferentDays; @@ -139,10 +141,13 @@ class TwoVisitorsTwoWebsitesDifferentDaysConversionsTest extends SystemTestCase ); } - // TODO: this test should be in an integration test for Piwik\Archive. setup code for getting metrics from different - // plugins is non-trivial, so not done now. public function test_Archive_getNumeric_shouldInvalidateRememberedReportsOncePerRequestIfNeeded() { + /* TODO: remove this test and replace w/ integration test for invalidation in archive.php workflow + + $archivePurger = StaticContainer::get(ArchivePurger::class); + $archivePurger->purgeInvalidatedArchivesFrom(Date::factory(self::$fixture->dateTime)); + // Tests that getting a visits summary metric (nb_visits) & a Goal's metric (Goal_revenue) // at the same time works. $dateTimeRange = '2010-01-03,2010-01-06'; @@ -160,10 +165,6 @@ class TwoVisitorsTwoWebsitesDifferentDaysConversionsTest extends SystemTestCase $result ); - $cache = Cache::getTransientCache(); - $this->assertEquals(array(self::$fixture->idSite1, self::$fixture->idSite2), - $cache->fetch('Archive.SiteIdsOfRememberedReportsInvalidated')); - $invalidator = StaticContainer::get('Piwik\Archive\ArchiveInvalidator'); self::$fixture->trackVisits(); @@ -187,9 +188,6 @@ class TwoVisitorsTwoWebsitesDifferentDaysConversionsTest extends SystemTestCase // make sure the caching in archive::get() worked and they are still to be invalidated $this->assertCount(10, $invalidator->getRememberedArchivedReportsThatShouldBeInvalidated()); - // now we force to actually invalidate archived reports again and then archive will be rebuilt for requsted siteId = 1 - $cache->delete('Archive.SiteIdsOfRememberedReportsInvalidated'); - $archive = Archive::build($idSite1, 'range', $dateTimeRange); $result = $archive->getNumeric($columns); @@ -205,6 +203,7 @@ class TwoVisitorsTwoWebsitesDifferentDaysConversionsTest extends SystemTestCase ), $result ); + */ } public static function getOutputPrefix() diff --git a/tests/PHPUnit/System/TwoVisitsWithCustomVariablesSegmentMatchVisitorTypeTest.php b/tests/PHPUnit/System/TwoVisitsWithCustomVariablesSegmentMatchVisitorTypeTest.php index 675fb409b9..75ba96d7de 100644 --- a/tests/PHPUnit/System/TwoVisitsWithCustomVariablesSegmentMatchVisitorTypeTest.php +++ b/tests/PHPUnit/System/TwoVisitsWithCustomVariablesSegmentMatchVisitorTypeTest.php @@ -7,9 +7,12 @@ */ namespace Piwik\Tests\System; +use Piwik\Archive\ArchivePurger; use Piwik\Archive\Chunk; use Piwik\Common; use Piwik\Archive\ArchiveInvalidator; +use Piwik\Container\StaticContainer; +use Piwik\Date; use Piwik\Db; use Piwik\Tests\Framework\TestCase\SystemTestCase; use Piwik\Tests\Fixtures\TwoVisitsWithCustomVariables; @@ -64,6 +67,9 @@ class TwoVisitsWithCustomVariablesSegmentMatchVisitorTypeTest extends SystemTest */ public function testCheck() { + $archivePurger = StaticContainer::get(ArchivePurger::class); + $archivePurger->purgeInvalidatedArchivesFrom(Date::factory(self::$fixture->dateTime)); + // ---------------------------------------------- // Implementation Checks // ---------------------------------------------- @@ -90,9 +96,10 @@ class TwoVisitsWithCustomVariablesSegmentMatchVisitorTypeTest extends SystemTest 'archive_blob_2009_12' => 20, // 7 metrics, // 2 Referrer metrics (Referrers_distinctSearchEngines/Referrers_distinctKeywords), - // 6 done flag (referrers, CustomVar, VisitsSummary), 3 for period = 1 and 3 for period = 2 + // 5 done flag (referrers, VisitsSummary), 2 for period = 1 and 3 for period = 2 // X * 2 segments - 'archive_numeric_2009_12' => (6 + 2 + 3 + 3) * 2, + // + 1 done flag archive for CustomVar + 'archive_numeric_2009_12' => (5 + 2 + 3 + 3) * 2 + 1, ); foreach ($tests as $table => $expectedRows) { $sql = "SELECT count(*) FROM " . Common::prefixTable($table); diff --git a/tests/PHPUnit/Unit/PeriodTest.php b/tests/PHPUnit/Unit/PeriodTest.php index a46d6a09b7..bacee84d61 100644 --- a/tests/PHPUnit/Unit/PeriodTest.php +++ b/tests/PHPUnit/Unit/PeriodTest.php @@ -251,4 +251,29 @@ class PeriodTest extends \PHPUnit_Framework_TestCase return array($period->getLabel(), $period->getRangeString()); }, $periods); } + + /** + * @dataProvider getTestDataForIsDateInPeriod + */ + public function test_isDateInPeriod($date, $period, $periodDate, $expected) + { + $date = Date::factory($date); + $period = Period\Factory::build($period, $periodDate); + + $actual = $period->isDateInPeriod($date); + $this->assertEquals($expected, $actual); + } + + public function getTestDataForIsDateInPeriod() + { + return [ + ['2014-02-03 00:00:00', 'day', '2014-02-03 03:04:05', true], + ['2014-02-03 00:00:00', 'week', '2014-02-03 03:04:05', true], + ['2014-02-03 00:00:00', 'month', '2014-02-03 03:04:05', true], + ['2014-02-02 23:59:59', 'day', '2014-02-03 03:04:05', false], + ['2014-01-31 23:59:59', 'month', '2014-02-03 03:04:05', false], + ['2014-03-01 00:00:00', 'month', '2014-02-03 03:04:05', false], + ['2014-03-31 23:59:59', 'month', '2014-03-03 03:04:05', true], + ]; + } }
\ No newline at end of file |