diff options
70 files changed, 4387 insertions, 859 deletions
diff --git a/config/global.ini.php b/config/global.ini.php index 7c1ba59e37..3103fbd4f0 100755 --- a/config/global.ini.php +++ b/config/global.ini.php @@ -741,6 +741,10 @@ custom_cacert_pem= ; Default is 1. enable_tracking_failures_notification = 1 +; Controls how many months in the past reports are re-archived for plugins that support +; doing this (such as CustomReports). Set to 0 to disable the feature. Default is 6. +rearchive_reports_in_past_last_n_months = last6 + [Tracker] ; When enabled and a userId is set, then the visitorId will be automatically set based on the userId. This allows to diff --git a/core/Access.php b/core/Access.php index b7adcfab33..87043193ad 100644 --- a/core/Access.php +++ b/core/Access.php @@ -673,7 +673,7 @@ class Access try { $result = $function(); - } catch (Exception $ex) { + } catch (\Throwable $ex) { $access->setSuperUserAccess($isSuperUser); if ($shouldResetLogin) { $access->login = null; diff --git a/core/Archive.php b/core/Archive.php index 545c4ff757..8f3a5fdbfa 100644 --- a/core/Archive.php +++ b/core/Archive.php @@ -14,6 +14,7 @@ use Piwik\Archive\Parameters; use Piwik\ArchiveProcessor\Rules; use Piwik\Container\StaticContainer; use Piwik\DataAccess\ArchiveSelector; +use Piwik\Plugins\CoreAdminHome\API; /** * The **Archive** class is used to query cached analytics statistics @@ -524,7 +525,7 @@ class Archive implements ArchiveQuery // then we have the archive IDs in $this->idarchives) $doneFlags = array(); $archiveGroups = array(); - foreach ($plugins as $plugin) { + foreach (array_merge($plugins, ['all']) as $plugin) { $doneFlag = $this->getDoneStringForPlugin($plugin, $this->params->getIdSites()); $doneFlags[$doneFlag] = true; @@ -537,12 +538,13 @@ class Archive implements ArchiveQuery $archiveGroups[] = $archiveGroup; } - $globalDoneFlag = Rules::getDoneFlagArchiveContainsAllPlugins($this->params->getSegment()); - if ($globalDoneFlag !== $doneFlag) { - $doneFlags[$globalDoneFlag] = true; - } + $doneFlag = Rules::getDoneFlagArchiveContainsOnePlugin($this->params->getSegment(), $plugin); + $doneFlags[$doneFlag] = true; } + $globalDoneFlag = Rules::getDoneFlagArchiveContainsAllPlugins($this->params->getSegment()); + $doneFlags[$globalDoneFlag] = true; + $archiveGroups = array_unique($archiveGroups); // cache id archives for plugins we haven't processed yet @@ -583,7 +585,6 @@ class Archive implements ArchiveQuery && Common::getRequestVar('skipArchiveSegmentToday', 0, 'int') && $period->getDateStart()->toString() === Date::factory('now', $site->getTimezone())->toString() ) { - Log::debug("Skipping archive %s for %s as segment today is disabled", $period->getLabel(), $period->getPrettyString()); continue; } @@ -632,6 +633,8 @@ class Archive implements ArchiveQuery foreach ($idarchivesByReport as $doneFlag => $idarchivesByDate) { foreach ($idarchivesByDate as $dateRange => $idarchives) { foreach ($idarchives as $idarchive) { + // idarchives selected can include all plugin archives, plugin specific archives and partial report + // archives. only the latest data in all of these archives will be selected. $this->idarchives[$doneFlag][$dateRange][] = $idarchive; } } @@ -785,25 +788,33 @@ class Archive implements ArchiveQuery */ private function prepareArchive(array $archiveGroups, Site $site, Period $period) { - // if cron archiving is running, we will invalidate in CronArchive, not here - $invalidateBeforeArchiving = !SettingsServer::isArchivePhpTriggered(); + $coreAdminHomeApi = API::getInstance(); - $parameters = new ArchiveProcessor\Parameters($site, $period, $this->params->getSegment()); - $archiveLoader = new ArchiveProcessor\Loader($parameters, $invalidateBeforeArchiving); + $requestedReport = null; + if (SettingsServer::isArchivePhpTriggered()) { + $requestedReport = Common::getRequestVar('requestedReport', '', 'string'); + } $periodString = $period->getRangeString(); + $periodDateStr = $period->getLabel() == 'range' ? $periodString : $period->getDateStart()->toString(); $idSites = array($site->getId()); - + // process for each plugin as well foreach ($archiveGroups as $plugin) { $doneFlag = $this->getDoneStringForPlugin($plugin, $idSites); $this->initializeArchiveIdCache($doneFlag); - $idArchive = $archiveLoader->prepareArchive($plugin); + $prepareResult = $coreAdminHomeApi->archiveReports( + $site->getId(), $period->getLabel(), $periodDateStr, $this->params->getSegment()->getString(), + $plugin, $requestedReport); - if ($idArchive) { - $this->idarchives[$doneFlag][$periodString][] = $idArchive; + if (!empty($prepareResult) + && !empty($prepareResult['idarchives']) + ) { + foreach ($prepareResult['idarchives'] as $idArchive) { + $this->idarchives[$doneFlag][$periodString][] = $idArchive; + } } } } diff --git a/core/Archive/ArchiveInvalidator.php b/core/Archive/ArchiveInvalidator.php index 392aae29cf..709a79e60f 100644 --- a/core/Archive/ArchiveInvalidator.php +++ b/core/Archive/ArchiveInvalidator.php @@ -12,6 +12,9 @@ namespace Piwik\Archive; use Piwik\Archive\ArchiveInvalidator\InvalidationResult; use Piwik\ArchiveProcessor\ArchivingStatus; use Piwik\ArchiveProcessor\Loader; +use Piwik\Config; +use Piwik\Container\StaticContainer; +use Piwik\CronArchive\SegmentArchiving; use Piwik\DataAccess\ArchiveTableCreator; use Piwik\DataAccess\Model; use Piwik\Date; @@ -19,6 +22,7 @@ use Piwik\Db; use Piwik\Option; use Piwik\Common; use Piwik\Piwik; +use Piwik\Plugin\Manager; use Piwik\Plugins\CoreAdminHome\Tasks\ArchivesToPurgeDistributedList; use Piwik\Plugins\PrivacyManager\PrivacyManager; use Piwik\Period; @@ -26,6 +30,7 @@ use Piwik\Segment; use Piwik\SettingsServer; use Piwik\Site; use Piwik\Tracker\Cache; +use Piwik\Tracker\Model as TrackerModel; /** * Service that can be used to invalidate archives or add archive references to a list so they will @@ -69,10 +74,16 @@ class ArchiveInvalidator */ private $archivingStatus; + /** + * @var SegmentArchiving + */ + private $segmentArchiving; + public function __construct(Model $model, ArchivingStatus $archivingStatus) { $this->model = $model; $this->archivingStatus = $archivingStatus; + $this->segmentArchiving = null; } public function getAllRememberToInvalidateArchivedReportsLater() @@ -228,16 +239,36 @@ class ArchiveInvalidator /** * @param $idSites int[] - * @param $dates Date[] + * @param $dates Date[]|string[] * @param $period string * @param $segment Segment * @param bool $cascadeDown + * @param bool $forceInvalidateNonexistantRanges set true to force inserting rows for ranges in archive_invalidations + * @param string $name null to make sure every plugin is archived when this invalidation is processed by core:archive, + * or a plugin name to only archive the specific plugin. * @return InvalidationResult * @throws \Exception */ public function markArchivesAsInvalidated(array $idSites, array $dates, $period, Segment $segment = null, $cascadeDown = false, - $forceInvalidateNonexistantRanges = false) + $forceInvalidateNonexistantRanges = false, $name = null) { + $plugin = null; + if ($name && strpos($name, '.') !== false) { + list($plugin) = explode('.', $name); + } + + // remove sites w/ no visits + $trackerModel = new TrackerModel(); + $idSites = array_filter($idSites, function ($idSite) use ($trackerModel) { + return !$trackerModel->isSiteEmpty($idSite); + }); + + if ($plugin + && !Manager::getInstance()->isPluginActivated($plugin) + ) { + throw new \Exception("Plugin is not activated: '$plugin'"); + } + $invalidationInfo = new InvalidationResult(); // quick fix for #15086, if we're only invalidating today's date for a site, don't add the site to the list of sites @@ -247,9 +278,9 @@ class ArchiveInvalidator $hasMoreThanJustToday[$idSite] = true; $tz = Site::getTimezoneFor($idSite); - if (($period === 'day' || $period === false) - && count($dates) === 1 - && $dates[0]->toString() == Date::factoryInTimezone('today', $tz) + if (($period == 'day' || $period === false) + && count($dates) == 1 + && ((string)$dates[0]) == ((string)Date::factoryInTimezone('today', $tz)) ) { $hasMoreThanJustToday[$idSite] = false; } @@ -282,7 +313,7 @@ class ArchiveInvalidator $allPeriodsToInvalidate = $this->getAllPeriodsByYearMonth($period, $datesToInvalidate, $cascadeDown); - $this->markArchivesInvalidated($idSites, $allPeriodsToInvalidate, $segment, $period != 'range', $forceInvalidateNonexistantRanges); + $this->markArchivesInvalidated($idSites, $allPeriodsToInvalidate, $segment, $period != 'range', $forceInvalidateNonexistantRanges, $name); foreach ($idSites as $idSite) { Loader::invalidateMinVisitTimeCache($idSite); @@ -414,11 +445,73 @@ class ArchiveInvalidator } /** + * Schedule rearchiving of reports for a single plugin or single report for N months in the past. The next time + * core:archive is run, they will be processed. + * + * @param int[] $idSite + * @param Date $date1 + * @param Date $date2 + * @param string $plugin + * @param string|null $report + * @throws \Exception + * @api + */ + public function reArchiveReport(array $idSites, string $plugin, string $report = null, int $lastNMonthsToInvalidate = null) + { + $lastNMonthsToInvalidate = $lastNMonthsToInvalidate ?: Config::getInstance()->General['rearchive_reports_in_past_last_n_months']; + if (empty($lastNMonthsToInvalidate)) { + return; + } + + $lastNMonthsToInvalidate = (int) substr($lastNMonthsToInvalidate, 4); + if (empty($lastNMonthsToInvalidate)) { + return; + } + + $date2 = Date::yesterday(); + $date1 = $date2->subMonth($lastNMonthsToInvalidate)->setDay(1); + + $dates = []; + $date = $date1; + while ($date->isEarlier($date2)) { + $dates[] = $date; + $date = $date->addDay(1); + } + + $name = $plugin; + if (!empty($report)) { + $name .= '.' . $report; + } + + $this->markArchivesAsInvalidated($idSites, $dates, 'day', null, $cascadeDown = false, $forceInvalidateRanges = false, $name); + + foreach ($idSites as $idSite) { + $segmentDatesToInvalidate = $this->getSegmentArchiving()->getSegmentArchivesToInvalidate($idSite); + foreach ($segmentDatesToInvalidate as $info) { + $latestDate = Date::factory($info['date']); + $latestDate = $latestDate->isEarlier($date1) ? $latestDate : $date1; + + $datesToInvalidateForSegment = []; + + $date = $latestDate; + while ($date->isEarlier($date2)) { + $datesToInvalidateForSegment[] = $date; + $date = $date->addDay(1); + } + + $this->markArchivesAsInvalidated($idSites, $datesToInvalidateForSegment, 'day', new Segment($info['segment'], [$idSite]), + $cascadeDown = false, $forceInvalidateRanges = false, $name); + } + } + } + + /** * @param int[] $idSites * @param string[][][] $dates * @throws \Exception */ - private function markArchivesInvalidated($idSites, $dates, Segment $segment = null, $removeRanges = false, $forceInvalidateNonexistantRanges = false) + private function markArchivesInvalidated($idSites, $dates, Segment $segment = null, $removeRanges = false, + $forceInvalidateNonexistantRanges = false, $name = null) { $idSites = array_map('intval', $idSites); @@ -430,7 +523,8 @@ class ArchiveInvalidator $table = ArchiveTableCreator::getNumericTable($tableDateObj); $yearMonths[] = $tableDateObj->toString('Y_m'); - $this->model->updateArchiveAsInvalidated($table, $idSites, $datesForTable, $segment, $forceInvalidateNonexistantRanges); + $this->model->updateArchiveAsInvalidated($table, $idSites, $datesForTable, $segment, $forceInvalidateNonexistantRanges, $name); + if ($removeRanges) { $this->model->updateRangeArchiveAsInvalidated($table, $idSites, $datesForTable, $segment); } @@ -512,4 +606,12 @@ class ArchiveInvalidator return Period\Factory::build($period, $date); } } + + private function getSegmentArchiving() + { + if (empty($this->segmentArchiving)) { + $this->segmentArchiving = new SegmentArchiving(StaticContainer::get('ini.General.process_new_segments_from')); + } + return $this->segmentArchiving; + } } diff --git a/core/ArchiveProcessor.php b/core/ArchiveProcessor.php index 202eb5f04d..2de3a3bc84 100644 --- a/core/ArchiveProcessor.php +++ b/core/ArchiveProcessor.php @@ -247,8 +247,7 @@ class ArchiveProcessor $metrics = $this->getAggregatedNumericMetrics($columns, $operationToApply); foreach ($metrics as $column => $value) { - $value = Common::forceDotAsSeparatorForDecimalPoint($value); - $this->archiveWriter->insertRecord($column, $value); + $this->insertNumericRecord($column, $value); } // if asked for only one field to sum if (count($metrics) === 1) { diff --git a/core/ArchiveProcessor/Loader.php b/core/ArchiveProcessor/Loader.php index 89babfb1ad..d7ea423246 100644 --- a/core/ArchiveProcessor/Loader.php +++ b/core/ArchiveProcessor/Loader.php @@ -10,6 +10,7 @@ namespace Piwik\ArchiveProcessor; use Piwik\Archive\ArchiveInvalidator; use Piwik\Cache; +use Piwik\Common; use Piwik\Config; use Piwik\Container\StaticContainer; use Piwik\Context; @@ -21,6 +22,7 @@ use Piwik\Date; use Piwik\Db; use Piwik\Period; use Piwik\Piwik; +use Piwik\SettingsServer; use Piwik\Site; use Psr\Log\LoggerInterface; @@ -99,9 +101,22 @@ class Loader { $this->params->setRequestedPlugin($pluginName); - list($idArchive, $visits, $visitsConverted, $isAnyArchiveExists) = $this->loadExistingArchiveIdFromDb(); - if (!empty($idArchive)) { // we have a usable idarchive (it's not invalidated and it's new enough) - return $idArchive; + if (SettingsServer::isArchivePhpTriggered()) { + $requestedReport = Common::getRequestVar('requestedReport', '', 'string'); + if (!empty($requestedReport)) { + $this->params->setArchiveOnlyReport($requestedReport); + } + } + + // NOTE: $idArchives will contain the latest DONE_OK/DONE_INVALIDATED archive as well as any partial archives + // with a ts_archived >= the DONE_OK/DONE_INVALIDATED date. + list($idArchives, $visits, $visitsConverted, $isAnyArchiveExists) = $this->loadExistingArchiveIdFromDb(); + if (!empty($idArchives) + && !$this->params->getArchiveOnlyReport() + ) { + // we have a usable idarchive (it's not invalidated and it's new enough), and we are not archiving + // a single report + return [$idArchives, $visits]; } // NOTE: this optimization helps when archiving large periods. eg, if archiving a year w/ a segment where @@ -111,7 +126,7 @@ class Loader // we don't create an archive in this case, because the archive may be in progress in some way, so a 0 // visits archive can be inaccurate in the long run. if ($this->canSkipThisArchive()) { - return false; + return [false, 0]; } // if there is an archive, but we can't use it for some reason, invalidate existing archives before @@ -136,10 +151,10 @@ class Loader } if ($this->isThereSomeVisits($visits) || PluginsArchiver::doesAnyPluginArchiveWithoutVisits()) { - return $idArchive; + return [[$idArchive], $visits]; } - return false; + return [false, false]; } /** @@ -155,14 +170,17 @@ class Loader if ($createSeparateArchiveForCoreMetrics) { $requestedPlugin = $this->params->getRequestedPlugin(); + $requestedReport = $this->params->getArchiveOnlyReport(); $this->params->setRequestedPlugin('VisitsSummary'); + $this->params->setArchiveOnlyReport(null); $pluginsArchiver = new PluginsArchiver($this->params); $metrics = $pluginsArchiver->callAggregateCoreMetrics(); $pluginsArchiver->finalizeArchive(); $this->params->setRequestedPlugin($requestedPlugin); + $this->params->setArchiveOnlyReport($requestedReport); $visits = $metrics['nb_visits']; $visitsConverted = $metrics['nb_visits_converted']; @@ -342,10 +360,12 @@ class Loader $idSite = $params->getSite()->getId(); $isWebsiteUsingTracker = $this->isWebsiteUsingTheTracker($idSite); + $isArchivingForcedWhenNoVisits = $this->shouldArchiveForSiteEvenWhenNoVisits(); $hasSiteVisitsBetweenTimeframe = $this->hasSiteVisitsBetweenTimeframe($idSite, $params->getPeriod()); $hasChildArchivesInPeriod = $this->dataAccessModel->hasChildArchivesInPeriod($idSite, $params->getPeriod()); return $isWebsiteUsingTracker + && !$isArchivingForcedWhenNoVisits && !$hasSiteVisitsBetweenTimeframe && !$hasChildArchivesInPeriod; } @@ -396,7 +416,6 @@ class Loader $timezone = Site::getTimezoneFor($idSite); list($date1, $date2) = $period->getBoundsInTimezone($timezone); - if ($date2->isEarlier($minVisitTimesPerSite)) { return false; } @@ -412,7 +431,9 @@ class Loader $value = $cache->fetch($cacheKey); if ($value === false) { $value = $this->rawLogDao->getMinimumVisitTimeForSite($idSite); - $cache->save($cacheKey, $value, $ttl = self::MIN_VISIT_TIME_TTL); + if (!empty($value)) { + $cache->save($cacheKey, $value, $ttl = self::MIN_VISIT_TIME_TTL); + } } if (!empty($value)) { diff --git a/core/ArchiveProcessor/Parameters.php b/core/ArchiveProcessor/Parameters.php index 52ec8bb12b..d3a23b1e8f 100644 --- a/core/ArchiveProcessor/Parameters.php +++ b/core/ArchiveProcessor/Parameters.php @@ -54,6 +54,16 @@ class Parameters private $isRootArchiveRequest = true; /** + * @var string + */ + private $archiveOnlyReport = null; + + /** + * @var bool + */ + private $isArchiveOnlyReportHandled; + + /** * Constructor. * * @ignore @@ -66,6 +76,29 @@ class Parameters } /** + * If we want to archive only a single report, we can request that via this method. + * It is up to each plugin's archiver to respect the setting. + * + * @param string $archiveOnlyReport + * @api + */ + public function setArchiveOnlyReport($archiveOnlyReport) + { + $this->archiveOnlyReport = $archiveOnlyReport; + } + + /** + * Gets the report we want to archive specifically, or null if none was specified. + * + * @return string|null + * @api + */ + public function getArchiveOnlyReport() + { + return $this->archiveOnlyReport; + } + + /** * @ignore */ public function setRequestedPlugin($plugin) @@ -266,4 +299,30 @@ class Parameters { return "[idSite = {$this->getSite()->getId()}, period = {$this->getPeriod()->getLabel()} {$this->getPeriod()->getRangeString()}, segment = {$this->getSegment()->getString()}]"; } + + /** + * Returns whether the setArchiveOnlyReport() was handled by an Archiver. + * + * @return bool + */ + public function isPartialArchive() + { + if (!$this->getRequestedPlugin()) { // sanity check, partial archives are only for + return false; + } + + return $this->isArchiveOnlyReportHandled; + } + + /** + * If a plugin's archiver handles the setArchiveOnlyReport() setting, it should call this method + * so it is known that the archive only contains the requested report. This should be called + * in an Archiver's __construct method. + * + * @param bool $isArchiveOnlyReportHandled + */ + public function setIsPartialArchive($isArchiveOnlyReportHandled) + { + $this->isArchiveOnlyReportHandled = $isArchiveOnlyReportHandled; + } } diff --git a/core/ArchiveProcessor/PluginsArchiver.php b/core/ArchiveProcessor/PluginsArchiver.php index 60d95d5a2d..4fd517a1df 100644 --- a/core/ArchiveProcessor/PluginsArchiver.php +++ b/core/ArchiveProcessor/PluginsArchiver.php @@ -72,6 +72,7 @@ class PluginsArchiver $this->archiveProcessor = new ArchiveProcessor($this->params, $this->archiveWriter, $this->logAggregator); + $shouldAggregateFromRawData = $this->params->isSingleSiteDayArchive(); /** @@ -149,7 +150,6 @@ class PluginsArchiver } if ($this->shouldProcessReportsForPlugin($pluginName)) { - $this->logAggregator->setQueryOriginHint($pluginName); try { @@ -266,13 +266,17 @@ class PluginsArchiver if (Rules::shouldProcessReportsAllPlugins( array($this->params->getSite()->getId()), $this->params->getSegment(), - $this->params->getPeriod()->getLabel())) { + $this->params->getPeriod()->getLabel()) + ) { return true; } - if (!\Piwik\Plugin\Manager::getInstance()->isPluginLoaded($this->params->getRequestedPlugin())) { - return true; + if ($this->params->getRequestedPlugin() && + !\Piwik\Plugin\Manager::getInstance()->isPluginLoaded($this->params->getRequestedPlugin()) + ) { + return false; } + return false; } diff --git a/core/ArchiveProcessor/Rules.php b/core/ArchiveProcessor/Rules.php index ae2a560cd6..24acf8a434 100644 --- a/core/ArchiveProcessor/Rules.php +++ b/core/ArchiveProcessor/Rules.php @@ -9,12 +9,14 @@ namespace Piwik\ArchiveProcessor; use Exception; +use Piwik\Common; use Piwik\Config; use Piwik\DataAccess\ArchiveWriter; use Piwik\Date; use Piwik\Log; use Piwik\Option; use Piwik\Piwik; +use Piwik\Plugin\Manager; use Piwik\Plugins\CoreAdminHome\Controller; use Piwik\Segment; use Piwik\SettingsPiwik; @@ -57,6 +59,10 @@ class Rules public static function shouldProcessReportsAllPlugins(array $idSites, Segment $segment, $periodLabel) { + if (self::isForceArchivingSinglePlugin()) { + return false; + } + if ($segment->isEmpty() && ($periodLabel != 'range' || SettingsServer::isArchivePhpTriggered())) { return true; } @@ -305,17 +311,27 @@ class Rules * * @return string[] */ - public static function getSelectableDoneFlagValues($includeInvalidated = true, Parameters $params = null) + public static function getSelectableDoneFlagValues($includeInvalidated = true, Parameters $params = null, $checkAuthorizedToArchive = true) { $possibleValues = array(ArchiveWriter::DONE_OK, ArchiveWriter::DONE_OK_TEMPORARY); - if (!Rules::isRequestAuthorizedToArchive($params) - && $includeInvalidated - ) { - //If request is not authorized to archive then fetch also invalidated archives - $possibleValues[] = ArchiveWriter::DONE_INVALIDATED; + if ($includeInvalidated) { + if (!$checkAuthorizedToArchive || !Rules::isRequestAuthorizedToArchive($params)) { + //If request is not authorized to archive then fetch also invalidated archives + $possibleValues[] = ArchiveWriter::DONE_INVALIDATED; + $possibleValues[] = ArchiveWriter::DONE_PARTIAL; + } } return $possibleValues; } + + public static function isForceArchivingSinglePlugin() + { + if (!SettingsServer::isArchivePhpTriggered()) { + return false; + } + + return !empty($_GET['pluginOnly']) || !empty($_POST['pluginOnly']); + } } diff --git a/core/CliMulti.php b/core/CliMulti.php index c604ab6702..3b0909d347 100644 --- a/core/CliMulti.php +++ b/core/CliMulti.php @@ -12,6 +12,8 @@ use Piwik\CliMulti\CliPhp; use Piwik\CliMulti\Output; use Piwik\CliMulti\Process; use Piwik\Container\StaticContainer; +use Psr\Log\LoggerInterface; +use Psr\Log\NullLogger; /** * Class CliMulti. @@ -71,9 +73,15 @@ class CliMulti protected $isTimingRequests = false; - public function __construct() + /** + * @var LoggerInterface + */ + private $logger; + + public function __construct(LoggerInterface $logger = null) { $this->supportsAsync = $this->supportsAsync(); + $this->logger = $logger ?: new NullLogger(); } /** @@ -333,7 +341,7 @@ class CliMulti $hostname = Url::getHost($checkIfTrusted = false); $command = $this->buildCommand($hostname, $query, $output->getPathToFile()); - Log::debug($command); + $this->logger->debug("Running command: {command}", ['command' => $command]); shell_exec($command); } @@ -349,6 +357,7 @@ class CliMulti $url = str_replace("http://", "https://", $url); } + $requestBody = null; if ($this->runAsSuperUser) { $tokenAuth = self::getSuperUserTokenAuth(); @@ -358,12 +367,12 @@ class CliMulti $url .= '&'; } - $url .= 'token_auth=' . $tokenAuth; + $requestBody = 'token_auth=' . $tokenAuth; } try { - Log::debug("Execute HTTP API request: " . $url); - $response = Http::sendHttpRequestBy('curl', $url, $timeout = 0, $userAgent = null, $destinationPath = null, $file = null, $followDepth = 0, $acceptLanguage = false, $this->acceptInvalidSSLCertificate); + $this->logger->debug("Execute HTTP API request: " . $url); + $response = Http::sendHttpRequestBy('curl', $url, $timeout = 0, $userAgent = null, $destinationPath = null, $file = null, $followDepth = 0, $acceptLanguage = false, $this->acceptInvalidSSLCertificate, false, false, 'POST', null, null, $requestBody, [], $forcePost = true); $output->write($response); } catch (\Exception $e) { $message = "Got invalid response from API request: $url. "; @@ -376,7 +385,7 @@ class CliMulti $output->write($message); - Log::debug($e); + $this->logger->debug($message, ['exception' => $e]); } } diff --git a/core/CronArchive.php b/core/CronArchive.php index b52fa24ab8..1dd1265070 100644 --- a/core/CronArchive.php +++ b/core/CronArchive.php @@ -19,12 +19,14 @@ use Piwik\CronArchive\FixedSiteIds; use Piwik\CronArchive\Performance\Logger; use Piwik\Archive\ArchiveInvalidator; use Piwik\CliMulti\RequestParser; +use Piwik\CronArchive\QueueConsumer; use Piwik\CronArchive\SharedSiteIds; use Piwik\DataAccess\ArchiveSelector; use Piwik\DataAccess\ArchiveTableCreator; use Piwik\DataAccess\Model; use Piwik\DataAccess\RawLogDao; use Piwik\Metrics\Formatter; +use Piwik\Period\Factory; use Piwik\Period\Factory as PeriodFactory; use Piwik\CronArchive\SegmentArchiving; use Piwik\Period\Range; @@ -139,7 +141,7 @@ class CronArchive * * @var int|false */ - public $dateLastForced = SegmentArchiving::DEFAULT_BEGINNIN_OF_TIME_LAST_N_YEARS; + public $dateLastForced = SegmentArchiving::DEFAULT_BEGINNING_OF_TIME_LAST_N_YEARS; /** * The number of concurrent requests to issue per website. Defaults to {@link MAX_CONCURRENT_API_REQUESTS}. @@ -208,11 +210,6 @@ class CronArchive private $archiveFilter; /** - * @var array - */ - private $invalidationsToExclude = []; - - /** * @var RequestParser */ private $cliMultiRequestParser; @@ -329,8 +326,6 @@ class CronArchive $pid = Common::getProcessId(); $timer = new Timer; - $siteTimer = null; - $siteRequests = 0; $this->logSection("START"); $this->logger->info("Starting Matomo reports archiving..."); @@ -344,14 +339,12 @@ class CronArchive $countOfProcesses = $this->getMaxConcurrentApiRequests(); + $queueConsumer = new QueueConsumer($this->logger, $this->websiteIdArchiveList, $countOfProcesses, $pid, + $this->model, $this->segmentArchiving, $this, $this->cliMultiRequestParser, $this->archiveFilter); + // invalidate once at the start no matter when the last invalidation occurred $this->invalidateArchivedReportsForSitesThatNeedToBeArchivedAgain(); - // if we skip or can't process an idarchive, we want to ignore it the next time we look for an invalidated - // archive. these IDs are stored here (using a list like this serves to keep our SQL simple). - $this->invalidationsToExclude = []; - - $idSite = null; while (true) { if ($this->isMaintenanceModeEnabled()) { $this->logger->info("Archiving will stop now because maintenance mode is enabled"); @@ -363,144 +356,15 @@ class CronArchive flush(); } - if (empty($idSite)) { - $idSite = $this->getNextIdSiteToArchive(); - if (empty($idSite)) { // no sites left to archive, stop - $this->logger->debug("No more sites left to archive, stopping."); - return; - } - - /** - * This event is triggered before the cron archiving process starts archiving data for a single - * site. - * - * Note: multiple archiving processes can post this event. - * - * @param int $idSite The ID of the site we're archiving data for. - * @param string $pid The PID of the process processing archives for this site. - */ - Piwik::postEvent('CronArchive.archiveSingleSite.start', array($idSite, $pid)); - - $this->logger->info("Start processing archives for site {idSite}.", ['idSite' => $idSite]); - - $siteTimer = new Timer(); - $siteRequests = 0; - } - - // we don't want to invalidate different periods together or segment archives w/ no-segment archives - // together, but it's possible to end up querying these archives. if we find one, we keep track of it - // in this array to exclude, but after we run the current batch, we reset the array so we'll still - // process them eventually. - $invalidationsToExcludeInBatch = []; - - // get archives to process simultaneously - $archivesToProcess = []; - while (count($archivesToProcess) < $countOfProcesses) { - $invalidatedArchive = $this->getNextInvalidatedArchive($idSite, array_keys($invalidationsToExcludeInBatch)); - if (empty($invalidatedArchive)) { - $this->logger->debug("No next invalidated archive."); - break; - } - - if ($this->hasDifferentPeriod($archivesToProcess, $invalidatedArchive['period'])) { - $this->logger->debug("Found archive with different period than others in concurrent batch, skipping until next batch: {$invalidatedArchive['period']}"); - - $idinvalidation = $invalidatedArchive['idinvalidation']; - $invalidationsToExcludeInBatch[$idinvalidation] = true; - continue; - } - - if ($this->hasDifferentDoneFlagType($archivesToProcess, $invalidatedArchive['name'])) { - $this->logger->debug("Found archive with different done flag type (segment vs. no segment) in concurrent batch, skipping until next batch: {$invalidatedArchive['name']}"); - - $idinvalidation = $invalidatedArchive['idinvalidation']; - $invalidationsToExcludeInBatch[$idinvalidation] = true; - - continue; - } - - if ($invalidatedArchive['segment'] === null) { - $this->logger->debug("Found archive for segment that is not auto archived, ignoring."); - $this->addInvalidationToExclude($invalidatedArchive); - continue; - } - - if ($this->isDoneFlagForPlugin($invalidatedArchive['name'])) { - $this->logger->debug("Found plugin specific invalidated archive, ignoring."); - $this->addInvalidationToExclude($invalidatedArchive); - continue; - } - - if ($this->archiveArrayContainsArchive($archivesToProcess, $invalidatedArchive)) { - $this->logger->debug("Found duplicate invalidated archive {$invalidatedArchive['idarchive']}, ignoring."); - $this->addInvalidationToExclude($invalidatedArchive); - $this->model->deleteInvalidations([$invalidatedArchive]); - continue; - } - - $reason = $this->shouldSkipArchive($invalidatedArchive); - if ($reason) { - $this->logger->debug("Skipping invalidated archive {$invalidatedArchive['idarchive']}: $reason"); - $this->addInvalidationToExclude($invalidatedArchive); - continue; - } - - if ($this->canSkipArchiveBecauseNoPoint($invalidatedArchive)) { - $this->logger->debug("Found invalidated archive we can skip (no visits or latest archive is not invalidated). " - . "[idSite = {$invalidatedArchive['idsite']}, dates = {$invalidatedArchive['date1']} - {$invalidatedArchive['date2']}, segment = {$invalidatedArchive['segment']}]"); - $this->addInvalidationToExclude($invalidatedArchive); - $this->model->deleteInvalidations([$invalidatedArchive]); - continue; - } - - // TODO: should use descriptive string instead of just invalidation ID - $reason = $this->shouldSkipArchiveBecauseLowerPeriodOrSegmentIsInProgress($invalidatedArchive); - if ($reason) { - $this->logger->debug("Skipping invalidated archive {$invalidatedArchive['idarchive']}: $reason"); - $invalidationsToExcludeInBatch[$invalidatedArchive['idinvalidation']] = true; - $this->addInvalidationToExclude($invalidatedArchive); - continue; - } - - $started = $this->model->startArchive($invalidatedArchive); - if (!$started) { // another process started on this archive, pull another one - $this->logger->debug("Archive invalidation {$invalidatedArchive['idinvalidation']} is being handled by another process."); - $this->addInvalidationToExclude($invalidatedArchive); - continue; - } - - $this->addInvalidationToExclude($invalidatedArchive); - - $archivesToProcess[] = $invalidatedArchive; + $archivesToProcess = $queueConsumer->getNextArchivesToProcess(); + if ($archivesToProcess === null) { + break; } - if (empty($archivesToProcess)) { // no invalidated archive left - /** - * This event is triggered immediately after the cron archiving process starts archiving data for a single - * site. - * - * Note: multiple archiving processes can post this event. - * - * @param int $idSite The ID of the site we're archiving data for. - * @param string $pid The PID of the process processing archives for this site. - */ - Piwik::postEvent('CronArchive.archiveSingleSite.finish', array($idSite, $pid)); - - $this->logger->info("Finished archiving for site {idSite}, {requests} API requests, {timer} [{processed} / {totalNum} done]", [ - 'idSite' => $idSite, - 'processed' => $this->websiteIdArchiveList->getNumProcessedWebsites(), - 'totalNum' => $this->websiteIdArchiveList->getNumSites(), - 'timer' => $siteTimer, - 'requests' => $siteRequests, - ]); - - $idSite = null; - + if (empty($archivesToProcess)) { continue; } - $siteRequests += count($archivesToProcess); - $successCount = $this->launchArchivingFor($archivesToProcess); $numArchivesFinished += $successCount; } @@ -521,64 +385,12 @@ class CronArchive $this->logger->info($timer->__toString()); } - private function isDoneFlagForPlugin($doneFlag) - { - return strpos($doneFlag, '.') !== false; - } - - private function archiveArrayContainsArchive($archiveArray, $archive) - { - foreach ($archiveArray as $entry) { - if ($entry['idsite'] == $archive['idsite'] - && $entry['period'] == $archive['period'] - && $entry['date1'] == $archive['date1'] - && $entry['date2'] == $archive['date2'] - && $entry['name'] == $archive['name'] - ) { - return true; - } - } - return false; - } - - // TODO: need to also delete rows from archive_invalidations via scheduled task, eg, if ts_invalidated is older than 3 days or something. - private function getNextInvalidatedArchive($idSite, $extraInvalidationsToIgnore) - { - $lastInvalidationTime = self::getLastInvalidationTime(); - if (empty($lastInvalidationTime) - || (time() - $lastInvalidationTime) >= 3600 - ) { - $this->invalidateArchivedReportsForSitesThatNeedToBeArchivedAgain(); - } - - $iterations = 0; - while ($iterations < 100) { - $invalidationsToExclude = array_merge($this->invalidationsToExclude, $extraInvalidationsToIgnore); - - $nextArchive = $this->model->getNextInvalidatedArchive($idSite, $invalidationsToExclude); - if (empty($nextArchive)) { - break; - } - - $isCronArchivingEnabled = $this->findSegmentForArchive($nextArchive); - if ($isCronArchivingEnabled) { - return $nextArchive; - } - - $this->invalidationsToExclude[] = $nextArchive['idinvalidation']; - - ++$iterations; - } - - return null; - } - private function launchArchivingFor($archives) { $urls = []; $archivesBeingQueried = []; foreach ($archives as $index => $archive) { - list($url, $segment) = $this->generateUrlToArchiveFromArchiveInfo($archive); + list($url, $segment, $plugin) = $this->generateUrlToArchiveFromArchiveInfo($archive); if (empty($url)) { // can happen if, for example, a segment was deleted after an archive was invalidated // in this case, we can just delete the archive entirely. @@ -591,6 +403,11 @@ class CronArchive $period = PeriodFactory::build($this->periodIdsToLabels[$archive['period']], $dateStr); $params = new Parameters(new Site($idSite), $period, new Segment($segment, [$idSite], $period->getDateStart(), $period->getDateEnd())); + if (!empty($plugin)) { + $params->setRequestedPlugin($plugin); + $params->onlyArchiveRequestedPlugin(); + } + $loader = new Loader($params); if ($loader->canSkipThisArchive()) { $this->logger->info("Found no visits for site ID = {idSite}, {period} ({date1},{date2}), site is using the tracker so skipping archiving...", [ @@ -620,7 +437,6 @@ class CronArchive $responses = $cliMulti->request($urls); $timers = $cliMulti->getTimers(); - $successCount = 0; foreach ($urls as $index => $url) { @@ -629,13 +445,15 @@ class CronArchive $stats = json_decode($content, $assoc = true); if (!is_array($stats)) { - $this->logError("Error unserializing the following response from $url: " . $content); + $this->logger->info(var_export($content, true)); + + $this->logError("Error unserializing the following response from $url: '" . $content . "'"); continue; } $visitsForPeriod = $this->getVisitsFromApiResponse($stats); - $this->logArchiveJobFinished($url, $timers[$index], $visitsForPeriod); + $this->logArchiveJobFinished($url, $timers[$index], $visitsForPeriod, $archivesBeingQueried[$index]['plugin'], $archivesBeingQueried[$index]['report']); // TODO: do in ArchiveWriter $this->deleteInvalidatedArchives($archivesBeingQueried[$index]); @@ -661,6 +479,8 @@ class CronArchive private function generateUrlToArchiveFromArchiveInfo($archive) { + $plugin = $archive['plugin']; + $report = $archive['report']; $period = $this->periodIdsToLabels[$archive['period']]; if ($period == 'range') { @@ -673,7 +493,7 @@ class CronArchive $segment = isset($archive['segment']) ? $archive['segment'] : ''; - $url = $this->getVisitsRequestUrl($idSite, $period, $date, $segment); + $url = $this->getVisitsRequestUrl($idSite, $period, $date, $segment, $plugin); $url = $this->makeRequestUrl($url); if (!empty($segment)) { @@ -685,35 +505,25 @@ class CronArchive } } - return [$url, $segment]; - } - - private function findSegmentForArchive(&$archive) - { - $flag = explode('.', $archive['name'])[0]; - if ($flag == 'done') { - $archive['segment'] = ''; - return true; + if (!empty($plugin)) { + $url .= "&pluginOnly=1"; } - $hash = substr($flag, 4); - $storedSegment = $this->segmentArchiving->findSegmentForHash($hash, $archive['idsite']); - if (!isset($storedSegment['definition'])) { - $archive['segment'] = null; - return false; + if (!empty($report)) { + $url .= "&requestedReport=" . urlencode($report); } - $archive['segment'] = $storedSegment['definition']; - return $this->segmentArchiving->isAutoArchivingEnabledFor($storedSegment); + return [$url, $segment, $plugin]; } - private function logArchiveJobFinished($url, $timer, $visits) + private function logArchiveJobFinished($url, $timer, $visits, $plugin = null, $report = null) { $params = UrlHelper::getArrayFromQueryString($url); $visits = (int) $visits; $this->logger->info("Archived website id {$params['idSite']}, period = {$params['period']}, date = " - . "{$params['date']}, segment = '" . (isset($params['segment']) ? $params['segment'] : '') . "', $visits visits found. $timer"); + . "{$params['date']}, segment = '" . (isset($params['segment']) ? $params['segment'] : '') . "', " + . ($plugin ? "plugin = $plugin, " : "") . ($report ? "report = $report, " : "") . "$visits visits found. $timer"); } public function getErrors() @@ -787,12 +597,15 @@ class CronArchive * @param bool|false $segment * @return string */ - private function getVisitsRequestUrl($idSite, $period, $date, $segment = false) + private function getVisitsRequestUrl($idSite, $period, $date, $segment = false, $plugin = null) { - $request = "?module=API&method=API.get&idSite=$idSite&period=$period&date=" . $date . "&format=json"; + $request = "?module=API&method=CoreAdminHome.archiveReports&idSite=$idSite&period=$period&date=" . $date . "&format=json"; if ($segment) { $request .= '&segment=' . urlencode($segment); } + if (!empty($plugin)) { + $request .= "&plugin=" . $plugin; + } return $request; } @@ -912,11 +725,26 @@ class CronArchive foreach ($sitesPerDays as $date => $siteIds) { //Concurrent transaction logic will end up with duplicates set. Adding array_unique to the siteIds. - $listSiteIds = implode(',', array_unique($siteIds)); + $siteIds = array_unique($siteIds); + + $period = Factory::build('day', $date); + + $siteIdsToInvalidate = []; + foreach ($siteIds as $idSite) { + $params = new Parameters(new Site($idSite), $period, new Segment('', [$idSite], $period->getDateStart(), $period->getDateEnd())); + if ($this->isThereExistingValidPeriod($params)) { + $this->logger->info(' Found usable archive for date range {date} for site {idSite}, skipping invalidation for now.', ['date' => $date, 'idSite' => $idSite]); + continue; + } + + $siteIdsToInvalidate[] = $idSite; + } + + $listSiteIds = implode(',', $siteIdsToInvalidate); try { $this->logger->info(' Will invalidate archived reports for ' . $date . ' for following websites ids: ' . $listSiteIds); - $this->getApiToInvalidateArchivedReport()->invalidateArchivedReports($siteIds, $date); + $this->getApiToInvalidateArchivedReport()->invalidateArchivedReports($siteIdsToInvalidate, $date); } catch (Exception $e) { $this->logger->info(' Failed to invalidate archived reports: ' . $e->getMessage()); } @@ -975,8 +803,6 @@ class CronArchive } } - Db::fetchAll("SELECT idinvalidation, idarchive, idsite, date1, date2, period, name, status FROM " . Common::prefixTable('archive_invalidations')); - $this->setInvalidationTime(); $this->logger->info("Done invalidating"); @@ -997,7 +823,7 @@ class CronArchive $loader = new Loader($params); if ($loader->canSkipThisArchive()) { - $this->logger->debug(" " . ucfirst($dateStr) . " archive can be skipped due to no visits, skipping invalidation..."); + $this->logger->debug(" " . ucfirst($dateStr) . " archive can be skipped due to no visits for idSite = $idSite, skipping invalidation..."); continue; } @@ -1010,15 +836,15 @@ class CronArchive } } - private function isThereExistingValidPeriod(Parameters $params, $isYesterday = false) + public function isThereExistingValidPeriod(Parameters $params, $isYesterday = false) { - $today = Date::factory('today'); + $today = Date::factoryInTimezone('today', Site::getTimezoneFor($params->getSite()->getId())); $isPeriodIncludesToday = $params->getPeriod()->isDateInPeriod($today); $minArchiveProcessedTime = $isPeriodIncludesToday ? Date::now()->subSeconds(Rules::getPeriodArchiveTimeToLiveDefault($params->getPeriod()->getLabel())) : null; // empty plugins param since we only check for an 'all' archive - list($idArchive, $visits, $visitsConverted, $ignore, $tsArchived) = ArchiveSelector::getArchiveIdAndVisits($params, $minArchiveProcessedTime, $includeInvalidated = false); + list($idArchive, $visits, $visitsConverted, $ignore, $tsArchived) = ArchiveSelector::getArchiveIdAndVisits($params, $minArchiveProcessedTime, $includeInvalidated = $isPeriodIncludesToday); // day has changed since the archive was created, we need to reprocess it if ($isYesterday @@ -1096,7 +922,7 @@ class CronArchive $this->logger->info(" See the doc at: https://matomo.org/docs/setup-auto-archiving/"); } - $cliMulti = new CliMulti(); + $cliMulti = new CliMulti($this->logger); $supportsAsync = $cliMulti->supportsAsync(); $this->logger->info("- " . ($supportsAsync ? 'Async process archiving supported, using CliMulti.' : 'Async process archiving not supported, using curl requests.')); @@ -1255,7 +1081,7 @@ class CronArchive private function makeCliMulti() { /** @var CliMulti $cliMulti */ - $cliMulti = StaticContainer::getContainer()->make('Piwik\CliMulti'); + $cliMulti = new CliMulti($this->logger); $cliMulti->setUrlToPiwik($this->urlToPiwik); $cliMulti->setPhpCliConfigurationOptions($this->phpCliConfigurationOptions); $cliMulti->setAcceptInvalidSSLCertificate($this->acceptInvalidSSLCertificate); @@ -1321,15 +1147,6 @@ class CronArchive return false; } - private function shouldSkipArchive($archive) - { - if ($this->archiveFilter) { - return $this->archiveFilter->filterArchive($archive); - } - - return false; - } - protected function wasSegmentChangedRecently($definition, $allSegments) { foreach ($allSegments as $segment) { @@ -1354,19 +1171,6 @@ class CronArchive $this->archiveFilter = $archiveFilter; } - private function addInvalidationToExclude(array $invalidatedArchive) - { - $id = $invalidatedArchive['idinvalidation']; - if (empty($this->invalidationsToExclude[$id])) { - $this->invalidationsToExclude[$id] = $id; - } - } - - private function getNextIdSiteToArchive() - { - return $this->websiteIdArchiveList->getNextSiteId(); - } - private function makeWebsiteIdArchiveList(array $websitesIds) { if ($this->shouldArchiveAllSites) { @@ -1381,117 +1185,4 @@ class CronArchive return new SharedSiteIds($websitesIds, SharedSiteIds::OPTION_ALL_WEBSITES); } - - private function hasDifferentPeriod(array $archivesToProcess, $period) - { - if (empty($archivesToProcess)) { - return false; - } - - return $archivesToProcess[0]['period'] != $period; - } - - private function hasDifferentDoneFlagType(array $archivesToProcess, $name) - { - if (empty($archivesToProcess)) { - return false; - } - - $existingDoneFlagType = $this->getDoneFlagType($archivesToProcess[0]['name']); - $newArchiveDoneFlagType = $this->getDoneFlagType($name); - - return $existingDoneFlagType != $newArchiveDoneFlagType; - } - - private function getDoneFlagType($name) - { - if ($name == 'done') { - return 'all'; - } else { - return 'segment'; - } - } - - private function canSkipArchiveBecauseNoPoint(array $invalidatedArchive) - { - $site = new Site($invalidatedArchive['idsite']); - - $periodLabel = $this->periodIdsToLabels[$invalidatedArchive['period']]; - $dateStr = $periodLabel == 'range' ? ($invalidatedArchive['date1'] . ',' . $invalidatedArchive['date2']) : $invalidatedArchive['date1']; - $period = PeriodFactory::build($periodLabel, $dateStr); - - $segment = new Segment($invalidatedArchive['segment'], [$invalidatedArchive['idsite']], $period->getDateStart(), $period->getDateEnd()); - - $params = new Parameters($site, $period, $segment); - - $loader = new Loader($params); - if ($loader->canSkipThisArchive()) { // if no point in archiving, skip - return true; - } - - // if valid archive already exists, do not re-archive - $minDateTimeProcessedUTC = Date::now()->subSeconds(Rules::getPeriodArchiveTimeToLiveDefault($periodLabel)); - $archiveIdAndVisits = ArchiveSelector::getArchiveIdAndVisits($params, $minDateTimeProcessedUTC, $includeInvalidated = false); - - $idArchive = $archiveIdAndVisits[0]; - return !empty($idArchive); - } - - private function shouldSkipArchiveBecauseLowerPeriodOrSegmentIsInProgress(array $archiveToProcess) - { - $inProgressArchives = $this->cliMultiRequestParser->getInProgressArchivingCommands(); - - $periodLabel = $this->periodIdsToLabels[$archiveToProcess['period']]; - $archiveToProcess['periodObj'] = PeriodFactory::build($periodLabel, $archiveToProcess['date1']); - - foreach ($inProgressArchives as $archiveBeingProcessed) { - if (empty($archiveBeingProcessed['period']) - || empty($archiveBeingProcessed['date']) - ) { - continue; - } - - $archiveBeingProcessed['periodObj'] = PeriodFactory::build($archiveBeingProcessed['period'], $archiveBeingProcessed['date']); - - if ($this->isArchiveOfLowerPeriod($archiveToProcess, $archiveBeingProcessed)) { - return "lower period in progress (period = {$archiveBeingProcessed['period']}, date = {$archiveBeingProcessed['date']})"; - } - - if ($this->isArchiveNonSegmentAndInProgressArchiveSegment($archiveToProcess, $archiveBeingProcessed)) { - return "segment archive in progress for same site/period ({$archiveBeingProcessed['segment']})"; - } - } - - return false; - } - - private function isArchiveOfLowerPeriod(array $archiveToProcess, $archiveBeingProcessed) - { - /** @var Period $archiveToProcessPeriodObj */ - $archiveToProcessPeriodObj = $archiveToProcess['periodObj']; - /** @var Period $archivePeriodObj */ - $archivePeriodObj = $archiveBeingProcessed['periodObj']; - - if ($archiveToProcessPeriodObj->getId() >= $archivePeriodObj->getId() - && $archiveToProcessPeriodObj->isPeriodIntersectingWith($archivePeriodObj) - ) { - return true; - } - - return false; - } - - private function isArchiveNonSegmentAndInProgressArchiveSegment(array $archiveToProcess, array $archiveBeingProcessed) - { - // archive is for different site/period - if (empty($archiveBeingProcessed['idSite']) - || $archiveToProcess['idsite'] != $archiveBeingProcessed['idSite'] - || $archiveToProcess['periodObj']->getId() != $archiveBeingProcessed['periodObj']->getId() - || $archiveToProcess['periodObj']->getDateStart()->toString() != $archiveBeingProcessed['periodObj']->getDateStart()->toString() - ) { - return false; - } - - return empty($archiveToProcess['segment']) && !empty($archiveBeingProcessed['segment']); - } -}
\ No newline at end of file +} diff --git a/core/CronArchive/QueueConsumer.php b/core/CronArchive/QueueConsumer.php new file mode 100644 index 0000000000..2d64701ef1 --- /dev/null +++ b/core/CronArchive/QueueConsumer.php @@ -0,0 +1,544 @@ +<?php +/** + * Matomo - free/libre analytics platform + * + * @link https://matomo.org + * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later + * + */ + +namespace Piwik\CronArchive; + + +use Piwik\ArchiveProcessor\Loader; +use Piwik\ArchiveProcessor\Parameters; +use Piwik\ArchiveProcessor\Rules; +use Piwik\CliMulti\RequestParser; +use Piwik\CronArchive; +use Piwik\DataAccess\ArchiveSelector; +use Piwik\DataAccess\Model; +use Piwik\Date; +use Piwik\Period; +use Piwik\Period\Factory as PeriodFactory; +use Piwik\Piwik; +use Piwik\Plugin\Manager; +use Piwik\Segment; +use Piwik\Site; +use Piwik\Timer; +use Psr\Log\LoggerInterface; + +class QueueConsumer +{ + /** + * @var LoggerInterface + */ + private $logger; + + /** + * @var FixedSiteIds|SharedSiteIds + */ + private $websiteIdArchiveList; + + /** + * @var int + */ + private $countOfProcesses; + + /** + * @var int + */ + private $pid; + + /** + * @var Model + */ + private $model; + + /** + * @var ArchiveFilter + */ + private $archiveFilter; + + /** + * @var SegmentArchiving + */ + private $segmentArchiving; + + /** + * @var CronArchive + */ + private $cronArchive; + + /** + * @var array + */ + private $invalidationsToExclude; + + /** + * @var string[] + */ + private $periodIdsToLabels; + + /** + * @var RequestParser + */ + private $cliMultiRequestParser; + + /** + * @var int + */ + private $idSite; + + /** + * @var int + */ + private $siteRequests; + + /** + * @var Timer + */ + private $siteTimer; + + public function __construct(LoggerInterface $logger, $websiteIdArchiveList, $countOfProcesses, $pid, Model $model, + SegmentArchiving $segmentArchiving, CronArchive $cronArchive, RequestParser $cliMultiRequestParser, + ArchiveFilter $archiveFilter = null) + { + $this->logger = $logger; + $this->websiteIdArchiveList = $websiteIdArchiveList; + $this->countOfProcesses = $countOfProcesses; + $this->pid = $pid; + $this->model = $model; + $this->segmentArchiving = $segmentArchiving; + $this->cronArchive = $cronArchive; + $this->cliMultiRequestParser = $cliMultiRequestParser; + $this->archiveFilter = $archiveFilter; + + // if we skip or can't process an idarchive, we want to ignore it the next time we look for an invalidated + // archive. these IDs are stored here (using a list like this serves to keep our SQL simple). + $this->invalidationsToExclude = []; + + $this->periodIdsToLabels = array_flip(Piwik::$idPeriods); + } + + public function getNextArchivesToProcess() + { + if (empty($this->idSite)) { + $this->idSite = $this->getNextIdSiteToArchive(); + if (empty($this->idSite)) { // no sites left to archive, stop + $this->logger->debug("No more sites left to archive, stopping."); + return null; + } + + /** + * This event is triggered before the cron archiving process starts archiving data for a single + * site. + * + * Note: multiple archiving processes can post this event. + * + * @param int $idSite The ID of the site we're archiving data for. + * @param string $pid The PID of the process processing archives for this site. + */ + Piwik::postEvent('CronArchive.archiveSingleSite.start', array($this->idSite, $this->pid)); + + $this->logger->info("Start processing archives for site {idSite}.", ['idSite' => $this->idSite]); + + $this->siteTimer = new Timer(); + $this->siteRequests = 0; + } + + // we don't want to invalidate different periods together or segment archives w/ no-segment archives + // together, but it's possible to end up querying these archives. if we find one, we keep track of it + // in this array to exclude, but after we run the current batch, we reset the array so we'll still + // process them eventually. + $invalidationsToExcludeInBatch = []; + + $siteCreationTime = Date::factory(Site::getCreationDateFor($this->idSite)); + + // get archives to process simultaneously + $archivesToProcess = []; + while (count($archivesToProcess) < $this->countOfProcesses) { + $invalidatedArchive = $this->getNextInvalidatedArchive($this->idSite, array_keys($invalidationsToExcludeInBatch)); + if (empty($invalidatedArchive)) { + $this->logger->debug("No next invalidated archive."); + break; + } + + $invalidationDesc = $this->getInvalidationDescription($invalidatedArchive); + + if ($invalidatedArchive['periodObj']->getDateEnd()->isEarlier($siteCreationTime)) { + $this->logger->debug("Invalidation is for period that is older than the site's creation time, ignoring: $invalidationDesc"); + $this->model->deleteInvalidations([$invalidatedArchive]); + continue; + } + + if (!empty($invalidatedArchive['plugin']) + && !Manager::getInstance()->isPluginActivated($invalidatedArchive['plugin']) + ) { + $this->logger->debug("Plugin specific archive {$invalidatedArchive['idarchive']}'s plugin is deactivated, ignoring $invalidationDesc."); + $this->model->deleteInvalidations([$invalidatedArchive]); + continue; + } + + if ($this->hasDifferentDoneFlagType($archivesToProcess, $invalidatedArchive['name'])) { + $this->logger->debug("Found archive with different done flag type (segment vs. no segment) in concurrent batch, skipping until next batch: $invalidationDesc"); + + $idinvalidation = $invalidatedArchive['idinvalidation']; + $invalidationsToExcludeInBatch[$idinvalidation] = true; + + continue; + } + + if ($invalidatedArchive['segment'] === null) { + $this->logger->debug("Found archive for segment that is not auto archived, ignoring: $invalidationDesc"); + $this->addInvalidationToExclude($invalidatedArchive); + continue; + } + + if ($this->archiveArrayContainsArchive($archivesToProcess, $invalidatedArchive)) { + $this->logger->debug("Found duplicate invalidated archive {$invalidatedArchive['idarchive']}, ignoring: $invalidationDesc"); + $this->addInvalidationToExclude($invalidatedArchive); + $this->model->deleteInvalidations([$invalidatedArchive]); + continue; + } + + if ($this->hasIntersectingPeriod($archivesToProcess, $invalidatedArchive)) { + $this->logger->debug("Found archive with intersecting period with others in concurrent batch, skipping until next batch: $invalidationDesc"); + + $idinvalidation = $invalidatedArchive['idinvalidation']; + $invalidationsToExcludeInBatch[$idinvalidation] = true; + continue; + } + + $reason = $this->shouldSkipArchive($invalidatedArchive); + if ($reason) { + $this->logger->debug("Skipping invalidated archive {$invalidatedArchive['idinvalidation']}, $reason: $invalidationDesc"); + $this->addInvalidationToExclude($invalidatedArchive); + continue; + } + + if ($this->usableArchiveExists($invalidatedArchive)) { + $this->logger->debug("Found invalidation with usable archive (not yet outdated) skipping until archive is out of date: $invalidationDesc"); + $this->addInvalidationToExclude($invalidatedArchive); + continue; + } + + if ($this->canSkipArchiveBecauseNoPoint($invalidatedArchive)) { + $this->logger->debug("Found invalidated archive we can skip (no visits): $invalidationDesc"); + $this->addInvalidationToExclude($invalidatedArchive); + $this->model->deleteInvalidations([$invalidatedArchive]); + continue; + } + + // TODO: should use descriptive string instead of just invalidation ID + $reason = $this->shouldSkipArchiveBecauseLowerPeriodOrSegmentIsInProgress($invalidatedArchive); + if ($reason) { + $this->logger->debug("Skipping invalidated archive, $reason: $invalidationDesc"); + $invalidationsToExcludeInBatch[$invalidatedArchive['idinvalidation']] = true; + $this->addInvalidationToExclude($invalidatedArchive); + continue; + } + + $started = $this->model->startArchive($invalidatedArchive); + if (!$started) { // another process started on this archive, pull another one + $this->logger->debug("Archive invalidation is being handled by another process: $invalidationDesc"); + $this->addInvalidationToExclude($invalidatedArchive); + continue; + } + + $this->addInvalidationToExclude($invalidatedArchive); + + $this->logger->debug("Processing invalidation: $invalidationDesc."); + + $archivesToProcess[] = $invalidatedArchive; + } + + if (empty($archivesToProcess) + && empty($invalidationsToExcludeInBatch) + ) { // no invalidated archive left + /** + * This event is triggered immediately after the cron archiving process starts archiving data for a single + * site. + * + * Note: multiple archiving processes can post this event. + * + * @param int $idSite The ID of the site we're archiving data for. + * @param string $pid The PID of the process processing archives for this site. + */ + Piwik::postEvent('CronArchive.archiveSingleSite.finish', array($this->idSite, $this->pid)); + + $this->logger->info("Finished archiving for site {idSite}, {requests} API requests, {timer} [{processed} / {totalNum} done]", [ + 'idSite' => $this->idSite, + 'processed' => $this->websiteIdArchiveList->getNumProcessedWebsites(), + 'totalNum' => $this->websiteIdArchiveList->getNumSites(), + 'timer' => $this->siteTimer, + 'requests' => $this->siteRequests, + ]); + + $this->idSite = null; + } + + $this->siteRequests += count($archivesToProcess); + + return $archivesToProcess; + } + + private function archiveArrayContainsArchive($archiveArray, $archive) + { + foreach ($archiveArray as $entry) { + if ($entry['idsite'] == $archive['idsite'] + && $entry['period'] == $archive['period'] + && $entry['date1'] == $archive['date1'] + && $entry['date2'] == $archive['date2'] + && $entry['name'] == $archive['name'] + && $entry['plugin'] == $archive['plugin'] + && $entry['report'] == $archive['report'] + ) { + return true; + } + } + return false; + } + + private function getNextInvalidatedArchive($idSite, $extraInvalidationsToIgnore) + { + $lastInvalidationTime = CronArchive::getLastInvalidationTime(); + if (empty($lastInvalidationTime) + || (time() - $lastInvalidationTime) >= 3600 + ) { + $this->cronArchive->invalidateArchivedReportsForSitesThatNeedToBeArchivedAgain(); + } + + $iterations = 0; + while ($iterations < 100) { + $invalidationsToExclude = array_merge($this->invalidationsToExclude, $extraInvalidationsToIgnore); + + $nextArchive = $this->model->getNextInvalidatedArchive($idSite, $invalidationsToExclude); + if (empty($nextArchive)) { + break; + } + + $this->detectPluginForArchive($nextArchive); + + $periodLabel = $this->periodIdsToLabels[$nextArchive['period']]; + $periodDate = $periodLabel == 'range' ? $nextArchive['date1'] . ',' . $nextArchive['date2'] : $nextArchive['date1']; + $nextArchive['periodObj'] = PeriodFactory::build($periodLabel, $periodDate); + + $isCronArchivingEnabled = $this->findSegmentForArchive($nextArchive); + if ($isCronArchivingEnabled) { + return $nextArchive; + } + + $this->logger->debug("Found invalidation for segment that does not have auto archiving enabled, skipping: {$nextArchive['idinvalidation']}"); + $this->invalidationsToExclude[] = $nextArchive['idinvalidation']; + + ++$iterations; + } + + return null; + } + + private function shouldSkipArchive($archive) + { + if ($this->archiveFilter) { + return $this->archiveFilter->filterArchive($archive); + } + + return false; + } + + // public for tests + public function canSkipArchiveBecauseNoPoint(array $invalidatedArchive) + { + $site = new Site($invalidatedArchive['idsite']); + + $periodLabel = $this->periodIdsToLabels[$invalidatedArchive['period']]; + $dateStr = $periodLabel == 'range' ? ($invalidatedArchive['date1'] . ',' . $invalidatedArchive['date2']) : $invalidatedArchive['date1']; + $period = PeriodFactory::build($periodLabel, $dateStr); + + $segment = new Segment($invalidatedArchive['segment'], [$invalidatedArchive['idsite']]); + + $params = new Parameters($site, $period, $segment); + + $loader = new Loader($params); + return $loader->canSkipThisArchive(); // if no point in archiving, skip + } + + private function shouldSkipArchiveBecauseLowerPeriodOrSegmentIsInProgress(array $archiveToProcess) + { + $inProgressArchives = $this->cliMultiRequestParser->getInProgressArchivingCommands(); + + foreach ($inProgressArchives as $archiveBeingProcessed) { + if (empty($archiveBeingProcessed['period']) + || empty($archiveBeingProcessed['date']) + ) { + continue; + } + + $archiveBeingProcessed['periodObj'] = PeriodFactory::build($archiveBeingProcessed['period'], $archiveBeingProcessed['date']); + + if ($this->isArchiveOfLowerPeriod($archiveToProcess, $archiveBeingProcessed)) { + return "lower period in progress (period = {$archiveBeingProcessed['period']}, date = {$archiveBeingProcessed['date']})"; + } + + if ($this->isArchiveNonSegmentAndInProgressArchiveSegment($archiveToProcess, $archiveBeingProcessed)) { + return "segment archive in progress for same site/period ({$archiveBeingProcessed['segment']})"; + } + } + + return false; + } + + private function isArchiveOfLowerPeriod(array $archiveToProcess, $archiveBeingProcessed) + { + /** @var Period $archiveToProcessPeriodObj */ + $archiveToProcessPeriodObj = $archiveToProcess['periodObj']; + /** @var Period $archivePeriodObj */ + $archivePeriodObj = $archiveBeingProcessed['periodObj']; + + if ($archiveToProcessPeriodObj->getId() >= $archivePeriodObj->getId() + && $archiveToProcessPeriodObj->isPeriodIntersectingWith($archivePeriodObj) + ) { + return true; + } + + return false; + } + + private function isArchiveNonSegmentAndInProgressArchiveSegment(array $archiveToProcess, array $archiveBeingProcessed) + { + // archive is for different site/period + if (empty($archiveBeingProcessed['idSite']) + || $archiveToProcess['idsite'] != $archiveBeingProcessed['idSite'] + || $archiveToProcess['periodObj']->getId() != $archiveBeingProcessed['periodObj']->getId() + || $archiveToProcess['periodObj']->getDateStart()->toString() != $archiveBeingProcessed['periodObj']->getDateStart()->toString() + ) { + return false; + } + + return empty($archiveToProcess['segment']) && !empty($archiveBeingProcessed['segment']); + } + + private function detectPluginForArchive(&$archive) + { + $archive['plugin'] = $this->getPluginNameForArchiveIfAny($archive); + } + + private function hasIntersectingPeriod(array $archivesToProcess, $invalidatedArchive) + { + if (empty($archivesToProcess)) { + return false; + } + + foreach ($archivesToProcess as $archive) { + if ($archive['periodObj']->isPeriodIntersectingWith($invalidatedArchive['periodObj'])) { + return true; + } + } + + return false; + } + + private function findSegmentForArchive(&$archive) + { + $flag = explode('.', $archive['name'])[0]; + if ($flag == 'done') { + $archive['segment'] = ''; + return true; + } + + $hash = substr($flag, 4); + $storedSegment = $this->segmentArchiving->findSegmentForHash($hash, $archive['idsite']); + if (!isset($storedSegment['definition'])) { + $archive['segment'] = null; + return false; + } + + $archive['segment'] = $storedSegment['definition']; + return $this->segmentArchiving->isAutoArchivingEnabledFor($storedSegment); + } + + private function hasDifferentDoneFlagType(array $archivesToProcess, $name) + { + if (empty($archivesToProcess)) { + return false; + } + + $existingDoneFlagType = $this->getDoneFlagType($archivesToProcess[0]['name']); + $newArchiveDoneFlagType = $this->getDoneFlagType($name); + + return $existingDoneFlagType != $newArchiveDoneFlagType; + } + + private function getPluginNameForArchiveIfAny($archive) + { + $name = $archive['name']; + if (strpos($name, '.') === false) { + return null; + } + + $parts = explode('.', $name); + return $parts[1]; + } + + private function getDoneFlagType($name) + { + if ($name == 'done') { + return 'all'; + } else { + return 'segment'; + } + } + + private function addInvalidationToExclude(array $invalidatedArchive) + { + $id = $invalidatedArchive['idinvalidation']; + if (empty($this->invalidationsToExclude[$id])) { + $this->invalidationsToExclude[$id] = $id; + } + } + + private function getNextIdSiteToArchive() + { + return $this->websiteIdArchiveList->getNextSiteId(); + } + + private function getInvalidationDescription(array $invalidatedArchive) + { + return sprintf("[idinvalidation = %s, idsite = %s, period = %s(%s - %s), name = %s]", + $invalidatedArchive['idinvalidation'], + $invalidatedArchive['idsite'], + $this->periodIdsToLabels[$invalidatedArchive['period']], + $invalidatedArchive['date1'], + $invalidatedArchive['date2'], + $invalidatedArchive['name'] + ); + } + + // public for test + public function usableArchiveExists(array $invalidatedArchive) + { + $site = new Site($invalidatedArchive['idsite']); + + $periodLabel = $this->periodIdsToLabels[$invalidatedArchive['period']]; + $dateStr = $periodLabel == 'range' ? ($invalidatedArchive['date1'] . ',' . $invalidatedArchive['date2']) : $invalidatedArchive['date1']; + $period = PeriodFactory::build($periodLabel, $dateStr); + + $segment = new Segment($invalidatedArchive['segment'], [$invalidatedArchive['idsite']]); + + $params = new Parameters($site, $period, $segment); + + // if latest archive includes today and is usable (DONE_OK or DONE_INVALIDATED and recent enough), skip + $today = Date::factoryInTimezone('today', Site::getTimezoneFor($site->getId()))->subSeconds(1); + $isArchiveIncludesToday = $period->isDateInPeriod($today); + if (!$isArchiveIncludesToday) { + return false; + } + + // if valid archive already exists, do not re-archive + $minDateTimeProcessedUTC = Date::now()->subSeconds(Rules::getPeriodArchiveTimeToLiveDefault($periodLabel)); + $archiveIdAndVisits = ArchiveSelector::getArchiveIdAndVisits($params, $minDateTimeProcessedUTC, $includeInvalidated = true); + + $idArchive = $archiveIdAndVisits[0]; + return !empty($idArchive); + } +}
\ No newline at end of file diff --git a/core/CronArchive/SegmentArchiving.php b/core/CronArchive/SegmentArchiving.php index 61418cd9bf..ba2c2a4790 100644 --- a/core/CronArchive/SegmentArchiving.php +++ b/core/CronArchive/SegmentArchiving.php @@ -31,7 +31,7 @@ class SegmentArchiving const BEGINNING_OF_TIME = 'beginning_of_time'; const CREATION_TIME = 'segment_creation_time'; const LAST_EDIT_TIME = 'segment_last_edit_time'; - const DEFAULT_BEGINNIN_OF_TIME_LAST_N_YEARS = 7; + const DEFAULT_BEGINNING_OF_TIME_LAST_N_YEARS = 7; /** * @var Model @@ -65,7 +65,7 @@ class SegmentArchiving */ private $forceArchiveAllSegments; - public function __construct($processNewSegmentsFrom, $beginningOfTimeLastNInYears = self::DEFAULT_BEGINNIN_OF_TIME_LAST_N_YEARS, + public function __construct($processNewSegmentsFrom, $beginningOfTimeLastNInYears = self::DEFAULT_BEGINNING_OF_TIME_LAST_N_YEARS, Model $segmentEditorModel = null, Cache $segmentListCache = null, Date $now = null, LoggerInterface $logger = null) { @@ -80,6 +80,11 @@ class SegmentArchiving public function getSegmentArchivesToInvalidateForNewSegments($idSite) { + return $this->getSegmentArchivesToInvalidate($idSite, true); + } + + public function getSegmentArchivesToInvalidate($idSite, $checkOnlyForNewSegments = false) + { $result = []; $segmentsForSite = $this->getAllSegments(); @@ -88,7 +93,7 @@ class SegmentArchiving continue; } - $oldestDateToProcessForNewSegment = $this->getOldestDateToProcessForNewSegment($idSite, $storedSegment); + $oldestDateToProcessForNewSegment = $this->getOldestDateToProcessForNewSegment($idSite, $storedSegment, $checkOnlyForNewSegments); if (empty($oldestDateToProcessForNewSegment)) { continue; } @@ -128,7 +133,7 @@ class SegmentArchiving return null; } - private function getOldestDateToProcessForNewSegment($idSite, $storedSegment) + private function getOldestDateToProcessForNewSegment($idSite, $storedSegment, $checkOnlyForNewSegments) { /** * @var Date $segmentCreatedTime @@ -145,11 +150,13 @@ class SegmentArchiving } $segmentTimeToUse = $segmentLastEditedTime ?: $segmentCreatedTime; - if (!empty($lastInvalidationTime) - && !empty($segmentTimeToUse) - && $segmentTimeToUse->isEarlier($lastInvalidationTime) - ) { - return null; // has already have been invalidated, ignore + if ($checkOnlyForNewSegments) { + if (!empty($lastInvalidationTime) + && !empty($segmentTimeToUse) + && $segmentTimeToUse->isEarlier($lastInvalidationTime) + ) { + return null; // has already have been invalidated, ignore + } } if ($this->processNewSegmentsFrom == self::CREATION_TIME) { diff --git a/core/DataAccess/ArchiveSelector.php b/core/DataAccess/ArchiveSelector.php index cb33bad34f..493e208fe6 100644 --- a/core/DataAccess/ArchiveSelector.php +++ b/core/DataAccess/ArchiveSelector.php @@ -57,7 +57,7 @@ class ArchiveSelector * - the ts_archived for the latest usable archive * @throws Exception */ - public static function getArchiveIdAndVisits(ArchiveProcessor\Parameters $params, $minDatetimeArchiveProcessedUTC = false, $includeInvalidated = true) + public static function getArchiveIdAndVisits(ArchiveProcessor\Parameters $params, $minDatetimeArchiveProcessedUTC = false, $includeInvalidated = null) { $idSite = $params->getSite()->getId(); $period = $params->getPeriod()->getId(); @@ -73,7 +73,7 @@ class ArchiveSelector $doneFlags = Rules::getDoneFlags($plugins, $segment); $requestedPluginDoneFlags = Rules::getDoneFlags([$requestedPlugin], $segment); - $doneFlagValues = Rules::getSelectableDoneFlagValues($includeInvalidated, $params); + $doneFlagValues = Rules::getSelectableDoneFlagValues($includeInvalidated === null ? true : $includeInvalidated, $params, $includeInvalidated === null); $results = self::getModel()->getArchiveIdAndVisits($numericTable, $idSite, $period, $dateStartIso, $dateEndIso, null, $doneFlags); if (empty($results)) { // no archive found @@ -86,6 +86,11 @@ class ArchiveSelector $visits = isset($result['nb_visits']) ? $result['nb_visits'] : false; $visitsConverted = isset($result['nb_visits_converted']) ? $result['nb_visits_converted'] : false; + $result['idarchive'] = empty($result['idarchive']) ? [] : [$result['idarchive']]; + if (isset($result['partial'])) { + $result['idarchive'] = array_merge($result['idarchive'], $result['partial']); + } + if (isset($result['value']) && !in_array($result['value'], $doneFlagValues) ) { // the archive cannot be considered valid for this request (has wrong done flag value) @@ -96,25 +101,17 @@ class ArchiveSelector $minDatetimeArchiveProcessedUTC = Date::factory($minDatetimeArchiveProcessedUTC); } - if (!empty($minDatetimeArchiveProcessedUTC) && !is_object($minDatetimeArchiveProcessedUTC)) { - $minDatetimeArchiveProcessedUTC = Date::factory($minDatetimeArchiveProcessedUTC); - } - - if (!empty($minDatetimeArchiveProcessedUTC) && !is_object($minDatetimeArchiveProcessedUTC)) { - $minDatetimeArchiveProcessedUTC = Date::factory($minDatetimeArchiveProcessedUTC); - } - // the archive is too old if ($minDatetimeArchiveProcessedUTC - && isset($result['idarchive']) + && !empty($result['idarchive']) && Date::factory($tsArchived)->isEarlier($minDatetimeArchiveProcessedUTC) ) { return [false, $visits, $visitsConverted, true, $tsArchived]; } - $idArchive = isset($result['idarchive']) ? $result['idarchive'] : false; + $idArchives = !empty($result['idarchive']) ? $result['idarchive'] : false; - return [$idArchive, $visits, $visitsConverted, true, $tsArchived]; + return [$idArchives, $visits, $visitsConverted, true, $tsArchived]; } /** @@ -264,7 +261,8 @@ class ArchiveSelector $getValuesSql = "SELECT value, name, idsite, date1, date2, ts_archived FROM %s WHERE idarchive IN (%s) - AND " . $whereNameIs; + AND " . $whereNameIs . " + ORDER BY ts_archived ASC"; // ascending order so we use the latest data found // get data from every table we're querying $rows = array(); @@ -381,10 +379,12 @@ class ArchiveSelector { // find latest idarchive for each done flag $idArchives = []; + $tsArchiveds = []; foreach ($results as $row) { $doneFlag = $row['name']; - if (!isset($idArchives[$doneFlag])) { + if (!isset($idArchives[$doneFlag]) && $row['value'] != ArchiveWriter::DONE_PARTIAL) { $idArchives[$doneFlag] = $row['idarchive']; + $tsArchiveds[$doneFlag] = $row['ts_archived']; } } @@ -393,7 +393,7 @@ class ArchiveSelector self::NB_VISITS_CONVERTED_RECORD_LOOKED_UP => false, ]; - foreach ($results as &$result) { + foreach ($results as $result) { if (in_array($result['name'], $requestedPluginDoneFlags) && in_array($result['idarchive'], $idArchives) ) { @@ -422,6 +422,22 @@ class ArchiveSelector } } + // add partial archives + foreach ($results as $row) { + if (!isset($idArchives[$row['name']])) { + continue; + } + + $mainTsArchived = $tsArchiveds[$row['name']]; + $thisTsArchived = $row['ts_archived']; + + if ($row['value'] === ArchiveWriter::DONE_PARTIAL + && ($mainTsArchived == $thisTsArchived || Date::factory($mainTsArchived)->isEarlier($thisTsArchived)) + ) { + $idArchives['partial'][] = $row['idarchive']; + } + } + return $archiveData; } } diff --git a/core/DataAccess/ArchiveWriter.php b/core/DataAccess/ArchiveWriter.php index a8446db9b4..49d8a52104 100644 --- a/core/DataAccess/ArchiveWriter.php +++ b/core/DataAccess/ArchiveWriter.php @@ -12,6 +12,7 @@ use Exception; use Piwik\Archive\Chunk; use Piwik\ArchiveProcessor\Rules; use Piwik\ArchiveProcessor; +use Piwik\Date; use Piwik\Db; use Piwik\Db\BatchInsert; @@ -56,12 +57,11 @@ class ArchiveWriter const DONE_INVALIDATED = 4; /** - * Flag indicating that the archive is currently being archived. If the archiving process is aborted or killed, the - * archive may remain w/ this flag. + * Flag indicating that the archive is * * @var int */ - const DONE_IN_PROGRESS = 5; + const DONE_PARTIAL = 5; protected $fields = array('idarchive', 'idsite', @@ -80,6 +80,16 @@ class ArchiveWriter const MAX_SPOOL_SIZE = 50; /** + * @var ArchiveProcessor\Parameters + */ + private $parameters; + + /** + * @var string + */ + private $earliestNow; + + /** * ArchiveWriter constructor. * @param ArchiveProcessor\Parameters $params * @param bool $isArchiveTemporary Deprecated. Has no effect. @@ -91,6 +101,7 @@ class ArchiveWriter $this->idSite = $params->getSite()->getId(); $this->segment = $params->getSegment(); $this->period = $params->getPeriod(); + $this->parameters = $params; $idSites = array($this->idSite); $this->doneFlag = Rules::getDoneStringFlagFor($idSites, $this->segment, $this->period->getLabel(), $params->getRequestedPlugin()); @@ -141,8 +152,9 @@ class ArchiveWriter public function initNewArchive() { - $this->allocateNewArchiveId(); + $idArchive = $this->allocateNewArchiveId(); $this->logArchiveStatusAsIncomplete(); + return $idArchive; } public function finalizeArchive() @@ -152,7 +164,15 @@ class ArchiveWriter $numericTable = $this->getTableNumeric(); $idArchive = $this->getIdArchive(); - $this->getModel()->updateArchiveStatus($numericTable, $idArchive, $this->doneFlag, self::DONE_OK); + $doneValue = $this->parameters->isPartialArchive() ? self::DONE_PARTIAL : self::DONE_OK; + $this->getModel()->updateArchiveStatus($numericTable, $idArchive, $this->doneFlag, $doneValue); + + if (!$this->parameters->isPartialArchive() + // sanity check, just in case nothing was inserted (the archive status should always be inserted) + && !empty($this->earliestNow) + ) { + $this->getModel()->deleteOlderArchives($this->parameters, $this->doneFlag, $this->earliestNow, $this->idArchive); + } } protected function compress($data) @@ -273,12 +293,16 @@ class ArchiveWriter protected function getInsertRecordBind() { + $now = Date::now()->getDatetime(); + if (empty($this->earliestNow)) { + $this->earliestNow = $now; + } return array($this->getIdArchive(), $this->idSite, $this->dateStart->toString('Y-m-d'), $this->period->getDateEnd()->toString('Y-m-d'), $this->period->getId(), - date("Y-m-d H:i:s")); + $now); } protected function getTableNameToInsert($value) diff --git a/core/DataAccess/LogAggregator.php b/core/DataAccess/LogAggregator.php index 860ab7bbc8..7697f6b9e4 100644 --- a/core/DataAccess/LogAggregator.php +++ b/core/DataAccess/LogAggregator.php @@ -166,6 +166,11 @@ class LogAggregator private $allowUsageSegmentCache = false; /** + * @var Parameters + */ + private $params; + + /** * Constructor. * * @param \Piwik\ArchiveProcessor\Parameters $params @@ -178,6 +183,7 @@ class LogAggregator $this->sites = $params->getIdSites(); $this->isRootArchiveRequest = $params->isRootArchiveRequest(); $this->logger = $logger ?: StaticContainer::get('Psr\Log\LoggerInterface'); + $this->params = $params; } public function setSites($sites) @@ -328,7 +334,7 @@ class LogAggregator if (!$this->segment->isEmpty() && $this->isSegmentCacheEnabled()) { // here we create the TMP table and apply the segment including the datetime and the requested idsite // at the end we generated query will no longer need to apply the datetime/idsite and segment - $segment = new Segment('', $this->sites); + $segment = new Segment('', $this->sites, $this->params->getPeriod()->getDateTimeStart(), $this->params->getPeriod()->getDateTimeEnd()); $segmentTable = $this->getSegmentTmpTableName(); diff --git a/core/DataAccess/Model.php b/core/DataAccess/Model.php index 7a9a322092..6faaeb05ed 100644 --- a/core/DataAccess/Model.php +++ b/core/DataAccess/Model.php @@ -21,6 +21,7 @@ use Piwik\DbHelper; use Piwik\Period; use Piwik\Segment; use Piwik\Sequence; +use Piwik\Site; use Psr\Log\LoggerInterface; /** @@ -75,12 +76,26 @@ class Model $rows = Db::fetchAll($sql); foreach ($rows as $row) { $duplicateArchives = explode(',', $row['archives']); - $countOfArchives = count($duplicateArchives); - // if there is more than one archive, the older invalidated ones can be deleted - if ($countOfArchives > 1) { - array_shift($duplicateArchives); // we don't want to delete the latest archive if it is usable + // do not consider purging partial archives, if they are the latest archive, + // and we don't want to delete the latest archive if it is usable + while (!empty($duplicateArchives)) { + $pair = $duplicateArchives[0]; + if (strpos($pair, '.') === false) { + continue; // see below + } + + list($idarchive, $value) = explode('.', $pair); + array_shift($duplicateArchives); + + if ($value != ArchiveWriter::DONE_PARTIAL) { + break; + } + } + + // if there is more than one archive, the older invalidated ones can be deleted + if (!empty($duplicateArchives)) { foreach ($duplicateArchives as $pair) { if (strpos($pair, '.') === false) { $this->logger->info("GROUP_CONCAT cut off the query result, you may have to purge archives again."); @@ -104,8 +119,13 @@ class Model return $result; } - public function updateArchiveAsInvalidated($archiveTable, $idSites, $allPeriodsToInvalidate, Segment $segment = null, $forceInvalidateNonexistantRanges = false) + public function updateArchiveAsInvalidated($archiveTable, $idSites, $allPeriodsToInvalidate, Segment $segment = null, + $forceInvalidateNonexistantRanges = false, $name = null) { + if (empty($idSites)) { + return 0; + } + // select all idarchive/name pairs we want to invalidate $sql = "SELECT idarchive, idsite, period, date1, date2, `name`, `value` FROM `$archiveTable` @@ -136,12 +156,23 @@ class Model $sql .= ")"; } - if ($segment) { - $nameCondition = "name LIKE '" . Rules::getDoneFlagArchiveContainsAllPlugins($segment) . "%'"; + if (!empty($name)) { + if (strpos($name, '.') !== false) { + list($plugin, $name) = explode('.', $name, 2); + } else { + $plugin = $name; + $name = null; + } + } + + if (empty($plugin)) { + $doneFlag = Rules::getDoneFlagArchiveContainsAllPlugins($segment ?: new Segment('', [])); } else { - $nameCondition = "name LIKE 'done%'"; + $doneFlag = Rules::getDoneFlagArchiveContainsOnePlugin($segment ?: new Segment('', []), $plugin); } + $nameCondition = "name LIKE '$doneFlag%'"; + $sql .= " AND $nameCondition"; $archivesToInvalidate = Db::fetchAll($sql); @@ -157,8 +188,6 @@ class Model Db::query($sql); } - $doneFlag = Rules::getDoneFlagArchiveContainsAllPlugins($segment ?: new Segment('', [])); - // we add every archive we need to invalidate + the archives that do not already exist to archive_invalidations. // except for archives that are DONE_IN_PROGRESS. $archivesToCreateInvalidationRowsFor = []; @@ -174,6 +203,7 @@ class Model $dummyArchives = []; foreach ($idSites as $idSite) { + $siteCreationTime = Date::factory(Site::getCreationDateFor($idSite)); foreach ($allPeriodsToInvalidate as $period) { if ($period->getLabel() == 'range' && !$forceInvalidateNonexistantRanges @@ -181,6 +211,10 @@ class Model continue; // range } + if ($period->getDateEnd()->isEarlier($siteCreationTime)) { + continue; // don't add entries if it is before the time the site was created + } + $date1 = $period->getDateStart()->toString(); $date2 = $period->getDateEnd()->toString(); $idArchive = $archivesToCreateInvalidationRowsFor[$idSite][$period->getId()][$date1][$date2] ?? null; @@ -188,6 +222,7 @@ class Model $dummyArchives[] = [ 'idarchive' => $idArchive, 'name' => $doneFlag, + 'report' => $name, 'idsite' => $idSite, 'date1' => $period->getDateStart()->getDatetime(), 'date2' => $period->getDateEnd()->getDatetime(), @@ -197,7 +232,8 @@ class Model } } - $fields = ['idarchive', 'name', 'idsite', 'date1', 'date2', 'period', 'ts_invalidated']; + $fields = ['idarchive', 'name', 'report', 'idsite', 'date1', 'date2', 'period', 'ts_invalidated']; + Db\BatchInsert::tableInsertBatch(Common::prefixTable('archive_invalidations'), $fields, $dummyArchives); return count($idArchives); @@ -213,6 +249,10 @@ class Model */ public function updateRangeArchiveAsInvalidated($archiveTable, $idSites, $allPeriodsToInvalidate, Segment $segment = null) { + if (empty($idSites)) { + return; + } + $bind = array(); $periodConditions = array(); @@ -321,6 +361,25 @@ class Model return $deletedRows; } + public function deleteOlderArchives(Parameters $params, $name, $tsArchived, $idArchive) + { + $dateStart = $params->getPeriod()->getDateStart(); + $dateEnd = $params->getPeriod()->getDateEnd(); + + $numericTable = ArchiveTableCreator::getNumericTable($dateStart); + $blobTable = ArchiveTableCreator::getBlobTable($dateEnd); + + $sql = "SELECT idarchive FROM `$numericTable` WHERE idsite = ? AND date1 = ? AND date2 = ? AND period = ? AND name = ? AND ts_archived < ? AND idarchive < ?"; + + $idArchives = Db::fetchAll($sql, [$params->getSite()->getId(), $dateStart, $dateEnd, $params->getPeriod()->getId(), $name, $tsArchived, $idArchive]); + $idArchives = array_column($idArchives, 'idarchive'); + if (empty($idArchives)) { + return; + } + + $this->deleteArchiveIds($numericTable, $blobTable, $idArchives); + } + public function getArchiveIdAndVisits($numericTable, $idSite, $period, $dateStartIso, $dateEndIso, $minDatetimeIsoArchiveProcessedUTC, $doneFlags, $doneFlagValues = null) { @@ -351,6 +410,7 @@ class Model $timeStampWhere AND arc1.ts_archived IS NOT NULL ORDER BY arc1.ts_archived DESC, arc1.idarchive DESC"; + $results = Db::fetchAll($sqlQuery, $bindSQL); return $results; @@ -599,11 +659,12 @@ class Model public function getNextInvalidatedArchive($idSite, $idInvalidationsToExclude = null, $useLimit = true) { $table = Common::prefixTable('archive_invalidations'); - $sql = "SELECT idinvalidation, idarchive, idsite, date1, date2, period, `name` + $sql = "SELECT idinvalidation, idarchive, idsite, date1, date2, period, `name`, report FROM `$table` - WHERE idsite = ?"; + WHERE idsite = ? AND status != ?"; $bind = [ $idSite, + ArchiveInvalidator::INVALIDATION_STATUS_IN_PROGRESS, ]; if (!empty($idInvalidationsToExclude)) { diff --git a/core/DataTable/Map.php b/core/DataTable/Map.php index 93921562c7..9169a36128 100644 --- a/core/DataTable/Map.php +++ b/core/DataTable/Map.php @@ -12,6 +12,7 @@ use Closure; use Piwik\Common; use Piwik\DataTable; use Piwik\DataTable\Renderer\Console; +use Piwik\DataTable\Renderer\Html; /** * Stores an array of {@link DataTable}s indexed by one type of {@link DataTable} metadata (such as site ID @@ -221,7 +222,7 @@ class Map implements DataTableInterface */ public function __toString() { - $renderer = new Console(); + $renderer = new Html(); $renderer->setTable($this); return (string)$renderer; } diff --git a/core/Date.php b/core/Date.php index 276cd6d0c3..4b67eeb9cb 100644 --- a/core/Date.php +++ b/core/Date.php @@ -556,7 +556,7 @@ class Date */ public static function today() { - return new Date(strtotime(date("Y-m-d 00:00:00"))); + return new Date(strtotime(date("Y-m-d 00:00:00", self::getNowTimestamp()))); } /** @@ -566,7 +566,7 @@ class Date */ public static function tomorrow() { - return new Date(strtotime('tomorrow')); + return new Date(strtotime('tomorrow', self::getNowTimestamp())); } /** @@ -576,7 +576,7 @@ class Date */ public static function yesterday() { - return new Date(strtotime("yesterday")); + return new Date(strtotime("yesterday", self::getNowTimestamp())); } /** @@ -586,7 +586,7 @@ class Date */ public static function yesterdaySameTime() { - return new Date(strtotime("yesterday " . date('H:i:s'))); + return new Date(strtotime("yesterday " . date('H:i:s'), self::getNowTimestamp())); } /** diff --git a/core/Db/Schema/Mysql.php b/core/Db/Schema/Mysql.php index 90c765ae21..5aaeb1c0f2 100644 --- a/core/Db/Schema/Mysql.php +++ b/core/Db/Schema/Mysql.php @@ -318,6 +318,7 @@ class Mysql implements SchemaInterface period TINYINT UNSIGNED NOT NULL, ts_invalidated DATETIME NULL, status TINYINT(1) UNSIGNED DEFAULT 0, + `report` VARCHAR(255) NULL, PRIMARY KEY(idinvalidation), INDEX index_idsite_dates_period_name(idsite, date1, period, name) ) ENGINE=$engine DEFAULT CHARSET=$charset diff --git a/core/Http.php b/core/Http.php index 4999cc912d..506e64380c 100644 --- a/core/Http.php +++ b/core/Http.php @@ -154,7 +154,8 @@ class Http $httpUsername = null, $httpPassword = null, $requestBody = null, - $additionalHeaders = array() + $additionalHeaders = array(), + $forcePost = null ) { if ($followDepth > 5) { throw new Exception('Too many redirects (' . $followDepth . ')'); @@ -617,6 +618,9 @@ class Http CURLOPT_FOLLOWLOCATION => true, CURLOPT_MAXREDIRS => 5, ); + if ($forcePost) { + $curl_options[CURLOPT_POSTREDIR] = CURL_REDIR_POST_ALL; + } @curl_setopt_array($ch, $curl_options); } diff --git a/core/Updater/Migration/Db/AddColumns.php b/core/Updater/Migration/Db/AddColumns.php index 2ab269d421..dcd882dd1b 100644 --- a/core/Updater/Migration/Db/AddColumns.php +++ b/core/Updater/Migration/Db/AddColumns.php @@ -7,6 +7,8 @@ */ namespace Piwik\Updater\Migration\Db; +use Piwik\DataAccess\TableMetadata; + /** * @see Factory::addColumns() * @ignore @@ -15,8 +17,19 @@ class AddColumns extends Sql { public function __construct($table, $columns, $placeColumnAfter) { + $tableMetadata = new TableMetadata(); + try { + $existingColumns = $tableMetadata->getColumns($table); + } catch (\Exception $ex) { + $existingColumns = []; + } + $changes = array(); foreach ($columns as $columnName => $columnType) { + if (in_array($columnName, $existingColumns)) { + continue; + } + $part = sprintf("ADD COLUMN `%s` %s", $columnName, $columnType); if (!empty($placeColumnAfter)) { diff --git a/core/Updates/4.0.0-b2.php b/core/Updates/4.0.0-b2.php index d71e55a7a4..2fd9c8b035 100644 --- a/core/Updates/4.0.0-b2.php +++ b/core/Updates/4.0.0-b2.php @@ -47,6 +47,7 @@ class Updates_4_0_0_b2 extends PiwikUpdates 'period' => 'TINYINT UNSIGNED NOT NULL', 'ts_invalidated' => 'DATETIME NOT NULL', 'status' => 'TINYINT(1) UNSIGNED DEFAULT 0', + 'report' => 'VARCHAR(255) NULL', ], ['idinvalidation']); $migrations[] = $this->migration->db->addIndex('archive_invalidations', ['idsite', 'date1', 'period'], 'index_idsite_dates_period_name'); diff --git a/lang/en.json b/lang/en.json index de0eceb6ea..af4a793302 100644 --- a/lang/en.json +++ b/lang/en.json @@ -519,7 +519,8 @@ "MaximumNumberOfPeriodsComparedIs": "The maximum number of periods that can be compared simultaneously is %s.", "Custom": "Custom", "PreviousPeriod": "Previous Period", - "PreviousYear": "Previous Year" + "PreviousYear": "Previous Year", + "ViewAccessRequired": "The token_auth used has too much access to be used in a non-API request URL. Please use an app specific password for a user that only has view access." }, "Mobile": { "AboutPiwikMobile": "About Matomo Mobile", diff --git a/plugins/API/Controller.php b/plugins/API/Controller.php index eb0b6e51f1..585cb14ba9 100644 --- a/plugins/API/Controller.php +++ b/plugins/API/Controller.php @@ -15,6 +15,7 @@ use Piwik\Common; use Piwik\Config; use Piwik\Piwik; use Piwik\Plugin\Report; +use Piwik\Plugins\API\Renderer\Original; use Piwik\Url; use Piwik\UrlHelper; use Piwik\View; @@ -27,6 +28,8 @@ class Controller extends \Piwik\Plugin\Controller function index() { $tokenAuth = Common::getRequestVar('token_auth', 'anonymous', 'string'); + $format = Common::getRequestVar('format', false); + $serialize = Common::getRequestVar('serialize', false); $token = 'token_auth=' . $tokenAuth; @@ -43,6 +46,12 @@ class Controller extends \Piwik\Plugin\Controller $response = $request->process(); if (is_array($response)) { + if ($format == 'original' + && $serialize != 1 + ) { + Original::sendPlainTextHeader(); + } + $response = var_export($response, true); } diff --git a/plugins/API/Renderer/Original.php b/plugins/API/Renderer/Original.php index 3bcc0631c4..a473278929 100644 --- a/plugins/API/Renderer/Original.php +++ b/plugins/API/Renderer/Original.php @@ -15,6 +15,11 @@ use Piwik\DataTable\DataTableInterface; class Original extends ApiRenderer { + public static function sendPlainTextHeader() + { + Common::sendHeader('Content-Type: text/plain; charset=utf-8'); + } + public function renderSuccess($message) { return true; @@ -72,7 +77,7 @@ class Original extends ApiRenderer public function sendHeader() { if ($this->shouldSerialize()) { - Common::sendHeader('Content-Type: text/plain; charset=utf-8'); + self::sendPlainTextHeader(); } } diff --git a/plugins/CoreAdminHome/API.php b/plugins/CoreAdminHome/API.php index 95b52fbb3d..6f84b954e9 100644 --- a/plugins/CoreAdminHome/API.php +++ b/plugins/CoreAdminHome/API.php @@ -13,6 +13,8 @@ use Monolog\Handler\StreamHandler; use Monolog\Logger; use Piwik\Access; use Piwik\ArchiveProcessor\Rules; +use Piwik\ArchiveProcessor; +use Piwik\Common; use Piwik\Config; use Piwik\Container\StaticContainer; use Piwik\Archive\ArchiveInvalidator; @@ -22,6 +24,7 @@ use Piwik\Period\Factory; use Piwik\Piwik; use Piwik\Segment; use Piwik\Scheduler\Scheduler; +use Piwik\SettingsServer; use Piwik\Site; use Piwik\Tracker\Failures; use Piwik\Url; @@ -241,6 +244,48 @@ class API extends \Piwik\Plugin\API } /** + * @param $idSite + * @param $period + * @param $date + * @param bool $segment + * @param bool $plugin + * @param bool $report + * @return mixed + * @throws \Piwik\Exception\UnexpectedWebsiteFoundException + * @internal + */ + public function archiveReports($idSite, $period, $date, $segment = false, $plugin = false, $report = false) + { + if (\Piwik\API\Request::getRootApiRequestMethod() === 'CoreAdminHome.archiveReports') { + Piwik::checkUserHasSuperUserAccess(); + } else { + Piwik::checkUserHasViewAccess($idSite); + } + + // if cron archiving is running, we will invalidate in CronArchive, not here + $isArchivePhpTriggered = SettingsServer::isArchivePhpTriggered(); + $invalidateBeforeArchiving = !$isArchivePhpTriggered; + + $period = Factory::build($period, $date); + $parameters = new ArchiveProcessor\Parameters(new Site($idSite), $period, new Segment($segment, [$idSite], $period->getDateTimeStart(), $period->getDateTimeEnd())); + if ($report) { + $parameters->setArchiveOnlyReport($report); + } + + // TODO: need to test case when there are multiple plugin archives w/ only some data each. does purging remove some that we need? + $archiveLoader = new ArchiveProcessor\Loader($parameters, $invalidateBeforeArchiving); + + $result = $archiveLoader->prepareArchive($plugin); + if (!empty($result)) { + $result = [ + 'idarchives' => $result[0], + 'nb_visits' => $result[1], + ]; + } + return $result; + } + + /** * Ensure the specified dates are valid. * Store invalid date so we can log them * @param array|string $dates diff --git a/plugins/CoreAdminHome/Commands/InvalidateReportData.php b/plugins/CoreAdminHome/Commands/InvalidateReportData.php index 3f1e7eacb0..b3bb2d115b 100644 --- a/plugins/CoreAdminHome/Commands/InvalidateReportData.php +++ b/plugins/CoreAdminHome/Commands/InvalidateReportData.php @@ -53,6 +53,7 @@ class InvalidateReportData extends ConsoleCommand . ' also invalidate all days within 2015-09-13,2015-09-19, even those outside the date range.'); $this->addOption('dry-run', null, InputOption::VALUE_NONE, 'For tests. Runs the command w/o actually ' . 'invalidating anything.'); + $this->addOption('plugin', null, InputOption::VALUE_REQUIRED, 'To invalidate data for a specific plugin only.'); $this->setHelp('Invalidate archived report data by date range, site and period. Invalidated archive data will ' . 'be re-archived during the next core:archive run. If your log data has changed for some reason, this ' . 'command can be used to make sure reports are generated using the new, changed log data.'); @@ -64,6 +65,7 @@ class InvalidateReportData extends ConsoleCommand $cascade = $input->getOption('cascade'); $dryRun = $input->getOption('dry-run'); + $plugin = $input->getOption('plugin'); $sites = $this->getSitesToInvalidateFor($input); $periodTypes = $this->getPeriodTypesToInvalidateFor($input); @@ -83,11 +85,16 @@ class InvalidateReportData extends ConsoleCommand $dates = $this->getPeriodDates($periodType, $dateRange); if ($dryRun) { - $output->writeln("[Dry-run] invalidating archives for site = [ " . implode(', ', $sites) + $message = "[Dry-run] invalidating archives for site = [ " . implode(', ', $sites) . " ], dates = [ " . implode(', ', $dates) . " ], period = [ $periodType ], segment = [ " - . "$segmentStr ], cascade = [ " . (int)$cascade . " ]"); + . "$segmentStr ], cascade = [ " . (int)$cascade . " ]"; + if (!empty($plugin)) { + $message .= ", plugin = [ $plugin ]"; + } + $output->writeln($message); } else { - $invalidationResult = $invalidator->markArchivesAsInvalidated($sites, $dates, $periodType, $segment, $cascade); + $invalidationResult = $invalidator->markArchivesAsInvalidated($sites, $dates, $periodType, $segment, $cascade, + false, $plugin); if ($output->getVerbosity() > OutputInterface::VERBOSITY_NORMAL) { $output->writeln($invalidationResult->makeOutputLogs()); diff --git a/plugins/CoreAdminHome/tests/Integration/Commands/InvalidateReportDataTest.php b/plugins/CoreAdminHome/tests/Integration/Commands/InvalidateReportDataTest.php index ccca19cb2d..8a4f9e83b3 100644 --- a/plugins/CoreAdminHome/tests/Integration/Commands/InvalidateReportDataTest.php +++ b/plugins/CoreAdminHome/tests/Integration/Commands/InvalidateReportDataTest.php @@ -123,9 +123,9 @@ class InvalidateReportDataTest extends ConsoleCommandTestCase /** * @dataProvider getTestDataForSuccessTests */ - public function test_Command_InvalidatesCorrectSitesAndDates($dates, $periods, $sites, $cascade, $segments, $expectedOutputs) + public function test_Command_InvalidatesCorrectSitesAndDates($dates, $periods, $sites, $cascade, $segments, $plugin, $expectedOutputs) { - $code = $this->applicationTester->run(array( + $options = array( 'command' => 'core:invalidate-report-data', '--dates' => $dates, '--periods' => $periods, @@ -134,7 +134,13 @@ class InvalidateReportDataTest extends ConsoleCommandTestCase '--segment' => $segments ?: array(), '--dry-run' => true, '-vvv' => true, - )); + ); + + if (!empty($plugin)) { + $options['--plugin'] = $plugin; + } + + $code = $this->applicationTester->run($options); $this->assertEquals(0, $code, $this->getCommandDisplayOutputErrorMessage()); @@ -278,6 +284,7 @@ class InvalidateReportDataTest extends ConsoleCommandTestCase '1', false, null, + null, array( '[Dry-run] invalidating archives for site = [ 1 ], dates = [ 2012-01-01 ], period = [ day ], segment = [ ]', ), @@ -289,6 +296,7 @@ class InvalidateReportDataTest extends ConsoleCommandTestCase '1', true, null, + null, array( '[Dry-run] invalidating archives for site = [ 1 ], dates = [ 2012-01-01 ], period = [ day ], segment = [ ]', ), @@ -300,6 +308,7 @@ class InvalidateReportDataTest extends ConsoleCommandTestCase '1', false, null, + null, array( '[Dry-run] invalidating archives for site = [ 1 ], dates = [ 2011-12-26 ], period = [ week ], segment = [ ]', ), @@ -311,6 +320,7 @@ class InvalidateReportDataTest extends ConsoleCommandTestCase '1,3', false, null, + null, array( '[Dry-run] invalidating archives for site = [ 1, 3 ], dates = [ 2012-01-01, 2012-02-01 ], period = [ month ], segment = [ ], cascade = [ 0 ]', '[Dry-run] invalidating archives for site = [ 1, 3 ], dates = [ 2012-01-01 ], period = [ month ], segment = [ ], cascade = [ 0 ]', @@ -327,6 +337,7 @@ class InvalidateReportDataTest extends ConsoleCommandTestCase '2', true, null, + null, array( '[Dry-run] invalidating archives for site = [ 2 ], dates = [ 2012-01-30, 2012-02-06 ], period = [ week ], segment = [ ], cascade = [ 1 ]', ), @@ -338,6 +349,7 @@ class InvalidateReportDataTest extends ConsoleCommandTestCase 'all', true, null, + null, array( '[Dry-run] invalidating archives for site = [ 1, 2, 3 ], dates = [ 2012-02-01 ], period = [ month ], segment = [ ], cascade = [ 1 ]', '[Dry-run] invalidating archives for site = [ 1, 2, 3 ], dates = [ 2012-03-01 ], period = [ month ], segment = [ ], cascade = [ 1 ]', @@ -354,10 +366,24 @@ class InvalidateReportDataTest extends ConsoleCommandTestCase 'all', true, array('browserCode==FF'), + null, array( '[Dry-run] invalidating archives for site = [ 1, 2, 3 ], dates = [ 2011-12-26, 2012-01-02, 2012-01-09 ], period = [ week ], segment = [ browserCode==FF ], cascade = [ 1 ]', ), ), + + // w/ plugin + [ + ['2015-05-04'], + 'day', + '1', + false, + null, + 'ExamplePlugin', + [ + '[Dry-run] invalidating archives for site = [ 1 ], dates = [ 2015-05-04 ], period = [ day ], segment = [ ], cascade = [ 0 ], plugin = [ ExamplePlugin ]', + ], + ], ); } } diff --git a/plugins/CoreConsole/tests/System/ArchiveCronTest.php b/plugins/CoreConsole/tests/System/ArchiveCronTest.php index e426778427..3b71dcd7ba 100644 --- a/plugins/CoreConsole/tests/System/ArchiveCronTest.php +++ b/plugins/CoreConsole/tests/System/ArchiveCronTest.php @@ -8,17 +8,21 @@ namespace Piwik\Plugins\CoreConsole\tests\System; use Interop\Container\ContainerInterface; +use Piwik\Archive\ArchiveInvalidator; use Piwik\Common; use Piwik\Config; -use Piwik\CronArchive; +use Piwik\Container\StaticContainer; +use Piwik\DataAccess\ArchiveTableCreator; use Piwik\Date; use Piwik\Db; use Piwik\Option; -use Piwik\Plugins\SitesManager\API; +use Piwik\Segment; +use Piwik\Sequence; use Piwik\Tests\Framework\TestCase\SystemTestCase; use Piwik\Tests\Fixtures\ManySitesImportedLogs; use Piwik\Tests\Framework\Fixture; use Exception; +use Psr\Log\LoggerInterface; /** * Tests to call the cron core:archive command script and check there is no error, @@ -36,6 +40,13 @@ class ArchiveCronTest extends SystemTestCase */ public static $fixture = null; // initialized below class definition + public static function setUpBeforeClass(): void + { + parent::setUpBeforeClass(); + + Db::exec("UPDATE " . Common::prefixTable('site') . ' SET ts_created = \'2005-01-02 00:00:00\''); + } + public function getApiForTesting() { $apiRequiringSegments = ['Goals.get', 'VisitFrequency.get']; @@ -48,14 +59,26 @@ class ArchiveCronTest extends SystemTestCase 'periods' => array('day', 'week', 'month', 'year'), 'segment' => $info['definition'], 'testSuffix' => '_' . $segmentName)); - - } + // ExamplePlugin metric + $results[] = ['ExamplePlugin.getExampleArchivedMetric', [ + 'idSite' => 'all', + 'date' => '2007-04-05', + 'periods' => ['day', 'week'], + ]]; + $results[] = ['Actions.get', [ + 'idSite' => 'all', + 'date' => '2007-04-05', + 'periods' => ['day', 'week'], + 'testSuffix' => '_examplePluginNoMetricsBecauseNoOtherPluginsArchived', + ]]; + // API Call Without segments $results[] = array('VisitsSummary.get', array('idSite' => 'all', 'date' => '2012-08-09', 'periods' => array('day', 'month', 'year', 'week'))); + $results[] = array($apiRequiringSegments, array('idSite' => 'all', 'date' => '2012-08-09', 'periods' => array('month'))); @@ -92,6 +115,18 @@ class ArchiveCronTest extends SystemTestCase public function testArchivePhpCron() { + // invalidate exampleplugin only archives in past + $invalidator = StaticContainer::get(ArchiveInvalidator::class); + $invalidator->markArchivesAsInvalidated([1], ['2007-04-05'], 'day', new Segment('', [1]), false, false, 'ExamplePlugin'); + + // track a visit in 2007-04-05 so it will archive (don't want to force archiving because then this test will take another 15 mins) + $tracker = Fixture::getTracker(1, '2007-04-05'); + $tracker->setUrl('http://example.com/test/url'); + Fixture::checkResponse($tracker->doTrackPageView('abcdefg')); + + // empty the list so nothing is invalidated during core:archive (so we only archive ExamplePlugin and not all plugins) + $invalidator->forgetRememberedArchivedReportsToInvalidate(1, Date::factory('2007-04-05')); + $output = $this->runArchivePhpCron(); $expectedInvalidations = []; @@ -118,6 +153,40 @@ class ArchiveCronTest extends SystemTestCase } } + /** + * @depends testArchivePhpCron + */ + public function testArchivePhpCronWithSingleReportRearchive() + { + // invalidate a report so we get a partial archive (using the metric that gets incremented each time it is archived) + // (do it after the last run so we don't end up just re-using the ExamplePlugin archive) + $invalidator = StaticContainer::get(ArchiveInvalidator::class); + $invalidator->markArchivesAsInvalidated([1], ['2007-04-05'], 'day', new Segment('', [1]), false, false, 'ExamplePlugin.ExamplePlugin_example_metric2'); + + $sequence = new Sequence('ExamplePlugin_archiveCount'); + $beforeCount = $sequence->getCurrentId(); + + $output = $this->runArchivePhpCron(['-vvv' => null]); + + $afterCount = $sequence->getCurrentId(); + + $this->assertNotEquals($beforeCount, $afterCount, 'example plugin archiving was not triggered'); + + $this->runApiTests('ExamplePlugin.getExampleArchivedMetric', [ + 'idSite' => 'all', + 'date' => '2007-04-05', + 'periods' => ['day', 'week'], + 'testSuffix' => '_singleMetric', + ]); + + // test that latest archives for ExamplePlugin are partial + $archiveValues = Db::fetchAll("SELECT value FROM " . ArchiveTableCreator::getNumericTable(Date::factory('2007-04-05')) + . " WHERE `name` = 'done.ExamplePlugin' ORDER BY ts_archived DESC LIMIT 8"); + $archiveValues = array_column($archiveValues, 'value'); + $archiveValues = array_unique($archiveValues); + $this->assertEquals([5], $archiveValues); + } + public function testArchivePhpCronArchivesFullRanges() { self::$fixture->getTestEnvironment()->overrideConfig('General', 'enable_browser_archiving_triggering', 0); @@ -133,9 +202,11 @@ class ArchiveCronTest extends SystemTestCase $expectedInvalidations = []; $invalidationEntries = $this->getInvalidatedArchiveTableEntries(); + $invalidationEntries = array_filter($invalidationEntries, function ($entry) { return $entry['period'] == 5; }); + $this->assertEquals($expectedInvalidations, $invalidationEntries); $this->runApiTests(array( @@ -143,8 +214,7 @@ class ArchiveCronTest extends SystemTestCase array('idSite' => '1', 'date' => '2012-08-09,2012-08-13', 'periods' => array('range'), - 'testSuffix' => '_range_archive' - ) + 'testSuffix' => '_range_archive') ); } diff --git a/plugins/CoreConsole/tests/System/expected/test_ArchiveCronTest__ExamplePlugin.getExampleArchivedMetric_day.xml b/plugins/CoreConsole/tests/System/expected/test_ArchiveCronTest__ExamplePlugin.getExampleArchivedMetric_day.xml new file mode 100644 index 0000000000..2e1a657b1f --- /dev/null +++ b/plugins/CoreConsole/tests/System/expected/test_ArchiveCronTest__ExamplePlugin.getExampleArchivedMetric_day.xml @@ -0,0 +1,9 @@ +<?xml version="1.0" encoding="utf-8" ?> +<results> + <result idSite="1"> + <ExamplePlugin_example_metric>3382</ExamplePlugin_example_metric> + <ExamplePlugin_example_metric2>51</ExamplePlugin_example_metric2> + </result> + <result idSite="2" /> + <result idSite="3" /> +</results>
\ No newline at end of file diff --git a/plugins/CoreConsole/tests/System/expected/test_ArchiveCronTest_archive_php_cron_output.txt b/plugins/CoreConsole/tests/System/expected/test_ArchiveCronTest_archive_php_cron_output.txt index e8b09e0083..81032bb1a7 100644 --- a/plugins/CoreConsole/tests/System/expected/test_ArchiveCronTest_archive_php_cron_output.txt +++ b/plugins/CoreConsole/tests/System/expected/test_ArchiveCronTest_archive_php_cron_output.txt @@ -10,90 +10,86 @@ try 'php archive.php --url=http://your.piwik/path' ------------------------------------------------------- -INFO [2020-06-25 09:52:47] 10351 --------------------------- -INFO [2020-06-25 09:52:47] 10351 INIT -INFO [2020-06-25 09:52:47] 10351 Running Matomo 4.0.0-b2 as Super User -INFO [2020-06-25 09:52:47] 10351 --------------------------- -INFO [2020-06-25 09:52:47] 10351 NOTES -INFO [2020-06-25 09:52:47] 10351 - If you execute this script at least once per hour (or more often) in a crontab, you may disable 'Browser trigger archiving' in Matomo UI > Settings > General Settings. -INFO [2020-06-25 09:52:47] 10351 See the doc at: https://matomo.org/docs/setup-auto-archiving/ -INFO [2020-06-25 09:52:47] 10351 - Async process archiving supported, using CliMulti. -INFO [2020-06-25 09:52:47] 10351 - Reports for today will be processed at most every 900 seconds. You can change this value in Matomo UI > Settings > General Settings. -INFO [2020-06-25 09:52:47] 10351 --------------------------- -INFO [2020-06-25 09:52:47] 10351 START -INFO [2020-06-25 09:52:47] 10351 Starting Matomo reports archiving... -INFO [2020-06-25 09:52:47] 10351 Checking for queued invalidations... -INFO [2020-06-25 09:52:47] 10351 Will invalidate archived reports for 2014-03-13 for following websites ids: 3,1 -INFO [2020-06-25 09:52:47] 10351 Will invalidate archived reports for 2014-03-12 for following websites ids: 1,3 -INFO [2020-06-25 09:52:47] 10351 Will invalidate archived reports for 2012-09-30 for following websites ids: 1 -INFO [2020-06-25 09:52:47] 10351 Will invalidate archived reports for 2012-08-19 for following websites ids: 1 -INFO [2020-06-25 09:52:47] 10351 Will invalidate archived reports for 2012-08-15 for following websites ids: 1,3,2 -INFO [2020-06-25 09:52:47] 10351 Will invalidate archived reports for 2012-08-11 for following websites ids: 1 -INFO [2020-06-25 09:52:47] 10351 Will invalidate archived reports for 2012-08-10 for following websites ids: 1 -INFO [2020-06-25 09:52:47] 10351 Will invalidate archived reports for 2012-08-09 for following websites ids: 1 -INFO [2020-06-25 09:52:47] 10351 Segment "browserCode==IE" was created or changed recently and will therefore archive today (for site ID = 1) -INFO [2020-06-25 09:52:47] 10351 Segment "visitCount<=5;visitorType!=non-existing-type;daysSinceFirstVisit<=50" was created or changed recently and will therefore archive today (for site ID = 1) -INFO [2020-06-25 09:52:47] 10351 Segment "visitCount<=5;visitorType!=re%2C%3Btest%20is%20encoded;daysSinceFirstVisit<=50" was created or changed recently and will therefore archive today (for site ID = 1) -INFO [2020-06-25 09:52:47] 10351 Done invalidating -INFO [2020-06-25 09:52:47] 10351 Start processing archives for site 1. -INFO [2020-06-25 09:52:50] 10351 Archived website id 1, period = day, date = 2014-03-13, segment = '', 11 visits found. Time elapsed: 1.477s -INFO [2020-06-25 09:52:50] 10351 Archived website id 1, period = day, date = 2014-03-12, segment = '', 1 visits found. Time elapsed: 1.477s -INFO [2020-06-25 09:52:50] 10351 Archived website id 1, period = day, date = 2012-09-30, segment = '', 1 visits found. Time elapsed: 1.477s -INFO [2020-06-25 09:52:51] 10351 Archived website id 1, period = week, date = 2014-03-10, segment = '', 12 visits found. Time elapsed: 1.116s -INFO [2020-06-25 09:52:51] 10351 Archived website id 1, period = week, date = 2012-09-24, segment = '', 1 visits found. Time elapsed: 1.151s -INFO [2020-06-25 09:52:51] 10351 Archived website id 1, period = week, date = 2012-08-13, segment = '', 3 visits found. Time elapsed: 1.425s -INFO [2020-06-25 09:52:54] 10351 Archived website id 1, period = month, date = 2014-03-01, segment = '', 12 visits found. Time elapsed: 1.115s -INFO [2020-06-25 09:52:54] 10351 Archived website id 1, period = month, date = 2012-09-01, segment = '', 1 visits found. Time elapsed: 1.115s -INFO [2020-06-25 09:52:54] 10351 Archived website id 1, period = month, date = 2012-08-01, segment = '', 33 visits found. Time elapsed: 1.936s -INFO [2020-06-25 09:52:55] 10351 Archived website id 1, period = year, date = 2014-01-01, segment = '', 12 visits found. Time elapsed: 0.919s -INFO [2020-06-25 09:52:55] 10351 Archived website id 1, period = year, date = 2012-01-01, segment = '', 34 visits found. Time elapsed: 0.919s -INFO [2020-06-25 09:52:59] 10351 Archived website id 1, period = year, date = 2014-01-01, segment = 'browserCode%3D%3DIE', 1 visits found. Time elapsed: 3.088s -INFO [2020-06-25 09:52:59] 10351 Archived website id 1, period = year, date = 2014-01-01, segment = 'visitCount%3C%3D5%3BvisitorType%21%3Dnon-existing-type%3BdaysSinceFirstVisit%3C%3D50', 12 visits found. Time elapsed: 3.967s -INFO [2020-06-25 09:52:59] 10351 Archived website id 1, period = year, date = 2014-01-01, segment = 'visitCount%3C%3D5%3BvisitorType%21%3Dre%252C%253Btest%2520is%2520encoded%3BdaysSinceFirstVisit%3C%3D50', 12 visits found. Time elapsed: 3.967s -INFO [2020-06-25 09:53:01] 10351 Archived website id 1, period = month, date = 2012-09-01, segment = 'browserCode%3D%3DIE', 0 visits found. Time elapsed: 1.113s -INFO [2020-06-25 09:53:01] 10351 Archived website id 1, period = month, date = 2012-09-01, segment = 'visitCount%3C%3D5%3BvisitorType%21%3Dnon-existing-type%3BdaysSinceFirstVisit%3C%3D50', 1 visits found. Time elapsed: 1.996s -INFO [2020-06-25 09:53:01] 10351 Archived website id 1, period = month, date = 2012-09-01, segment = 'visitCount%3C%3D5%3BvisitorType%21%3Dre%252C%253Btest%2520is%2520encoded%3BdaysSinceFirstVisit%3C%3D50', 1 visits found. Time elapsed: 1.996s -INFO [2020-06-25 09:53:04] 10351 Archived website id 1, period = week, date = 2012-08-13, segment = 'browserCode%3D%3DIE', 1 visits found. Time elapsed: 1.807s -INFO [2020-06-25 09:53:04] 10351 Archived website id 1, period = week, date = 2012-08-13, segment = 'visitCount%3C%3D5%3BvisitorType%21%3Dnon-existing-type%3BdaysSinceFirstVisit%3C%3D50', 3 visits found. Time elapsed: 1.859s -INFO [2020-06-25 09:53:04] 10351 Archived website id 1, period = week, date = 2012-08-13, segment = 'visitCount%3C%3D5%3BvisitorType%21%3Dre%252C%253Btest%2520is%2520encoded%3BdaysSinceFirstVisit%3C%3D50', 3 visits found. Time elapsed: 2.129s -INFO [2020-06-25 09:53:05] 10351 Archived website id 1, period = day, date = 2012-08-11, segment = 'browserCode%3D%3DIE', 1 visits found. Time elapsed: 1.130s -INFO [2020-06-25 09:53:05] 10351 Archived website id 1, period = day, date = 2012-08-11, segment = 'visitCount%3C%3D5%3BvisitorType%21%3Dnon-existing-type%3BdaysSinceFirstVisit%3C%3D50', 11 visits found. Time elapsed: 1.130s -INFO [2020-06-25 09:53:05] 10351 Archived website id 1, period = day, date = 2012-08-11, segment = 'visitCount%3C%3D5%3BvisitorType%21%3Dre%252C%253Btest%2520is%2520encoded%3BdaysSinceFirstVisit%3C%3D50', 11 visits found. Time elapsed: 1.403s -INFO [2020-06-25 09:53:06] 10351 Archived website id 1, period = day, date = 2012-08-10, segment = 'browserCode%3D%3DIE', 3 visits found. Time elapsed: 1.114s -INFO [2020-06-25 09:53:06] 10351 Archived website id 1, period = day, date = 2012-08-10, segment = 'visitCount%3C%3D5%3BvisitorType%21%3Dnon-existing-type%3BdaysSinceFirstVisit%3C%3D50', 8 visits found. Time elapsed: 1.426s -INFO [2020-06-25 09:53:06] 10351 Archived website id 1, period = day, date = 2012-08-10, segment = 'visitCount%3C%3D5%3BvisitorType%21%3Dre%252C%253Btest%2520is%2520encoded%3BdaysSinceFirstVisit%3C%3D50', 8 visits found. Time elapsed: 1.426s -INFO [2020-06-25 09:53:08] 10351 Archived website id 1, period = day, date = 2012-08-09, segment = 'browserCode%3D%3DIE', 1 visits found. Time elapsed: 1.099s -INFO [2020-06-25 09:53:08] 10351 Archived website id 1, period = day, date = 2012-08-09, segment = 'visitCount%3C%3D5%3BvisitorType%21%3Dnon-existing-type%3BdaysSinceFirstVisit%3C%3D50', 11 visits found. Time elapsed: 1.417s -INFO [2020-06-25 09:53:08] 10351 Archived website id 1, period = day, date = 2012-08-09, segment = 'visitCount%3C%3D5%3BvisitorType%21%3Dre%252C%253Btest%2520is%2520encoded%3BdaysSinceFirstVisit%3C%3D50', 11 visits found. Time elapsed: 1.417s -INFO [2020-06-25 09:53:10] 10351 Archived website id 1, period = week, date = 2012-08-06, segment = 'browserCode%3D%3DIE', 5 visits found. Time elapsed: 1.442s -INFO [2020-06-25 09:53:10] 10351 Archived website id 1, period = week, date = 2012-08-06, segment = 'visitCount%3C%3D5%3BvisitorType%21%3Dnon-existing-type%3BdaysSinceFirstVisit%3C%3D50', 30 visits found. Time elapsed: 1.442s -INFO [2020-06-25 09:53:10] 10351 Archived website id 1, period = week, date = 2012-08-06, segment = 'visitCount%3C%3D5%3BvisitorType%21%3Dre%252C%253Btest%2520is%2520encoded%3BdaysSinceFirstVisit%3C%3D50', 30 visits found. Time elapsed: 1.714s -INFO [2020-06-25 09:53:11] 10351 Archived website id 1, period = month, date = 2012-08-01, segment = 'browserCode%3D%3DIE', 6 visits found. Time elapsed: 1.403s -INFO [2020-06-25 09:53:11] 10351 Archived website id 1, period = month, date = 2012-08-01, segment = 'visitCount%3C%3D5%3BvisitorType%21%3Dnon-existing-type%3BdaysSinceFirstVisit%3C%3D50', 33 visits found. Time elapsed: 1.403s -INFO [2020-06-25 09:53:11] 10351 Archived website id 1, period = month, date = 2012-08-01, segment = 'visitCount%3C%3D5%3BvisitorType%21%3Dre%252C%253Btest%2520is%2520encoded%3BdaysSinceFirstVisit%3C%3D50', 33 visits found. Time elapsed: 1.676s -INFO [2020-06-25 09:53:13] 10351 Archived website id 1, period = year, date = 2012-01-01, segment = 'browserCode%3D%3DIE', 6 visits found. Time elapsed: 1.421s -INFO [2020-06-25 09:53:13] 10351 Archived website id 1, period = year, date = 2012-01-01, segment = 'visitCount%3C%3D5%3BvisitorType%21%3Dnon-existing-type%3BdaysSinceFirstVisit%3C%3D50', 34 visits found. Time elapsed: 1.728s -INFO [2020-06-25 09:53:13] 10351 Archived website id 1, period = year, date = 2012-01-01, segment = 'visitCount%3C%3D5%3BvisitorType%21%3Dre%252C%253Btest%2520is%2520encoded%3BdaysSinceFirstVisit%3C%3D50', 34 visits found. Time elapsed: 1.728s -INFO [2020-06-25 09:53:13] 10351 Finished archiving for site 1, 38 API requests, Time elapsed: 25.964s [1 / 3 done] -INFO [2020-06-25 09:53:13] 10351 Start processing archives for site 2. -INFO [2020-06-25 09:53:14] 10351 Archived website id 2, period = day, date = 2012-08-15, segment = '', 1 visits found. Time elapsed: 0.556s -INFO [2020-06-25 09:53:14] 10351 Archived website id 2, period = week, date = 2012-08-13, segment = '', 1 visits found. Time elapsed: 0.555s -INFO [2020-06-25 09:53:15] 10351 Archived website id 2, period = month, date = 2012-08-01, segment = '', 1 visits found. Time elapsed: 0.555s -INFO [2020-06-25 09:53:16] 10351 Archived website id 2, period = year, date = 2012-01-01, segment = '', 1 visits found. Time elapsed: 0.554s -INFO [2020-06-25 09:53:16] 10351 Finished archiving for site 2, 4 API requests, Time elapsed: 2.374s [2 / 3 done] -INFO [2020-06-25 09:53:16] 10351 Start processing archives for site 3. -INFO [2020-06-25 09:53:17] 10351 Archived website id 3, period = day, date = 2014-03-13, segment = '', 11 visits found. Time elapsed: 1.116s -INFO [2020-06-25 09:53:17] 10351 Archived website id 3, period = day, date = 2014-03-12, segment = '', 1 visits found. Time elapsed: 1.116s -INFO [2020-06-25 09:53:17] 10351 Archived website id 3, period = day, date = 2012-08-15, segment = '', 3 visits found. Time elapsed: 1.116s -INFO [2020-06-25 09:53:18] 10351 Archived website id 3, period = week, date = 2014-03-10, segment = '', 12 visits found. Time elapsed: 0.930s -INFO [2020-06-25 09:53:18] 10351 Archived website id 3, period = week, date = 2012-08-13, segment = '', 3 visits found. Time elapsed: 0.930s -INFO [2020-06-25 09:53:19] 10351 Archived website id 3, period = month, date = 2014-03-01, segment = '', 12 visits found. Time elapsed: 0.932s -INFO [2020-06-25 09:53:19] 10351 Archived website id 3, period = month, date = 2012-08-01, segment = '', 3 visits found. Time elapsed: 0.932s -INFO [2020-06-25 09:53:20] 10351 Archived website id 3, period = year, date = 2014-01-01, segment = '', 12 visits found. Time elapsed: 0.923s -INFO [2020-06-25 09:53:20] 10351 Archived website id 3, period = year, date = 2012-01-01, segment = '', 3 visits found. Time elapsed: 0.923s -INFO [2020-06-25 09:53:20] 10351 Finished archiving for site 3, 9 API requests, Time elapsed: 4.088s [3 / 3 done] -INFO [2020-06-25 09:53:20] 10351 --------------------------- -INFO [2020-06-25 09:53:20] 10351 SCHEDULED TASKS -INFO [2020-06-25 09:53:20] 10351 Starting Scheduled tasks... -INFO [2020-06-25 09:53:20] 10351 done -INFO [2020-06-25 09:53:20] 10351 ---------------------------
\ No newline at end of file +INFO [2020-07-06 21:59:34] 10307 --------------------------- +INFO [2020-07-06 21:59:34] 10307 INIT +INFO [2020-07-06 21:59:34] 10307 Running Matomo 4.0.0-b2 as Super User +INFO [2020-07-06 21:59:34] 10307 --------------------------- +INFO [2020-07-06 21:59:34] 10307 NOTES +INFO [2020-07-06 21:59:34] 10307 - If you execute this script at least once per hour (or more often) in a crontab, you may disable 'Browser trigger archiving' in Matomo UI > Settings > General Settings. +INFO [2020-07-06 21:59:34] 10307 See the doc at: https://matomo.org/docs/setup-auto-archiving/ +INFO [2020-07-06 21:59:34] 10307 - Async process archiving supported, using CliMulti. +INFO [2020-07-06 21:59:34] 10307 - Reports for today will be processed at most every 900 seconds. You can change this value in Matomo UI > Settings > General Settings. +INFO [2020-07-06 21:59:34] 10307 --------------------------- +INFO [2020-07-06 21:59:34] 10307 START +INFO [2020-07-06 21:59:34] 10307 Starting Matomo reports archiving... +INFO [2020-07-06 21:59:34] 10307 Checking for queued invalidations... +INFO [2020-07-06 21:59:34] 10307 Will invalidate archived reports for 2014-03-13 for following websites ids: 1,3 +INFO [2020-07-06 21:59:34] 10307 Will invalidate archived reports for 2014-03-12 for following websites ids: 3,1 +INFO [2020-07-06 21:59:34] 10307 Will invalidate archived reports for 2012-09-30 for following websites ids: 1 +INFO [2020-07-06 21:59:34] 10307 Will invalidate archived reports for 2012-08-19 for following websites ids: 1 +INFO [2020-07-06 21:59:34] 10307 Will invalidate archived reports for 2012-08-15 for following websites ids: 2,1,3 +INFO [2020-07-06 21:59:34] 10307 Will invalidate archived reports for 2012-08-11 for following websites ids: 1 +INFO [2020-07-06 21:59:34] 10307 Will invalidate archived reports for 2012-08-10 for following websites ids: 1 +INFO [2020-07-06 21:59:34] 10307 Will invalidate archived reports for 2012-08-09 for following websites ids: 1 +INFO [2020-07-06 21:59:34] 10307 Segment "browserCode==IE" was created or changed recently and will therefore archive today (for site ID = 1) +INFO [2020-07-06 21:59:34] 10307 Segment "visitCount<=5;visitorType!=non-existing-type;daysSinceFirstVisit<=50" was created or changed recently and will therefore archive today (for site ID = 1) +INFO [2020-07-06 21:59:34] 10307 Segment "visitCount<=5;visitorType!=re%2C%3Btest%20is%20encoded;daysSinceFirstVisit<=50" was created or changed recently and will therefore archive today (for site ID = 1) +INFO [2020-07-06 21:59:34] 10307 Done invalidating +INFO [2020-07-06 21:59:34] 10307 Start processing archives for site 1. +INFO [2020-07-06 21:59:37] 10307 Archived website id 1, period = day, date = 2014-03-13, segment = '', 11 visits found. Time elapsed: 1.124s +INFO [2020-07-06 21:59:37] 10307 Archived website id 1, period = day, date = 2014-03-12, segment = '', 1 visits found. Time elapsed: 1.124s +INFO [2020-07-06 21:59:37] 10307 Archived website id 1, period = day, date = 2012-09-30, segment = '', 1 visits found. Time elapsed: 1.124s +INFO [2020-07-06 21:59:38] 10307 Archived website id 1, period = week, date = 2014-03-10, segment = '', 12 visits found. Time elapsed: 1.761s +INFO [2020-07-06 21:59:38] 10307 Archived website id 1, period = week, date = 2012-09-24, segment = '', 1 visits found. Time elapsed: 1.761s +INFO [2020-07-06 21:59:38] 10307 Archived website id 1, period = day, date = 2012-08-19, segment = '', 1 visits found. Time elapsed: 1.761s +INFO [2020-07-06 21:59:40] 10307 Archived website id 1, period = month, date = 2014-03-01, segment = '', 12 visits found. Time elapsed: 1.743s +INFO [2020-07-06 21:59:40] 10307 Archived website id 1, period = month, date = 2012-09-01, segment = '', 1 visits found. Time elapsed: 1.743s +INFO [2020-07-06 21:59:40] 10307 Archived website id 1, period = day, date = 2012-08-15, segment = '', 2 visits found. Time elapsed: 1.743s +INFO [2020-07-06 21:59:43] 10307 Archived website id 1, period = year, date = 2014-01-01, segment = '', 12 visits found. Time elapsed: 2.510s +INFO [2020-07-06 21:59:43] 10307 Archived website id 1, period = week, date = 2012-08-13, segment = '', 3 visits found. Time elapsed: 2.510s +INFO [2020-07-06 21:59:43] 10307 Archived website id 1, period = day, date = 2012-08-11, segment = '', 11 visits found. Time elapsed: 2.510s +INFO [2020-07-06 21:59:47] 10307 Archived website id 1, period = year, date = 2014-01-01, segment = 'browserCode%3D%3DIE', 1 visits found. Time elapsed: 3.015s +INFO [2020-07-06 21:59:47] 10307 Archived website id 1, period = year, date = 2012-01-01, segment = 'browserCode%3D%3DIE', 6 visits found. Time elapsed: 4.118s +INFO [2020-07-06 21:59:47] 10307 Archived website id 1, period = day, date = 2007-04-05, segment = '', plugin = ExamplePlugin, 1 visits found. Time elapsed: 4.118s +INFO [2020-07-06 21:59:53] 10307 Archived website id 1, period = year, date = 2014-01-01, segment = 'visitCount%3C%3D5%3BvisitorType%21%3Dnon-existing-type%3BdaysSinceFirstVisit%3C%3D50', 12 visits found. Time elapsed: 4.601s +INFO [2020-07-06 21:59:53] 10307 Archived website id 1, period = year, date = 2012-01-01, segment = 'visitCount%3C%3D5%3BvisitorType%21%3Dnon-existing-type%3BdaysSinceFirstVisit%3C%3D50', 34 visits found. Time elapsed: 5.707s +INFO [2020-07-06 21:59:53] 10307 Archived website id 1, period = week, date = 2007-04-02, segment = '', plugin = ExamplePlugin, 1 visits found. Time elapsed: 5.707s +INFO [2020-07-06 21:59:59] 10307 Archived website id 1, period = year, date = 2014-01-01, segment = 'visitCount%3C%3D5%3BvisitorType%21%3Dre%252C%253Btest%2520is%2520encoded%3BdaysSinceFirstVisit%3C%3D50', 12 visits found. Time elapsed: 3.018s +INFO [2020-07-06 21:59:59] 10307 Archived website id 1, period = year, date = 2012-01-01, segment = 'visitCount%3C%3D5%3BvisitorType%21%3Dre%252C%253Btest%2520is%2520encoded%3BdaysSinceFirstVisit%3C%3D50', 34 visits found. Time elapsed: 5.246s +INFO [2020-07-06 21:59:59] 10307 Archived website id 1, period = month, date = 2007-04-01, segment = '', plugin = ExamplePlugin, 1 visits found. Time elapsed: 5.246s +INFO [2020-07-06 22:00:00] 10307 Archived website id 1, period = day, date = 2012-08-10, segment = '', 8 visits found. Time elapsed: 0.632s +INFO [2020-07-06 22:00:00] 10307 Archived website id 1, period = day, date = 2012-08-09, segment = '', 11 visits found. Time elapsed: 0.916s +INFO [2020-07-06 22:00:01] 10307 Archived website id 1, period = week, date = 2012-08-06, segment = '', 30 visits found. Time elapsed: 0.827s +INFO [2020-07-06 22:00:02] 10307 Archived website id 1, period = month, date = 2012-08-01, segment = '', 33 visits found. Time elapsed: 1.104s +INFO [2020-07-06 22:00:03] 10307 Archived website id 1, period = year, date = 2012-01-01, segment = '', 34 visits found. Time elapsed: 1.102s +INFO [2020-07-06 22:00:03] 10307 Archived website id 1, period = year, date = 2007-01-01, segment = '', plugin = ExamplePlugin, 1 visits found. Time elapsed: 0.564s +INFO [2020-07-06 22:00:03] 10307 Finished archiving for site 1, 27 API requests, Time elapsed: 28.981s [1 / 3 done] +INFO [2020-07-06 22:00:04] 10307 Start processing archives for site 2. +INFO [2020-07-06 22:00:04] 10307 Archived website id 2, period = day, date = 2012-08-15, segment = '', 1 visits found. Time elapsed: 0.549s +INFO [2020-07-06 22:00:05] 10307 Archived website id 2, period = week, date = 2012-08-13, segment = '', 1 visits found. Time elapsed: 0.826s +INFO [2020-07-06 22:00:06] 10307 Archived website id 2, period = month, date = 2012-08-01, segment = '', 1 visits found. Time elapsed: 0.825s +INFO [2020-07-06 22:00:07] 10307 Archived website id 2, period = year, date = 2012-01-01, segment = '', 1 visits found. Time elapsed: 1.110s +INFO [2020-07-06 22:00:07] 10307 Finished archiving for site 2, 4 API requests, Time elapsed: 3.469s [2 / 3 done] +INFO [2020-07-06 22:00:07] 10307 Start processing archives for site 3. +INFO [2020-07-06 22:00:08] 10307 Archived website id 3, period = day, date = 2014-03-13, segment = '', 11 visits found. Time elapsed: 0.834s +INFO [2020-07-06 22:00:08] 10307 Archived website id 3, period = day, date = 2014-03-12, segment = '', 1 visits found. Time elapsed: 0.834s +INFO [2020-07-06 22:00:08] 10307 Archived website id 3, period = day, date = 2012-08-15, segment = '', 3 visits found. Time elapsed: 1.115s +INFO [2020-07-06 22:00:09] 10307 Archived website id 3, period = week, date = 2014-03-10, segment = '', 12 visits found. Time elapsed: 1.257s +INFO [2020-07-06 22:00:09] 10307 Archived website id 3, period = week, date = 2012-08-13, segment = '', 3 visits found. Time elapsed: 1.257s +INFO [2020-07-06 22:00:11] 10307 Archived website id 3, period = month, date = 2014-03-01, segment = '', 12 visits found. Time elapsed: 1.266s +INFO [2020-07-06 22:00:11] 10307 Archived website id 3, period = month, date = 2012-08-01, segment = '', 3 visits found. Time elapsed: 1.818s +INFO [2020-07-06 22:00:13] 10307 Archived website id 3, period = year, date = 2014-01-01, segment = '', 12 visits found. Time elapsed: 1.899s +INFO [2020-07-06 22:00:13] 10307 Archived website id 3, period = year, date = 2012-01-01, segment = '', 3 visits found. Time elapsed: 1.920s +INFO [2020-07-06 22:00:13] 10307 Finished archiving for site 3, 9 API requests, Time elapsed: 6.303s [3 / 3 done] +INFO [2020-07-06 22:00:13] 10307 Done archiving! +INFO [2020-07-06 22:00:13] 10307 --------------------------- +INFO [2020-07-06 22:00:13] 10307 SUMMARY +INFO [2020-07-06 22:00:13] 10307 Processed 40 archives. +INFO [2020-07-06 22:00:13] 10307 Total API requests: 40 +INFO [2020-07-06 22:00:13] 10307 done: 40 req, 39477 ms, no error +INFO [2020-07-06 22:00:13] 10307 Time elapsed: 39.477s +INFO [2020-07-06 22:00:13] 10307 --------------------------- +INFO [2020-07-06 22:00:13] 10307 SCHEDULED TASKS +INFO [2020-07-06 22:00:13] 10307 Starting Scheduled tasks... +INFO [2020-07-06 22:00:13] 10307 done +INFO [2020-07-06 22:00:13] 10307 ---------------------------
\ No newline at end of file diff --git a/plugins/CoreConsole/tests/System/expected/test_ArchiveCronTest_examplePluginNoMetricsBecauseNoOtherPluginsArchived_noOptions__Actions.get_day.xml b/plugins/CoreConsole/tests/System/expected/test_ArchiveCronTest_examplePluginNoMetricsBecauseNoOtherPluginsArchived_noOptions__Actions.get_day.xml new file mode 100644 index 0000000000..dd52dc4ff4 --- /dev/null +++ b/plugins/CoreConsole/tests/System/expected/test_ArchiveCronTest_examplePluginNoMetricsBecauseNoOtherPluginsArchived_noOptions__Actions.get_day.xml @@ -0,0 +1,6 @@ +<?xml version="1.0" encoding="utf-8" ?> +<results> + <result idSite="1" /> + <result idSite="2" /> + <result idSite="3" /> +</results>
\ No newline at end of file diff --git a/plugins/CoreConsole/tests/System/expected/test_ArchiveCronTest_examplePluginNoMetricsBecauseNoOtherPluginsArchived_noOptions__Actions.get_week.xml b/plugins/CoreConsole/tests/System/expected/test_ArchiveCronTest_examplePluginNoMetricsBecauseNoOtherPluginsArchived_noOptions__Actions.get_week.xml new file mode 100644 index 0000000000..dd52dc4ff4 --- /dev/null +++ b/plugins/CoreConsole/tests/System/expected/test_ArchiveCronTest_examplePluginNoMetricsBecauseNoOtherPluginsArchived_noOptions__Actions.get_week.xml @@ -0,0 +1,6 @@ +<?xml version="1.0" encoding="utf-8" ?> +<results> + <result idSite="1" /> + <result idSite="2" /> + <result idSite="3" /> +</results>
\ No newline at end of file diff --git a/plugins/CoreConsole/tests/System/expected/test_ArchiveCronTest_noOptions__ExamplePlugin.getExampleArchivedMetric_day.xml b/plugins/CoreConsole/tests/System/expected/test_ArchiveCronTest_noOptions__ExamplePlugin.getExampleArchivedMetric_day.xml new file mode 100644 index 0000000000..bbba10abb2 --- /dev/null +++ b/plugins/CoreConsole/tests/System/expected/test_ArchiveCronTest_noOptions__ExamplePlugin.getExampleArchivedMetric_day.xml @@ -0,0 +1,9 @@ +<?xml version="1.0" encoding="utf-8" ?> +<results> + <result idSite="1"> + <ExamplePlugin_example_metric>3382</ExamplePlugin_example_metric> + <ExamplePlugin_example_metric2>60</ExamplePlugin_example_metric2> + </result> + <result idSite="2" /> + <result idSite="3" /> +</results>
\ No newline at end of file diff --git a/plugins/CoreConsole/tests/System/expected/test_ArchiveCronTest_noOptions__ExamplePlugin.getExampleArchivedMetric_week.xml b/plugins/CoreConsole/tests/System/expected/test_ArchiveCronTest_noOptions__ExamplePlugin.getExampleArchivedMetric_week.xml new file mode 100644 index 0000000000..bbba10abb2 --- /dev/null +++ b/plugins/CoreConsole/tests/System/expected/test_ArchiveCronTest_noOptions__ExamplePlugin.getExampleArchivedMetric_week.xml @@ -0,0 +1,9 @@ +<?xml version="1.0" encoding="utf-8" ?> +<results> + <result idSite="1"> + <ExamplePlugin_example_metric>3382</ExamplePlugin_example_metric> + <ExamplePlugin_example_metric2>60</ExamplePlugin_example_metric2> + </result> + <result idSite="2" /> + <result idSite="3" /> +</results>
\ No newline at end of file diff --git a/plugins/CoreConsole/tests/System/expected/test_ArchiveCronTest_singleMetric__ExamplePlugin.getExampleArchivedMetric_day.xml b/plugins/CoreConsole/tests/System/expected/test_ArchiveCronTest_singleMetric__ExamplePlugin.getExampleArchivedMetric_day.xml new file mode 100644 index 0000000000..4214cca45d --- /dev/null +++ b/plugins/CoreConsole/tests/System/expected/test_ArchiveCronTest_singleMetric__ExamplePlugin.getExampleArchivedMetric_day.xml @@ -0,0 +1,9 @@ +<?xml version="1.0" encoding="utf-8" ?> +<results> + <result idSite="1"> + <ExamplePlugin_example_metric>3382</ExamplePlugin_example_metric> + <ExamplePlugin_example_metric2>90</ExamplePlugin_example_metric2> + </result> + <result idSite="2" /> + <result idSite="3" /> +</results>
\ No newline at end of file diff --git a/plugins/CoreConsole/tests/System/expected/test_ArchiveCronTest_singleMetric__ExamplePlugin.getExampleArchivedMetric_week.xml b/plugins/CoreConsole/tests/System/expected/test_ArchiveCronTest_singleMetric__ExamplePlugin.getExampleArchivedMetric_week.xml new file mode 100644 index 0000000000..4214cca45d --- /dev/null +++ b/plugins/CoreConsole/tests/System/expected/test_ArchiveCronTest_singleMetric__ExamplePlugin.getExampleArchivedMetric_week.xml @@ -0,0 +1,9 @@ +<?xml version="1.0" encoding="utf-8" ?> +<results> + <result idSite="1"> + <ExamplePlugin_example_metric>3382</ExamplePlugin_example_metric> + <ExamplePlugin_example_metric2>90</ExamplePlugin_example_metric2> + </result> + <result idSite="2" /> + <result idSite="3" /> +</results>
\ No newline at end of file diff --git a/plugins/CustomVariables b/plugins/CustomVariables -Subproject 1aee5523fb807e0d8a8cc5290e81133ca7dd563 +Subproject 6f2b1419fdf11f4ce845658e4d5d2634b61dcda diff --git a/plugins/Diagnostics/tests/Integration/Commands/AnalyzeArchiveTableTest.php b/plugins/Diagnostics/tests/Integration/Commands/AnalyzeArchiveTableTest.php index 9e3fbd224a..b71601e41e 100644 --- a/plugins/Diagnostics/tests/Integration/Commands/AnalyzeArchiveTableTest.php +++ b/plugins/Diagnostics/tests/Integration/Commands/AnalyzeArchiveTableTest.php @@ -41,9 +41,9 @@ Statistics for the archive_numeric_2010_03 and archive_blob_2010_03 tables: +-------------------------------------------+------------+---------------+-------------+---------+-----------+----------------+-------------+-------------+ | Group | # Archives | # Invalidated | # Temporary | # Error | # Segment | # Numeric Rows | # Blob Rows | # Blob Data | +-------------------------------------------+------------+---------------+-------------+---------+-----------+----------------+-------------+-------------+ -| week[2010-03-01 - 2010-03-07] idSite = 1 | 7 | 0 | 0 | 0 | 6 | 73 | 97 | %d | -| month[2010-03-01 - 2010-03-31] idSite = 1 | 7 | 0 | 0 | 0 | 6 | 73 | 97 | %d | -| day[2010-03-06 - 2010-03-06] idSite = 1 | 7 | 0 | 0 | 0 | 6 | 73 | 73 | %d | +| week[2010-03-01 - 2010-03-07] idSite = 1 | 7 | 0 | 0 | 0 | 6 | 75 | 97 | %d | +| month[2010-03-01 - 2010-03-31] idSite = 1 | 7 | 0 | 0 | 0 | 6 | 75 | 97 | %d | +| day[2010-03-06 - 2010-03-06] idSite = 1 | 7 | 0 | 0 | 0 | 6 | 75 | 73 | %d | +-------------------------------------------+------------+---------------+-------------+---------+-----------+----------------+-------------+-------------+ Total # Archives: 21 diff --git a/plugins/ExamplePlugin/API.php b/plugins/ExamplePlugin/API.php index 44d67cd80c..88f6249ecb 100644 --- a/plugins/ExamplePlugin/API.php +++ b/plugins/ExamplePlugin/API.php @@ -8,6 +8,7 @@ namespace Piwik\Plugins\ExamplePlugin; +use Piwik\Archive; use Piwik\DataTable; use Piwik\DataTable\Row; @@ -54,4 +55,19 @@ class API extends \Piwik\Plugin\API return $table; } + + /** + * Returns the example metric we archive in Archiver.php. + * @param int $idSite + * @param string $period + * @param string $date + * @param bool|string $segment + * @return DataTable + */ + public function getExampleArchivedMetric($idSite, $period, $date, $segment = false) + { + $archive = Archive::build($idSite, $period, $date, $segment); + $dataTable = $archive->getDataTableFromNumeric([Archiver::EXAMPLEPLUGIN_METRIC_NAME, Archiver::EXAMPLEPLUGIN_CONST_METRIC_NAME]); + return $dataTable; + } } diff --git a/plugins/ExamplePlugin/Archiver.php b/plugins/ExamplePlugin/Archiver.php index f507c6087a..13455ce6eb 100644 --- a/plugins/ExamplePlugin/Archiver.php +++ b/plugins/ExamplePlugin/Archiver.php @@ -8,6 +8,13 @@ namespace Piwik\Plugins\ExamplePlugin; +use Piwik\ArchiveProcessor; +use Piwik\Container\StaticContainer; +use Piwik\Date; +use Piwik\Option; +use Piwik\Sequence; +use Psr\Log\LoggerInterface; + /** * Class Archiver * @@ -31,6 +38,27 @@ class Archiver extends \Piwik\Plugin\Archiver * This is only an example record name, so feel free to change it to suit your needs. */ const EXAMPLEPLUGIN_ARCHIVE_RECORD = "ExamplePlugin_archive_record"; + const EXAMPLEPLUGIN_METRIC_NAME = 'ExamplePlugin_example_metric'; + const EXAMPLEPLUGIN_CONST_METRIC_NAME = 'ExamplePlugin_example_metric2'; + + private $daysFrom = '2016-07-08'; + + /** + * @var string + */ + private $requestedReport = null; + + public function __construct(ArchiveProcessor $processor) + { + parent::__construct($processor); + + $this->requestedReport = $processor->getParams()->getArchiveOnlyReport(); + if ($this->requestedReport) { + $processor->getParams()->setIsPartialArchive(true); + } + + $this->createSequence(); + } public function aggregateDayReport() { @@ -45,6 +73,25 @@ class Archiver extends \Piwik\Plugin\Archiver * $visitorReport = $visitorMetrics->getSerialized(); * $this->getProcessor()->insertBlobRecord(self::EXAMPLEPLUGIN_ARCHIVE_RECORD, $visitorReport); */ + + if ($this->isArchiving(self::EXAMPLEPLUGIN_METRIC_NAME)) { + // insert a test numeric metric that is the difference in days between the day we're archiving and + // $this->daysFrom. + $daysFrom = Date::factory($this->daysFrom); + $date = $this->getProcessor()->getParams()->getPeriod()->getDateStart(); + + $differenceInSeconds = $daysFrom->getTimestamp() - $date->getTimestamp(); + $differenceInDays = round($differenceInSeconds / 86400); + + $this->getProcessor()->insertNumericRecord(self::EXAMPLEPLUGIN_METRIC_NAME, $differenceInDays); + } + + if ($this->isArchiving(self::EXAMPLEPLUGIN_CONST_METRIC_NAME)) { + $archiveCount = $this->incrementArchiveCount(); + $archiveCount = 50 + $archiveCount; + $archiveCount += 5 - ($archiveCount % 5); // round up to nearest 5 multiple to avoid random test failures + $this->getProcessor()->insertNumericRecord(self::EXAMPLEPLUGIN_CONST_METRIC_NAME, $archiveCount); + } } public function aggregateMultipleReports() @@ -57,5 +104,41 @@ class Archiver extends \Piwik\Plugin\Archiver * * $this->getProcessor()->aggregateDataTableRecords(self::EXAMPLEPLUGIN_ARCHIVE_RECORD); */ + + $reports = []; + if ($this->isArchiving(self::EXAMPLEPLUGIN_METRIC_NAME)) { + $reports[] = self::EXAMPLEPLUGIN_METRIC_NAME; + } + if ($this->isArchiving(self::EXAMPLEPLUGIN_CONST_METRIC_NAME)) { + $reports[] = self::EXAMPLEPLUGIN_CONST_METRIC_NAME; + } + $this->getProcessor()->aggregateNumericMetrics($reports); + } + + private function incrementArchiveCount() + { + $sequence = new Sequence('ExamplePlugin_archiveCount'); + $result = $sequence->getNextId(); + return $result; + } + + private function isArchiving(string $reportName) + { + return empty($this->requestedReport) || $this->requestedReport == $reportName; + } + + private function createSequence() + { + $sequence = new Sequence('ExamplePlugin_archiveCount'); + if (!$sequence->exists()) { + for ($i = 0; $i < 100; ++$i) { + try { + $sequence->create(); + break; + } catch (\Exception $ex) { + // ignore + } + } + } } } diff --git a/plugins/ExamplePlugin/ExamplePlugin.php b/plugins/ExamplePlugin/ExamplePlugin.php index f974975c42..8b737f5784 100644 --- a/plugins/ExamplePlugin/ExamplePlugin.php +++ b/plugins/ExamplePlugin/ExamplePlugin.php @@ -10,4 +10,18 @@ namespace Piwik\Plugins\ExamplePlugin; class ExamplePlugin extends \Piwik\Plugin { + public function registerEvents() + { + return [ + 'CronArchive.getArchivingAPIMethodForPlugin' => 'getArchivingAPIMethodForPlugin', + ]; + } + + // support archiving just this plugin via core:archive + public function getArchivingAPIMethodForPlugin(&$method, $plugin) + { + if ($plugin == 'ExamplePlugin') { + $method = 'ExamplePlugin.getExampleArchivedMetric'; + } + } } diff --git a/plugins/GeoIp2/LocationProvider/GeoIp2/ServerModule.php b/plugins/GeoIp2/LocationProvider/GeoIp2/ServerModule.php index 5ebe6114ce..f1baf19552 100644 --- a/plugins/GeoIp2/LocationProvider/GeoIp2/ServerModule.php +++ b/plugins/GeoIp2/LocationProvider/GeoIp2/ServerModule.php @@ -16,7 +16,9 @@ use Piwik\Piwik; use Piwik\Plugins\GeoIp2\LocationProvider\GeoIp2; use Piwik\Plugins\UserCountry\LocationProvider; use Piwik\Plugins\GeoIp2\SystemSettings; +use Piwik\SettingsServer; use Piwik\Url; +use Piwik\View; /** * A LocationProvider that uses an GeoIP 2 module installed in an HTTP Server. @@ -250,7 +252,12 @@ class ServerModule extends GeoIp2 $configUrl = Url::getCurrentQueryStringWithParametersModified(array( 'module' => 'CoreAdminHome', 'action' => 'generalSettings' )); - $extraMessage .= '<br />'.Piwik::translate('GeoIp2_GeoIPVariablesConfigurationHere', ['<a href="'.$configUrl.'">', '</a>']); + if (!SettingsServer::isTrackerApiRequest()) { + // can't render in tracking mode as there is no theme + $view = new View('@GeoIp2/serverModule'); + $view->configUrl = $configUrl; + $extraMessage .= $view->render(); + } return array('id' => self::ID, 'title' => $title, diff --git a/plugins/GeoIp2/templates/serverModule.twig b/plugins/GeoIp2/templates/serverModule.twig new file mode 100644 index 0000000000..66df4331fb --- /dev/null +++ b/plugins/GeoIp2/templates/serverModule.twig @@ -0,0 +1 @@ +<br />{{ 'GeoIp2_GeoIPVariablesConfigurationHere'|translate('<a href="' ~ configUrl|e('html_attr') ~ '">', '</a>')|raw }}
\ No newline at end of file diff --git a/plugins/Widgetize/Controller.php b/plugins/Widgetize/Controller.php index e53c27589f..6a8376b16e 100644 --- a/plugins/Widgetize/Controller.php +++ b/plugins/Widgetize/Controller.php @@ -8,9 +8,13 @@ */ namespace Piwik\Plugins\Widgetize; +use Piwik\Access; use Piwik\Common; +use Piwik\Container\StaticContainer; use Piwik\FrontController; use Piwik\Piwik; +use Piwik\Session\SessionInitializer; +use Piwik\Url; use Piwik\View; /** @@ -27,6 +31,12 @@ class Controller extends \Piwik\Plugin\Controller public function iframe() { + $token_auth = Common::getRequestVar('token_auth', '', 'string'); + + if (!empty($token_auth) && Access::getInstance()->isUserHasSomeAdminAccess() && !defined('PIWIK_TEST_MODE')) { + throw new \Exception(Piwik::translate('Widgetize_ViewAccessRequired')); + } + $this->init(); $controllerName = Common::getRequestVar('moduleToWidgetize'); @@ -82,4 +92,5 @@ class Controller extends \Piwik\Plugin\Controller return $view->render(); } + } diff --git a/tests/PHPUnit/Fixtures/RawArchiveDataWithTempAndInvalidated.php b/tests/PHPUnit/Fixtures/RawArchiveDataWithTempAndInvalidated.php index 36c1a0ba49..408a090d18 100644 --- a/tests/PHPUnit/Fixtures/RawArchiveDataWithTempAndInvalidated.php +++ b/tests/PHPUnit/Fixtures/RawArchiveDataWithTempAndInvalidated.php @@ -70,7 +70,6 @@ class RawArchiveDataWithTempAndInvalidated extends Fixture 'ts_archived' => '2015-02-15 08:12:13' ), - // valid temporary array( // only valid 'idarchive' => 5, @@ -262,12 +261,35 @@ class RawArchiveDataWithTempAndInvalidated extends Fixture 'ts_archived' => '2015-02-09 14:13:14' ), + // old done partial (should be purged) + array( + 'idarchive' => 22, + 'idsite' => 1, + 'name' => 'done', + 'value' => ArchiveWriter::DONE_PARTIAL, + 'date1' => '2015-02-10', + 'date2' => '2015-02-10', + 'period' => 1, + 'ts_archived' => '2015-02-11 10:13:14' + ), + + // new done partial (should not be purged) + array( + 'idarchive' => 23, + 'idsite' => 1, + 'name' => 'done', + 'value' => ArchiveWriter::DONE_PARTIAL, + 'date1' => '2015-02-10', + 'date2' => '2015-02-10', + 'period' => 1, + 'ts_archived' => '2015-02-11 16:13:14' + ), ); private static $segmentArchiveData = array( array( - 'idarchive' => 22, + 'idarchive' => 24, 'idsite' => 1, 'name' => 'doneeb5d2797aedd15d819b1a20425982850', // Raw segment = abcd1234abcd5678 'value' => ArchiveWriter::DONE_OK, @@ -277,7 +299,7 @@ class RawArchiveDataWithTempAndInvalidated extends Fixture 'ts_archived' => '2015-02-03 12:12:12' ), array( - 'idarchive' => 23, + 'idarchive' => 25, 'idsite' => 1, 'name' => 'doneeb5d2797aedd15d819b1a20425982850.MyPlugin', // Raw segment = abcd1234abcd5678 'value' => ArchiveWriter::DONE_OK, @@ -287,7 +309,7 @@ class RawArchiveDataWithTempAndInvalidated extends Fixture 'ts_archived' => '2015-02-03 12:12:12' ), array( - 'idarchive' => 24, + 'idarchive' => 26, 'idsite' => 2, 'name' => 'doneeb5d2797aedd15d819b1a20425982850', // Raw segment = abcd1234abcd5678 'value' => ArchiveWriter::DONE_OK, @@ -297,7 +319,7 @@ class RawArchiveDataWithTempAndInvalidated extends Fixture 'ts_archived' => '2015-02-03 12:12:12' ), array( - 'idarchive' => 25, + 'idarchive' => 27, 'idsite' => 2, 'name' => 'doneeb5d2797aedd15d819b1a20425982850.MyPlugin', // Raw segment = abcd1234abcd5678 'value' => ArchiveWriter::DONE_OK, @@ -307,7 +329,7 @@ class RawArchiveDataWithTempAndInvalidated extends Fixture 'ts_archived' => '2015-02-03 12:12:12' ), array( - 'idarchive' => 26, + 'idarchive' => 28, 'idsite' => 1, 'name' => 'done1e39a89fcc269acc36bd4d7c742763ed', // Raw segment = 9876fedc5432abcd 'value' => ArchiveWriter::DONE_OK, @@ -317,7 +339,7 @@ class RawArchiveDataWithTempAndInvalidated extends Fixture 'ts_archived' => '2015-02-03 12:12:12' ), array( - 'idarchive' => 27, + 'idarchive' => 29, 'idsite' => 2, 'name' => 'done00c6ee2e21a7548de6260cf72c4f4b5b', // Raw segment = hash1 'value' => ArchiveWriter::DONE_OK, @@ -327,7 +349,7 @@ class RawArchiveDataWithTempAndInvalidated extends Fixture 'ts_archived' => '2015-02-03 12:12:12' ), array( - 'idarchive' => 28, + 'idarchive' => 30, 'idsite' => 2, 'name' => 'done58833651db311ba4bc11cb26b1900b0f', // Raw segment = hash2 'value' => ArchiveWriter::DONE_OK, @@ -337,7 +359,7 @@ class RawArchiveDataWithTempAndInvalidated extends Fixture 'ts_archived' => '2015-02-03 12:12:12' ), array( - 'idarchive' => 29, + 'idarchive' => 31, 'idsite' => 2, 'name' => 'done58833651db311ba4bc11cb26b1900b0f.MyPlugin', // Raw segment = hash2 'value' => ArchiveWriter::DONE_OK, @@ -347,7 +369,7 @@ class RawArchiveDataWithTempAndInvalidated extends Fixture 'ts_archived' => '2015-02-03 12:12:12' ), array( - 'idarchive' => 30, + 'idarchive' => 32, 'idsite' => 2, 'name' => 'done1a4ead8b39d17dfe89418452c9bba770', // Raw segment = hash3 'value' => ArchiveWriter::DONE_OK, @@ -357,7 +379,7 @@ class RawArchiveDataWithTempAndInvalidated extends Fixture 'ts_archived' => '2015-02-03 12:12:12' ), array( - 'idarchive' => 31, + 'idarchive' => 33, 'idsite' => 2, 'name' => 'done1a4ead8b39d17dfe89418452c9bba770', // Raw segment = hash3 'value' => ArchiveWriter::DONE_OK, @@ -371,7 +393,7 @@ class RawArchiveDataWithTempAndInvalidated extends Fixture public static $dummyArchiveDataNoInvalidated = [ // two archives w/ DONE_OK for a new site (no invalidated archives for site) [ - 'idarchive' => 32, + 'idarchive' => 34, 'idsite' => 4, 'name' => 'done', 'value' => ArchiveWriter::DONE_OK, @@ -381,7 +403,7 @@ class RawArchiveDataWithTempAndInvalidated extends Fixture 'ts_archived' => '2015-02-27 10:12:12' ], [ - 'idarchive' => 33, + 'idarchive' => 35, 'idsite' => 4, 'name' => 'done', 'value' => ArchiveWriter::DONE_OK, @@ -558,4 +580,13 @@ class RawArchiveDataWithTempAndInvalidated extends Fixture $expectedExistingArchives = array(15); $this->assertArchivesExist($expectedExistingArchives, $date); } + + public function assertPartialArchivesPurged(Date $date) + { + $expectedPurgedArchives = [22]; + $this->assertArchivesDoNotExist($expectedPurgedArchives, $date); + + $expectedExistingArchives = [23]; + $this->assertArchivesExist($expectedExistingArchives, $date); + } }
\ No newline at end of file diff --git a/tests/PHPUnit/Framework/Mock/FakeLogger.php b/tests/PHPUnit/Framework/Mock/FakeLogger.php index 3ee8a182d0..7cc89eca55 100644 --- a/tests/PHPUnit/Framework/Mock/FakeLogger.php +++ b/tests/PHPUnit/Framework/Mock/FakeLogger.php @@ -25,6 +25,10 @@ class FakeLogger extends AbstractLogger implements LoggerInterface public function log($level, $message, array $context = array()) { + if (strpos($message, 'Running command') !== false) { + return; + } + $record = $this->processor->__invoke(array('message' => $message, 'context' => $context)); $this->output .= $record['message'] . PHP_EOL; diff --git a/tests/PHPUnit/Integration/Archive/ArchivePurgerTest.php b/tests/PHPUnit/Integration/Archive/ArchivePurgerTest.php index 4c2d6c6dc0..c73feeb7b1 100644 --- a/tests/PHPUnit/Integration/Archive/ArchivePurgerTest.php +++ b/tests/PHPUnit/Integration/Archive/ArchivePurgerTest.php @@ -99,8 +99,9 @@ class ArchivePurgerTest extends IntegrationTestCase self::$fixture->assertInvalidatedArchivesPurged($this->february); self::$fixture->assertInvalidatedArchivesNotPurged($this->january); + self::$fixture->assertPartialArchivesPurged($this->february); - $this->assertEquals(9 * RawArchiveDataWithTempAndInvalidated::ROWS_PER_ARCHIVE, $deletedRowCount); + $this->assertEquals(10 * RawArchiveDataWithTempAndInvalidated::ROWS_PER_ARCHIVE, $deletedRowCount); $this->checkNoDuplicateArchives(); } @@ -142,7 +143,7 @@ class ArchivePurgerTest extends IntegrationTestCase //Archive #29 also has a deleted segment but it's before the purge threshold so it stays for now. $deletedRowCount = $this->archivePurger->purgeDeletedSegmentArchives($this->january, $segmentsToDelete); $this->assertEquals(4 * RawArchiveDataWithTempAndInvalidated::ROWS_PER_ARCHIVE, $deletedRowCount); - self::$fixture->assertArchivesDoNotExist(array(24, 25, 26, 30), $this->january); + self::$fixture->assertArchivesDoNotExist(array(26, 27, 28, 32), $this->january); } public function test_purgeNoSegmentArchives_preservesSingleSiteSegmentArchivesForDeletedAllSiteSegment() @@ -158,7 +159,7 @@ class ArchivePurgerTest extends IntegrationTestCase // Archives for idsite=1 should be purged, but those for idsite=2 can stay $deletedRowCount = $this->archivePurger->purgeDeletedSegmentArchives($this->january, $segmentsToDelete); $this->assertEquals(2 * RawArchiveDataWithTempAndInvalidated::ROWS_PER_ARCHIVE, $deletedRowCount); - self::$fixture->assertArchivesDoNotExist(array(22, 23), $this->january); + self::$fixture->assertArchivesDoNotExist(array(24, 25), $this->january); } public function test_purgeNoSegmentArchives_blankSegmentName() diff --git a/tests/PHPUnit/Integration/ArchiveProcessingTest.php b/tests/PHPUnit/Integration/ArchiveProcessingTest.php index 9e63b2536d..b1df5a98a9 100644 --- a/tests/PHPUnit/Integration/ArchiveProcessingTest.php +++ b/tests/PHPUnit/Integration/ArchiveProcessingTest.php @@ -10,10 +10,14 @@ namespace Piwik\Tests\Integration; use Exception; use Piwik\Access; +use Piwik\Archive; use Piwik\ArchiveProcessor; use Piwik\ArchiveProcessor\Rules; use Piwik\Common; use Piwik\DataAccess\ArchiveTableCreator; +use Piwik\DataAccess\ArchiveWriter; +use Piwik\DataAccess\LogAggregator; +use Piwik\DataTable; use Piwik\Date; use Piwik\Db; use Piwik\Db\BatchInsert; @@ -24,6 +28,7 @@ use Piwik\Plugins\SitesManager\API; use Piwik\Segment; use Piwik\SettingsServer; use Piwik\Site; +use Piwik\Tests\Framework\Fixture; use Piwik\Tests\Framework\Mock\FakeAccess; use Piwik\Tests\Framework\TestCase\IntegrationTestCase; @@ -63,15 +68,7 @@ class ArchiveProcessingTest extends IntegrationTestCase */ private function _createWebsite($timezone = 'UTC') { - $idSite = API::getInstance()->addSite( - "site1", - array("http://piwik.net"), - $ecommerce = 0, - $siteSearch = 1, $searchKeywordParameters = null, $searchCategoryParameters = null, - $excludedIps = "", - $excludedQueryParameters = "", - $timezone); - + $idSite = Fixture::createWebsite('2013-03-04', 0, false, false, 1, null, null, $timezone); Site::clearCache(); return new Site($idSite); } @@ -89,12 +86,60 @@ class ArchiveProcessingTest extends IntegrationTestCase $site = $this->_createWebsite($siteTimezone); $date = Date::factory($dateLabel); $period = Period\Factory::build($periodLabel, $date); - $segment = new Segment('', $site->getId(), $period->getDateStart(), $period->getDateEnd()); + $segment = new Segment('', [$site->getId()], $period->getDateStart(), $period->getDateEnd()); $params = new ArchiveProcessor\Parameters($site, $period, $segment); return new ArchiveProcessorTest($params); } + private function _createArchiveProcessorInst($periodLabel, $dateLabel, $idSite, $archiveOnly = false, $plugin = false) + { + $period = Period\Factory::build($periodLabel, $dateLabel); + $segment = new Segment('', [$idSite]); + + $params = new ArchiveProcessor\Parameters(new Site($idSite), $period, $segment); + if ($archiveOnly) { + $params->setRequestedPlugin($plugin); + $params->setArchiveOnlyReport($archiveOnly); + } + $archiveWriter = new ArchiveWriter($params); + $logAggregator = new LogAggregator($params); + $archiveProcessor = new class($params, $archiveWriter, $logAggregator) extends ArchiveProcessor { + private $captureInserts = false; + private $capturedInserts = []; + + public function captureInserts() + { + $this->captureInserts = true; + } + + public function insertNumericRecord($name, $value) + { + if ($this->captureInserts) { + $this->capturedInserts[] = [$name, $value]; + } else { + parent::insertNumericRecord($name, $value); + } + } + + public function insertBlobRecord($name, $values) + { + if ($this->captureInserts) { + $this->capturedInserts[] = [$name, $values]; + } else { + parent::insertBlobRecord($name, $values); + } + } + + public function getCapturedInserts() + { + return $this->capturedInserts; + } + }; + + return [$archiveProcessor, $archiveWriter, $params]; + } + /** * test of validity of an archive, for a month not finished */ @@ -103,7 +148,7 @@ class ArchiveProcessingTest extends IntegrationTestCase $siteTimezone = 'UTC+10'; $now = time(); // this test fails in the last 10 hours of the last day of the month - if(date('m', $now) != date('m', $now + 10 * 3600)) { + if (date('m', $now) != date('m', $now + 10 * 3600)) { $this->markTestSkipped('testInitCurrentMonth will fail in the last hours of the month, skipping...'); } @@ -120,7 +165,7 @@ class ArchiveProcessingTest extends IntegrationTestCase { // $messageIfFails = Date::factory($expected)->getDatetime() . " != " . Date::factory($processed)->getDatetime(); $messageIfFails = "Expected [$expected] but got [$processed]"; - $this->assertTrue( abs($expected-$processed) <=4 , $messageIfFails); + $this->assertTrue(abs($expected - $processed) <= 4, $messageIfFails); } /** @@ -277,12 +322,12 @@ class ArchiveProcessingTest extends IntegrationTestCase $skippedOnce = true; $this->fail( 'Performance notice: LOAD DATA [LOCAL] INFILE query is not working, so Piwik will fallback to using plain INSERTs ' - . ' which will result in a slightly slower Archiving process.' - . ". \n" - . ' The error Messages from MySQL were: ' - . $didWeUseBulk - . "\n\n Learn more how to enable LOAD LOCAL DATA INFILE see the Mysql doc (http://dev.mysql.com/doc/refman/5.0/en/load-data-local.html) " - . "\n or ask in this Piwik ticket (https://github.com/matomo-org/matomo/issues/3605)" + . ' which will result in a slightly slower Archiving process.' + . ". \n" + . ' The error Messages from MySQL were: ' + . $didWeUseBulk + . "\n\n Learn more how to enable LOAD LOCAL DATA INFILE see the Mysql doc (http://dev.mysql.com/doc/refman/5.0/en/load-data-local.html) " + . "\n or ask in this Piwik ticket (https://github.com/matomo-org/matomo/issues/3605)" ); } return $didWeUseBulk; @@ -364,6 +409,232 @@ class ArchiveProcessingTest extends IntegrationTestCase $this->fail('Exception expected'); } + public function test_aggregateNumericMetrics_aggregatesCorrectly() + { + $allMetrics = [ + '2015-02-03' => [ + 'nb_visits' => 2, + 'max_actions' => 3, + ], + '2015-02-04' => [ + 'nb_visits' => 2, + 'max_actions' => 4, + ], + '2015-02-05' => [ + 'nb_visits' => 2, + 'max_actions' => 1, + ], + ]; + + $site = $this->_createWebsite('UTC'); + + foreach ($allMetrics as $date => $metrics) { + /** @var ArchiveWriter $archiveWriter */ + list($archiveProcessor, $archiveWriter, $params) = $this->_createArchiveProcessorInst('day', $date, $site->getId()); + $archiveWriter->initNewArchive(); + + $archiveProcessor->insertNumericRecords($metrics); + + $archiveWriter->finalizeArchive(); + } + + /** @var ArchiveProcessor $archiveProcessor */ + list($archiveProcessor, $archiveWriter, $params) = $this->_createArchiveProcessorInst('week', '2015-02-03', $site->getId()); + $archiveWriter->initNewArchive(); + + $archiveProcessor->captureInserts(); + $archiveProcessor->aggregateNumericMetrics(['nb_visits', 'max_actions']); + + $archiveWriter->finalizeArchive(); + + $capturedInserts = $archiveProcessor->getCapturedInserts(); + + $expected = [ + [ + 'nb_visits', + 6, + ], + [ + 'max_actions', + 4, + ] + ]; + + $this->assertEquals($expected, $capturedInserts); + } + + public function test_aggregateNumericMetrics_handlesPartialArchives() + { + $allMetrics = [ + '2015-02-03' => [ + 'nb_visits' => 2, + 'max_actions' => 1, + ], + '2015-02-04' => [ + 'nb_visits' => 2, + 'max_actions' => 3, + ], + '2015-02-05' => [ + 'nb_visits' => 2, + 'max_actions' => 4, + ], + ]; + + $site = $this->_createWebsite('UTC'); + + foreach ($allMetrics as $date => $metrics) { + /** @var ArchiveWriter $archiveWriter */ + list($archiveProcessor, $archiveWriter) = $this->_createArchiveProcessorInst('day', $date, $site->getId()); + $archiveWriter->initNewArchive(); + + $archiveProcessor->insertNumericRecords($metrics); + + $archiveWriter->finalizeArchive(); + } + + /** @var ArchiveProcessor $archiveProcessor */ + list($archiveProcessor, $archiveWriter, $params) = $this->_createArchiveProcessorInst('week', '2015-02-03', $site->getId(), 'nb_visits', 'VisitsSummary'); + $params->setIsPartialArchive(true); + $idArchive = $archiveWriter->initNewArchive(); + + $archiveProcessor->captureInserts(); + $archiveProcessor->aggregateNumericMetrics(['nb_visits']); + + $archiveWriter->finalizeArchive(); + + $capturedInserts = $archiveProcessor->getCapturedInserts(); + + $expected = [ + [ + 'nb_visits', + 6, + ], + ]; + + $archiveDoneFlag = Db::fetchOne("SELECT `value` FROM " . ArchiveTableCreator::getNumericTable(Date::factory('2015-02-03')) . " WHERE idarchive = ? AND name LIKE 'done%'", [$idArchive]); + $this->assertEquals(ArchiveWriter::DONE_PARTIAL, $archiveDoneFlag); + + $this->assertEquals($expected, $capturedInserts); + } + + public function test_aggregateDataTableRecords_aggregatesCorrectly() + { + $table1 = new DataTable(); + $table1->addRowsFromSimpleArray([ + ['label' => 'a', 'nb_visits' => 5, 'nb_actions' => 1], + ['label' => 'b', 'nb_visits' => 3, 'nb_actions' => 1], + ]); + $table2 = new DataTable(); + $table2->addRowsFromSimpleArray([ + ['label' => 'a', 'nb_visits' => 2, 'nb_actions' => 2], + ]); + $table3 = new DataTable(); + $table3->addRowsFromSimpleArray([ + ['label' => 'b', 'nb_visits' => 4, 'nb_actions' => 3], + ]); + + $tables = [ + '2015-02-03' => $table1, + '2015-02-04' => $table2, + '2015-02-05' => $table3, + ]; + + $site = $this->_createWebsite('UTC'); + + foreach ($tables as $date => $table) { + /** @var ArchiveWriter $archiveWriter */ + list($archiveProcessor, $archiveWriter) = $this->_createArchiveProcessorInst('day', $date, $site->getId()); + $archiveWriter->initNewArchive(); + + $tableSerialized = $table->getSerialized(); + $archiveProcessor->insertBlobRecord('Actions_test_value', $tableSerialized); + + $archiveWriter->finalizeArchive(); + } + + list($archiveProcessor, $archiveWriter) = $this->_createArchiveProcessorInst('week', '2015-02-03', $site->getId()); + $archiveWriter->initNewArchive(); + + $archiveProcessor->captureInserts(); + $archiveProcessor->aggregateDataTableRecords('Actions_test_value'); + + $archiveWriter->finalizeArchive(); + + $capturedInserts = $archiveProcessor->getCapturedInserts(); + $capturedInsertTable = DataTable::fromSerializedArray($capturedInserts[0][1][0]); + $capturedInsertTable = $this->getXml($capturedInsertTable); + + $expectedXml = <<<END +<?xml version="1.0" encoding="utf-8" ?> +<result> + <row> + <label>a</label> + <nb_visits>7</nb_visits> + <nb_actions>3</nb_actions> + </row> + <row> + <label>b</label> + <nb_visits>7</nb_visits> + <nb_actions>4</nb_actions> + </row> +</result> +END; + + $this->assertEquals($expectedXml, $capturedInsertTable); + } + + public function test_aggregateDataTableRecords_handlesPartialArchives() + { + $table1 = new DataTable(); + $table1->addRowsFromSimpleArray([ + ['label' => 'a', 'nb_visits' => 5, 'nb_actions' => 1], + ['label' => 'b', 'nb_visits' => 3, 'nb_actions' => 1], + ]); + $table2 = new DataTable(); + $table2->addRowsFromSimpleArray([ + ['label' => 'a', 'nb_visits' => 2, 'nb_actions' => 2], + ]); + $table3 = new DataTable(); + $table3->addRowsFromSimpleArray([ + ['label' => 'b', 'nb_visits' => 4, 'nb_actions' => 3], + ]); + + $tables = [ + '2015-02-03' => $table1, + '2015-02-04' => $table2, + '2015-02-05' => $table3, + ]; + + $site = $this->_createWebsite('UTC'); + + foreach ($tables as $date => $table) { + /** @var ArchiveWriter $archiveWriter */ + list($archiveProcessor, $archiveWriter) = $this->_createArchiveProcessorInst('day', $date, $site->getId()); + $archiveWriter->initNewArchive(); + + $tableSerialized = $table->getSerialized(); + $archiveProcessor->insertBlobRecord('Actions_test_value', $tableSerialized); + + $archiveWriter->finalizeArchive(); + } + + /** @var ArchiveProcessor $archiveProcessor */ + list($archiveProcessor, $archiveWriter, $params) = $this->_createArchiveProcessorInst('week', '2015-02-03', $site->getId(), 'Actions_test_value', 'VisitsSummary'); + $params->setIsPartialArchive(true); + $idArchive = $archiveWriter->initNewArchive(); + + $archiveProcessor->captureInserts(); + $archiveProcessor->aggregateDataTableRecords('Actions_test_value'); + + $archiveWriter->finalizeArchive(); + + $capturedInserts = $archiveProcessor->getCapturedInserts(); + $this->assertNotEmpty($capturedInserts); + + $archiveDoneFlag = Db::fetchOne("SELECT `value` FROM " . ArchiveTableCreator::getNumericTable(Date::factory('2015-02-03')) . " WHERE idarchive = ? AND name LIKE 'done%'", [$idArchive]); + $this->assertEquals(ArchiveWriter::DONE_PARTIAL, $archiveDoneFlag); + } + protected function _checkTableIsExpected($table, $data) { $fetched = Db::fetchAll('SELECT * FROM ' . $table); @@ -450,4 +721,11 @@ class ArchiveProcessingTest extends IntegrationTestCase 'Piwik\Access' => new FakeAccess() ); } + + private function getXml(DataTable $capturedInsertTable) + { + $xml = new DataTable\Renderer\Xml(); + $xml->setTable($capturedInsertTable); + return $xml->render(); + } } diff --git a/tests/PHPUnit/Integration/ArchiveProcessor/LoaderTest.php b/tests/PHPUnit/Integration/ArchiveProcessor/LoaderTest.php index 2b5f18c238..3690ef45ea 100644 --- a/tests/PHPUnit/Integration/ArchiveProcessor/LoaderTest.php +++ b/tests/PHPUnit/Integration/ArchiveProcessor/LoaderTest.php @@ -13,6 +13,7 @@ namespace Piwik\Tests\Integration\ArchiveProcessor; use Piwik\Archive\ArchiveInvalidator; use Piwik\ArchiveProcessor\Parameters; use Piwik\ArchiveProcessor\Loader; +use Piwik\ArchiveProcessor\PluginsArchiver; use Piwik\Common; use Piwik\Config; use Piwik\Container\StaticContainer; @@ -22,7 +23,9 @@ use Piwik\Date; use Piwik\Db; use Piwik\Period\Factory; use Piwik\Piwik; +use Piwik\Plugins\ExamplePlugin\Archiver; use Piwik\Segment; +use Piwik\Sequence; use Piwik\Site; use Piwik\Tests\Framework\Fixture; use Piwik\Tests\Framework\TestCase\IntegrationTestCase; @@ -37,6 +40,760 @@ class LoaderTest extends IntegrationTestCase Fixture::createWebsite('2012-02-03 00:00:00'); } + /** + * @dataProvider getTestDataForArchiving + */ + public function test_pluginOnlyArchivingCreatesAndReusesCorrectArchives($archiveData, $params, $expectedArchives, $archiveTwice) + { + $_GET['pluginOnly'] = 1; + $_GET['trigger'] = 'archivephp'; + + Date::$now = strtotime('2018-03-04 05:00:00'); + + list($idSite, $period, $date, $segment, $plugin, $report) = $params; + + $t = Fixture::getTracker($idSite, $date); + $t->setUrl('http://slkdfj.com'); + $t->doTrackPageView('alsdkjf'); + + $params = new Parameters(new Site($idSite), Factory::build($period, $date), new Segment($segment, [$idSite])); + $params->setRequestedPlugin($plugin); + if ($report) { + $params->setArchiveOnlyReport($report); + } + + $this->insertArchiveData($archiveData); + + $loader = new Loader($params); + $loader->prepareArchive($params->getRequestedPlugin()); + + if ($archiveTwice) { + if (is_array($archiveTwice)) { + list($idSite2, $period2, $date2, $segment2, $plugin2, $report2) = $archiveTwice; + + $params2 = new Parameters(new Site($idSite2), Factory::build($period2, $date2), new Segment($segment2, [$idSite2])); + $params2->setRequestedPlugin($plugin2); + if ($report2) { + $params2->setArchiveOnlyReport($report2); + } + } else { + $params2 = $params; + } + + $loader2 = new Loader($params2); + $loader2->prepareArchive($params->getRequestedPlugin()); + } + + $actualArchives = $this->getArchives(); + if ($actualArchives != $expectedArchives) { + var_export($actualArchives); + } + $this->assertEquals($expectedArchives, $actualArchives); + } + + public function getTestDataForArchiving() + { + $pluginSpecificArchive = [1, 'day', '2018-03-03', '', 'ExamplePlugin', false]; + + $reportSpecificArchive1 = [1, 'day', '2018-03-03', '', 'ExamplePlugin', Archiver::EXAMPLEPLUGIN_METRIC_NAME]; + $reportSpecificArchive2 = [1, 'day', '2018-03-03', '', 'ExamplePlugin', Archiver::EXAMPLEPLUGIN_CONST_METRIC_NAME]; + + $unloadedPluginArchive = [1, 'day', '2018-03-03', '', 'MyImaginaryPlugin', false]; + + return [ + // no archive, archive specific plugin + [ + [], + $pluginSpecificArchive, + array ( + array ( + 'idarchive' => '1', + 'idsite' => '1', + 'date1' => '2018-03-03', + 'date2' => '2018-03-03', + 'period' => '1', + 'name' => 'bounce_count', + 'value' => '1', + ), + array ( + 'idarchive' => '1', + 'idsite' => '1', + 'date1' => '2018-03-03', + 'date2' => '2018-03-03', + 'period' => '1', + 'name' => 'done.VisitsSummary', + 'value' => '1', + ), + array ( + 'idarchive' => '1', + 'idsite' => '1', + 'date1' => '2018-03-03', + 'date2' => '2018-03-03', + 'period' => '1', + 'name' => 'max_actions', + 'value' => '1', + ), + array ( + 'idarchive' => '1', + 'idsite' => '1', + 'date1' => '2018-03-03', + 'date2' => '2018-03-03', + 'period' => '1', + 'name' => 'nb_actions', + 'value' => '1', + ), + array ( + 'idarchive' => '1', + 'idsite' => '1', + 'date1' => '2018-03-03', + 'date2' => '2018-03-03', + 'period' => '1', + 'name' => 'nb_uniq_visitors', + 'value' => '1', + ), + array ( + 'idarchive' => '1', + 'idsite' => '1', + 'date1' => '2018-03-03', + 'date2' => '2018-03-03', + 'period' => '1', + 'name' => 'nb_visits', + 'value' => '1', + ), + array ( + 'idarchive' => '2', + 'idsite' => '1', + 'date1' => '2018-03-03', + 'date2' => '2018-03-03', + 'period' => '1', + 'name' => 'done.ExamplePlugin', + 'value' => '1', + ), + array ( + 'idarchive' => '2', + 'idsite' => '1', + 'date1' => '2018-03-03', + 'date2' => '2018-03-03', + 'period' => '1', + 'name' => 'ExamplePlugin_example_metric', + 'value' => '-603', + ), + array ( + 'idarchive' => '2', + 'idsite' => '1', + 'date1' => '2018-03-03', + 'date2' => '2018-03-03', + 'period' => '1', + 'name' => 'ExamplePlugin_example_metric2', + 'value' => '55', + ), + ), + false, + ], + + // all plugins, recent, archive specific plugin + [ + [ + ['idarchive' => 1, 'idsite' => 1, 'date1' => '2018-03-03', 'date2' => '2018-03-03', 'period' => 1, 'name' => 'done', 'value' => ArchiveWriter::DONE_OK, 'ts_archived' => '2018-03-04 04:50:00'], + ['idarchive' => 1, 'idsite' => 1, 'date1' => '2018-03-03', 'date2' => '2018-03-03', 'period' => 1, 'name' => 'nb_visits', 'value' => 12, 'ts_archived' => '2018-03-04 04:50:00'], + ['idarchive' => 1, 'idsite' => 1, 'date1' => '2018-03-03', 'date2' => '2018-03-03', 'period' => 1, 'name' => 'nb_visits_converted', 'value' => 3, 'ts_archived' => '2018-03-04 04:50:00'], + ], + $pluginSpecificArchive, + array ( // done archive already exists and is recent, so we don't archive the plugin + array ( + 'idarchive' => '1', + 'idsite' => '1', + 'date1' => '2018-03-03', + 'date2' => '2018-03-03', + 'period' => '1', + 'name' => 'done', + 'value' => '1', + ), + array ( + 'idarchive' => '1', + 'idsite' => '1', + 'date1' => '2018-03-03', + 'date2' => '2018-03-03', + 'period' => '1', + 'name' => 'nb_visits', + 'value' => '12', + ), + array ( + 'idarchive' => '1', + 'idsite' => '1', + 'date1' => '2018-03-03', + 'date2' => '2018-03-03', + 'period' => '1', + 'name' => 'nb_visits_converted', + 'value' => '3', + ), + ), + false, + ], + + // visitssummary, recent, archive specific plugin + [ + [ + ['idarchive' => 1, 'idsite' => 1, 'date1' => '2018-03-03', 'date2' => '2018-03-03', 'period' => 1, 'name' => 'done.VisitsSummary', 'value' => ArchiveWriter::DONE_OK, 'ts_archived' => '2018-03-04 04:50:00'], + ['idarchive' => 1, 'idsite' => 1, 'date1' => '2018-03-03', 'date2' => '2018-03-03', 'period' => 1, 'name' => 'nb_visits', 'value' => 12, 'ts_archived' => '2018-03-04 04:50:00'], + ['idarchive' => 1, 'idsite' => 1, 'date1' => '2018-03-03', 'date2' => '2018-03-03', 'period' => 1, 'name' => 'nb_visits_converted', 'value' => 3, 'ts_archived' => '2018-03-04 04:50:00'], + ], + $pluginSpecificArchive, + array ( + array ( + 'idarchive' => '1', + 'idsite' => '1', + 'date1' => '2018-03-03', + 'date2' => '2018-03-03', + 'period' => '1', + 'name' => 'done.VisitsSummary', + 'value' => '1', + ), + array ( + 'idarchive' => '1', + 'idsite' => '1', + 'date1' => '2018-03-03', + 'date2' => '2018-03-03', + 'period' => '1', + 'name' => 'nb_visits', + 'value' => '12', + ), + array ( + 'idarchive' => '1', + 'idsite' => '1', + 'date1' => '2018-03-03', + 'date2' => '2018-03-03', + 'period' => '1', + 'name' => 'nb_visits_converted', + 'value' => '3', + ), + array ( + 'idarchive' => '2', + 'idsite' => '1', + 'date1' => '2018-03-03', + 'date2' => '2018-03-03', + 'period' => '1', + 'name' => 'done.ExamplePlugin', + 'value' => '1', + ), + array ( + 'idarchive' => '2', + 'idsite' => '1', + 'date1' => '2018-03-03', + 'date2' => '2018-03-03', + 'period' => '1', + 'name' => 'ExamplePlugin_example_metric', + 'value' => '-603', + ), + array ( + 'idarchive' => '2', + 'idsite' => '1', + 'date1' => '2018-03-03', + 'date2' => '2018-03-03', + 'period' => '1', + 'name' => 'ExamplePlugin_example_metric2', + 'value' => '55', + ), + ), + false, + ], + + // all plugins, old, archive specific plugin + [ + [ + ['idarchive' => 1, 'idsite' => 1, 'date1' => '2018-03-03', 'date2' => '2018-03-03', 'period' => 1, 'name' => 'done', 'value' => ArchiveWriter::DONE_OK, 'ts_archived' => '2018-03-01 04:50:00'], + ['idarchive' => 1, 'idsite' => 1, 'date1' => '2018-03-03', 'date2' => '2018-03-03', 'period' => 1, 'name' => 'nb_visits', 'value' => 12, 'ts_archived' => '2018-03-01 04:50:00'], + ['idarchive' => 1, 'idsite' => 1, 'date1' => '2018-03-03', 'date2' => '2018-03-03', 'period' => 1, 'name' => 'nb_visits_converted', 'value' => 3, 'ts_archived' => '2018-03-01 04:50:00'], + ], + $pluginSpecificArchive, + array ( + array ( + 'idarchive' => '1', + 'idsite' => '1', + 'date1' => '2018-03-03', + 'date2' => '2018-03-03', + 'period' => '1', + 'name' => 'done', + 'value' => '1', + ), + array ( + 'idarchive' => '1', + 'idsite' => '1', + 'date1' => '2018-03-03', + 'date2' => '2018-03-03', + 'period' => '1', + 'name' => 'nb_visits', + 'value' => '12', + ), + array ( + 'idarchive' => '1', + 'idsite' => '1', + 'date1' => '2018-03-03', + 'date2' => '2018-03-03', + 'period' => '1', + 'name' => 'nb_visits_converted', + 'value' => '3', + ), + array ( + 'idarchive' => '2', + 'idsite' => '1', + 'date1' => '2018-03-03', + 'date2' => '2018-03-03', + 'period' => '1', + 'name' => 'done.ExamplePlugin', + 'value' => '1', + ), + array ( + 'idarchive' => '2', + 'idsite' => '1', + 'date1' => '2018-03-03', + 'date2' => '2018-03-03', + 'period' => '1', + 'name' => 'ExamplePlugin_example_metric', + 'value' => '-603', + ), + array ( + 'idarchive' => '2', + 'idsite' => '1', + 'date1' => '2018-03-03', + 'date2' => '2018-03-03', + 'period' => '1', + 'name' => 'ExamplePlugin_example_metric2', + 'value' => '55', + ), + ), + false, + ], + + // visitssummary, old, archive specific plugin + [ + [ + ['idarchive' => 1, 'idsite' => 1, 'date1' => '2018-03-03', 'date2' => '2018-03-03', 'period' => 1, 'name' => 'done.VisitsSummary', 'value' => ArchiveWriter::DONE_OK, 'ts_archived' => '2018-03-01 04:50:00'], + ['idarchive' => 1, 'idsite' => 1, 'date1' => '2018-03-03', 'date2' => '2018-03-03', 'period' => 1, 'name' => 'nb_visits', 'value' => 12, 'ts_archived' => '2018-03-01 04:50:00'], + ['idarchive' => 1, 'idsite' => 1, 'date1' => '2018-03-03', 'date2' => '2018-03-03', 'period' => 1, 'name' => 'nb_visits_converted', 'value' => 3, 'ts_archived' => '2018-03-01 04:50:00'], + ], + $pluginSpecificArchive, + array ( + array ( + 'idarchive' => '1', + 'idsite' => '1', + 'date1' => '2018-03-03', + 'date2' => '2018-03-03', + 'period' => '1', + 'name' => 'done.VisitsSummary', + 'value' => '1', + ), + array ( + 'idarchive' => '1', + 'idsite' => '1', + 'date1' => '2018-03-03', + 'date2' => '2018-03-03', + 'period' => '1', + 'name' => 'nb_visits', + 'value' => '12', + ), + array ( + 'idarchive' => '1', + 'idsite' => '1', + 'date1' => '2018-03-03', + 'date2' => '2018-03-03', + 'period' => '1', + 'name' => 'nb_visits_converted', + 'value' => '3', + ), + array ( + 'idarchive' => '2', + 'idsite' => '1', + 'date1' => '2018-03-03', + 'date2' => '2018-03-03', + 'period' => '1', + 'name' => 'done.ExamplePlugin', + 'value' => '1', + ), + array ( + 'idarchive' => '2', + 'idsite' => '1', + 'date1' => '2018-03-03', + 'date2' => '2018-03-03', + 'period' => '1', + 'name' => 'ExamplePlugin_example_metric', + 'value' => '-603', + ), + array ( + 'idarchive' => '2', + 'idsite' => '1', + 'date1' => '2018-03-03', + 'date2' => '2018-03-03', + 'period' => '1', + 'name' => 'ExamplePlugin_example_metric2', + 'value' => '55', + ), + ), + false, + ], + + // no archive, archive specific plugin, archive specific plugin again + [ + [], + $pluginSpecificArchive, + array ( + array ( + 'idarchive' => '1', + 'idsite' => '1', + 'date1' => '2018-03-03', + 'date2' => '2018-03-03', + 'period' => '1', + 'name' => 'bounce_count', + 'value' => '1', + ), + array ( + 'idarchive' => '1', + 'idsite' => '1', + 'date1' => '2018-03-03', + 'date2' => '2018-03-03', + 'period' => '1', + 'name' => 'done.VisitsSummary', + 'value' => '1', + ), + array ( + 'idarchive' => '1', + 'idsite' => '1', + 'date1' => '2018-03-03', + 'date2' => '2018-03-03', + 'period' => '1', + 'name' => 'max_actions', + 'value' => '1', + ), + array ( + 'idarchive' => '1', + 'idsite' => '1', + 'date1' => '2018-03-03', + 'date2' => '2018-03-03', + 'period' => '1', + 'name' => 'nb_actions', + 'value' => '1', + ), + array ( + 'idarchive' => '1', + 'idsite' => '1', + 'date1' => '2018-03-03', + 'date2' => '2018-03-03', + 'period' => '1', + 'name' => 'nb_uniq_visitors', + 'value' => '1', + ), + array ( + 'idarchive' => '1', + 'idsite' => '1', + 'date1' => '2018-03-03', + 'date2' => '2018-03-03', + 'period' => '1', + 'name' => 'nb_visits', + 'value' => '1', + ), + array ( + 'idarchive' => '2', + 'idsite' => '1', + 'date1' => '2018-03-03', + 'date2' => '2018-03-03', + 'period' => '1', + 'name' => 'done.ExamplePlugin', + 'value' => '1', + ), + array ( + 'idarchive' => '2', + 'idsite' => '1', + 'date1' => '2018-03-03', + 'date2' => '2018-03-03', + 'period' => '1', + 'name' => 'ExamplePlugin_example_metric', + 'value' => '-603', + ), + array ( + 'idarchive' => '2', + 'idsite' => '1', + 'date1' => '2018-03-03', + 'date2' => '2018-03-03', + 'period' => '1', + 'name' => 'ExamplePlugin_example_metric2', + 'value' => '55', + ), + ), + true, + ], + + // no archive, archive specific report, archive specific report again + [ + [], + $reportSpecificArchive1, + array ( + array ( + 'idarchive' => '1', + 'idsite' => '1', + 'date1' => '2018-03-03', + 'date2' => '2018-03-03', + 'period' => '1', + 'name' => 'bounce_count', + 'value' => '1', + ), + array ( + 'idarchive' => '1', + 'idsite' => '1', + 'date1' => '2018-03-03', + 'date2' => '2018-03-03', + 'period' => '1', + 'name' => 'done.VisitsSummary', + 'value' => '1', + ), + array ( + 'idarchive' => '1', + 'idsite' => '1', + 'date1' => '2018-03-03', + 'date2' => '2018-03-03', + 'period' => '1', + 'name' => 'max_actions', + 'value' => '1', + ), + array ( + 'idarchive' => '1', + 'idsite' => '1', + 'date1' => '2018-03-03', + 'date2' => '2018-03-03', + 'period' => '1', + 'name' => 'nb_actions', + 'value' => '1', + ), + array ( + 'idarchive' => '1', + 'idsite' => '1', + 'date1' => '2018-03-03', + 'date2' => '2018-03-03', + 'period' => '1', + 'name' => 'nb_uniq_visitors', + 'value' => '1', + ), + array ( + 'idarchive' => '1', + 'idsite' => '1', + 'date1' => '2018-03-03', + 'date2' => '2018-03-03', + 'period' => '1', + 'name' => 'nb_visits', + 'value' => '1', + ), + array ( + 'idarchive' => '2', + 'idsite' => '1', + 'date1' => '2018-03-03', + 'date2' => '2018-03-03', + 'period' => '1', + 'name' => 'done.ExamplePlugin', + 'value' => '5', + ), + array ( + 'idarchive' => '2', + 'idsite' => '1', + 'date1' => '2018-03-03', + 'date2' => '2018-03-03', + 'period' => '1', + 'name' => 'ExamplePlugin_example_metric', + 'value' => '-603', + ), + array ( + 'idarchive' => '3', + 'idsite' => '1', + 'date1' => '2018-03-03', + 'date2' => '2018-03-03', + 'period' => '1', + 'name' => 'done.ExamplePlugin', + 'value' => '5', + ), + array ( + 'idarchive' => '3', + 'idsite' => '1', + 'date1' => '2018-03-03', + 'date2' => '2018-03-03', + 'period' => '1', + 'name' => 'ExamplePlugin_example_metric', + 'value' => '-603', + ), + ), + true, + ], + + // no archive, archive specific report, archive different report again + [ + [], + $reportSpecificArchive1, + array ( + array ( + 'idarchive' => '1', + 'idsite' => '1', + 'date1' => '2018-03-03', + 'date2' => '2018-03-03', + 'period' => '1', + 'name' => 'bounce_count', + 'value' => '1', + ), + array ( + 'idarchive' => '1', + 'idsite' => '1', + 'date1' => '2018-03-03', + 'date2' => '2018-03-03', + 'period' => '1', + 'name' => 'done.VisitsSummary', + 'value' => '1', + ), + array ( + 'idarchive' => '1', + 'idsite' => '1', + 'date1' => '2018-03-03', + 'date2' => '2018-03-03', + 'period' => '1', + 'name' => 'max_actions', + 'value' => '1', + ), + array ( + 'idarchive' => '1', + 'idsite' => '1', + 'date1' => '2018-03-03', + 'date2' => '2018-03-03', + 'period' => '1', + 'name' => 'nb_actions', + 'value' => '1', + ), + array ( + 'idarchive' => '1', + 'idsite' => '1', + 'date1' => '2018-03-03', + 'date2' => '2018-03-03', + 'period' => '1', + 'name' => 'nb_uniq_visitors', + 'value' => '1', + ), + array ( + 'idarchive' => '1', + 'idsite' => '1', + 'date1' => '2018-03-03', + 'date2' => '2018-03-03', + 'period' => '1', + 'name' => 'nb_visits', + 'value' => '1', + ), + array ( + 'idarchive' => '2', + 'idsite' => '1', + 'date1' => '2018-03-03', + 'date2' => '2018-03-03', + 'period' => '1', + 'name' => 'done.ExamplePlugin', + 'value' => '5', + ), + array ( + 'idarchive' => '2', + 'idsite' => '1', + 'date1' => '2018-03-03', + 'date2' => '2018-03-03', + 'period' => '1', + 'name' => 'ExamplePlugin_example_metric', + 'value' => '-603', + ), + array ( + 'idarchive' => '3', + 'idsite' => '1', + 'date1' => '2018-03-03', + 'date2' => '2018-03-03', + 'period' => '1', + 'name' => 'done.ExamplePlugin', + 'value' => '5', + ), + array ( + 'idarchive' => '3', + 'idsite' => '1', + 'date1' => '2018-03-03', + 'date2' => '2018-03-03', + 'period' => '1', + 'name' => 'ExamplePlugin_example_metric2', + 'value' => '55', + ), + ), + $reportSpecificArchive2, + ], + + // no archive, unloaded plugin + [ + [], + $unloadedPluginArchive, + array ( + array ( + 'idarchive' => '1', + 'idsite' => '1', + 'date1' => '2018-03-03', + 'date2' => '2018-03-03', + 'period' => '1', + 'name' => 'bounce_count', + 'value' => '1', + ), + array ( + 'idarchive' => '1', + 'idsite' => '1', + 'date1' => '2018-03-03', + 'date2' => '2018-03-03', + 'period' => '1', + 'name' => 'done.VisitsSummary', + 'value' => '1', + ), + array ( + 'idarchive' => '1', + 'idsite' => '1', + 'date1' => '2018-03-03', + 'date2' => '2018-03-03', + 'period' => '1', + 'name' => 'max_actions', + 'value' => '1', + ), + array ( + 'idarchive' => '1', + 'idsite' => '1', + 'date1' => '2018-03-03', + 'date2' => '2018-03-03', + 'period' => '1', + 'name' => 'nb_actions', + 'value' => '1', + ), + array ( + 'idarchive' => '1', + 'idsite' => '1', + 'date1' => '2018-03-03', + 'date2' => '2018-03-03', + 'period' => '1', + 'name' => 'nb_uniq_visitors', + 'value' => '1', + ), + array ( + 'idarchive' => '1', + 'idsite' => '1', + 'date1' => '2018-03-03', + 'date2' => '2018-03-03', + 'period' => '1', + 'name' => 'nb_visits', + 'value' => '1', + ), + array ( + 'idarchive' => '2', + 'idsite' => '1', + 'date1' => '2018-03-03', + 'date2' => '2018-03-03', + 'period' => '1', + 'name' => 'done.MyImaginaryPlugin', + 'value' => '1', + ), + ), + false, + ], + ]; + } + public function test_loadExistingArchiveIdFromDb_returnsFalsesIfNoArchiveFound() { $params = new Parameters(new Site(1), Factory::build('day', '2015-03-03'), new Segment('', [1])); @@ -95,7 +852,7 @@ class LoaderTest extends IntegrationTestCase $this->assertNotEmpty($archiveInfo[4]); unset($archiveInfo[4]); - $this->assertEquals(['1', '10', '0', true], $archiveInfo); + $this->assertEquals([['1'], '10', '0', true], $archiveInfo); } public function test_loadExistingArchiveIdFromDb_returnsArchiveIfForACurrentPeriod_AndNewEnough() @@ -110,7 +867,7 @@ class LoaderTest extends IntegrationTestCase $this->assertNotEmpty($archiveInfo[4]); unset($archiveInfo[4]); - $this->assertEquals(['1', '10', '0', true], $archiveInfo); + $this->assertEquals([['1'], '10', '0', true], $archiveInfo); } public function test_loadExistingArchiveIdFromDb_returnsNoArchiveIfForACurrentPeriod_AndNoneAreNewEnough() @@ -331,6 +1088,26 @@ class LoaderTest extends IntegrationTestCase $this->assertFalse($loader->canSkipThisArchive()); } + public function test_forcePluginArchiving_createsPluginSpecificArchive() + { + $_GET['trigger'] = 'archivephp'; + $_GET['pluginOnly'] = '1'; + + $params = new Parameters(new Site(1), Factory::build('day', '2016-02-03'), new Segment('', [1])); + $loader = new Loader($params); + + $tracker = Fixture::getTracker(1, '2016-02-03 00:00:00'); + $tracker->setUrl('http://example.org/abc'); + Fixture::checkResponse($tracker->doTrackPageView('abc')); + + $idArchive = $loader->prepareArchive('Actions')[0]; + $this->assertNotEmpty($idArchive); + + $table = ArchiveTableCreator::getNumericTable(Date::factory('2016-02-03')); + $doneFlag = Db::fetchOne("SELECT `name` FROM `$table` WHERE `name` LIKE 'done%' AND idarchive IN (" . implode($idArchive, ',') . ")"); + $this->assertEquals('done.Actions', $doneFlag); + } + private function insertArchive(Parameters $params, $tsArchived = null, $visits = 10) { $archiveWriter = new ArchiveWriter($params); @@ -349,4 +1126,38 @@ class LoaderTest extends IntegrationTestCase parent::configureFixture($fixture); $fixture->createSuperUser = true; } -}
\ No newline at end of file + + private function insertArchiveData($archiveRows) + { + foreach ($archiveRows as $row) { + if (!empty($row['is_blob_data'])) { + $row['value'] = gzcompress($row['value']); + } + + $d = Date::factory($row['date1']); + $table = !empty($row['is_blob_data']) ? ArchiveTableCreator::getBlobTable($d) : ArchiveTableCreator::getNumericTable($d); + $tsArchived = isset($row['ts_archived']) ? $row['ts_archived'] : Date::now()->getDatetime(); + + Db::query("INSERT INTO `$table` (idarchive, idsite, period, date1, date2, `name`, `value`, ts_archived) VALUES (?, ?, ?, ?, ?, ?, ?, ?)", + [$row['idarchive'], $row['idsite'], $row['period'], $row['date1'], $row['date2'], $row['name'], $row['value'], $tsArchived]); + } + + if (!empty($archiveRows)) { + $idarchives = array_column($archiveRows, 'idarchive'); + $max = max($idarchives); + + $seq = new Sequence(ArchiveTableCreator::getNumericTable(Date::factory($archiveRows[0]['date1']))); + $seq->create($max); + } + } + + private function getArchives() + { + $results = []; + foreach (ArchiveTableCreator::getTablesArchivesInstalled('numeric', true) as $table) { + $queryResults = Db::fetchAll("SELECT idarchive, idsite, date1, date2, period, `name`, `value` FROM `$table`"); + $results = array_merge($results, $queryResults); + } + return $results; + } +} diff --git a/tests/PHPUnit/Integration/ArchiveProcessor/PluginsArchiverTest.php b/tests/PHPUnit/Integration/ArchiveProcessor/PluginsArchiverTest.php index 08a3c1ab2a..8b45eac55a 100644 --- a/tests/PHPUnit/Integration/ArchiveProcessor/PluginsArchiverTest.php +++ b/tests/PHPUnit/Integration/ArchiveProcessor/PluginsArchiverTest.php @@ -8,13 +8,10 @@ namespace Piwik\Tests\Integration\Archive; use Piwik\ArchiveProcessor\PluginsArchiver; -use Piwik\Config; use Piwik\Piwik; use Piwik\Segment; use Piwik\Site; -use Piwik\Db; use Piwik\ArchiveProcessor\Parameters; -use Exception; use Piwik\Plugin\Archiver; use Piwik\Tests\Framework\Fixture; use Piwik\Tests\Framework\TestCase\IntegrationTestCase; @@ -111,5 +108,4 @@ class PluginsArchiverTest extends IntegrationTestCase $this->assertTrue(true); // pass } - } diff --git a/tests/PHPUnit/Integration/ArchiveTest.php b/tests/PHPUnit/Integration/ArchiveTest.php new file mode 100644 index 0000000000..b2a75fc528 --- /dev/null +++ b/tests/PHPUnit/Integration/ArchiveTest.php @@ -0,0 +1,91 @@ +<?php +/** + * Matomo - free/libre analytics platform + * + * @link https://matomo.org + * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later + * + */ + +namespace PHPUnit\Integration; + +use Piwik\Archive; +use Piwik\ArchiveProcessor\Parameters; +use Piwik\ArchiveProcessor\Rules; +use Piwik\Config; +use Piwik\DataAccess\ArchiveWriter; +use Piwik\Period\Factory; +use Piwik\Segment; +use Piwik\Site; +use Piwik\Tests\Framework\Fixture; +use Piwik\Tests\Framework\TestCase\IntegrationTestCase; + +class ArchiveTest extends IntegrationTestCase +{ + public function setUp(): void + { + parent::setUp(); + + Fixture::createWebsite('2014-05-06'); + } + + public function test_pluginSpecificArchiveUsed_EvenIfAllArchiveExists_IfThereAreNoDataInAllArchive() + { + $idSite = 1; + + // insert all plugin archive + $params = new Parameters(new Site($idSite), Factory::build('day', '2014-05-07'), new Segment('', [$idSite])); + $archiveWriter = new ArchiveWriter($params); + $archiveWriter->initNewArchive(); + $archiveWriter->insertRecord('ExamplePlugin_archive1metric', 1); + $archiveWriter->insertRecord('ExamplePlugin_archive2metric', 5); + $archiveWriter->finalizeArchive(); + + // insert single plugin archive + $_GET['pluginOnly'] = 1; + $_GET['trigger'] = 'archivephp'; + + $params = new Parameters(new Site($idSite), Factory::build('day', '2014-05-07'), new Segment('', [$idSite])); + $params->setRequestedPlugin('ExamplePlugin'); + $params->onlyArchiveRequestedPlugin(); + $archiveWriter = new ArchiveWriter($params); + $archiveWriter->initNewArchive(); + $archiveWriter->insertRecord('ExamplePlugin_archive2metric', 2); + $archiveWriter->insertRecord('ExamplePlugin_archive3metric', 3); + $archiveWriter->finalizeArchive(); + + // insert single plugin archive + $params = new Parameters(new Site($idSite), Factory::build('day', '2014-05-07'), new Segment('', [$idSite])); + $params->setRequestedPlugin('ExamplePlugin'); + $params->onlyArchiveRequestedPlugin(); + $archiveWriter = new ArchiveWriter($params); + $archiveWriter->initNewArchive(); + $archiveWriter->insertRecord('ExamplePlugin_archive3metric', 7); + $archiveWriter->finalizeArchive(); + + $archive = Archive::build($idSite, 'day', '2014-05-07'); + $metrics = $archive->getNumeric(['ExamplePlugin_archive1metric', 'ExamplePlugin_archive2metric', 'ExamplePlugin_archive3metric']); + + $expected = [ + 'ExamplePlugin_archive1metric' => 0, + 'ExamplePlugin_archive2metric' => 0, + 'ExamplePlugin_archive3metric' => 7, + ]; + + $this->assertEquals($expected, $metrics); + } + + public function test_pluginSpecificArchiveUsed_EvenIfAllArchiveExists_IfThereAreNoDataInAllArchive_WithBrowserArchivingDisabled() + { + self::$fixture->getTestEnvironment()->overrideConfig('General', 'enable_browser_archiving_triggering', 0); + self::$fixture->getTestEnvironment()->overrideConfig('General', 'archiving_range_force_on_browser_request', 0); + self::$fixture->getTestEnvironment()->save(); + + Config::getInstance()->General['enable_browser_archiving_triggering'] = 0; + Config::getInstance()->General['archiving_range_force_on_browser_request'] = 0; + + $this->assertTrue(Rules::isArchivingDisabledFor([1], new Segment('', [1]), 'day')); + + $this->test_pluginSpecificArchiveUsed_EvenIfAllArchiveExists_IfThereAreNoDataInAllArchive(); + } +}
\ No newline at end of file diff --git a/tests/PHPUnit/Integration/ArchiveWithNoVisitsTest.php b/tests/PHPUnit/Integration/ArchiveWithNoVisitsTest.php index 799036f1b3..44d1f3ea80 100644 --- a/tests/PHPUnit/Integration/ArchiveWithNoVisitsTest.php +++ b/tests/PHPUnit/Integration/ArchiveWithNoVisitsTest.php @@ -59,7 +59,6 @@ class ArchiveWithNoVisitsTest extends IntegrationTestCase $this->assertEmpty(ArchiveWithNoVisitsTest_MockArchiver::$methodsCalled); } - // TODO: changed this and another test, will it be an issue? now, CronArchive.getIdSitesNotUsingTracker has to add the site for it to archive no matter what. public function test_getIdSitesToArchiveWhenNoVisits_DoesNotTriggerArchiving_IfSiteHasNoVisits() { // add our mock archiver instance @@ -78,7 +77,16 @@ class ArchiveWithNoVisitsTest extends IntegrationTestCase // initiate archiving and make sure both aggregate methods are called correctly VisitsSummaryAPI::getInstance()->get($idSite = 1, 'week', '2012-01-10'); - $expectedMethodCalls = array(); + $expectedMethodCalls = array( + 'aggregateDayReport', + 'aggregateDayReport', + 'aggregateDayReport', + 'aggregateDayReport', + 'aggregateDayReport', + 'aggregateDayReport', + 'aggregateDayReport', + 'aggregateMultipleReports', + ); $this->assertEquals($expectedMethodCalls, ArchiveWithNoVisitsTest_MockArchiver::$methodsCalled); } diff --git a/tests/PHPUnit/Integration/CronArchive/QueueConsumerTest.php b/tests/PHPUnit/Integration/CronArchive/QueueConsumerTest.php new file mode 100644 index 0000000000..1d14e457b9 --- /dev/null +++ b/tests/PHPUnit/Integration/CronArchive/QueueConsumerTest.php @@ -0,0 +1,549 @@ +<?php +/** + * Matomo - free/libre analytics platform + * + * @link https://matomo.org + * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later + * + */ + +namespace Piwik\Tests\Integration\CronArchive; + +use Piwik\CliMulti\RequestParser; +use Piwik\Common; +use Piwik\Container\StaticContainer; +use Piwik\CronArchive; +use Piwik\CronArchive\FixedSiteIds; +use Piwik\CronArchive\QueueConsumer; +use Piwik\CronArchive\SegmentArchiving; +use Piwik\DataAccess\ArchiveTableCreator; +use Piwik\DataAccess\ArchiveWriter; +use Piwik\DataAccess\Model; +use Piwik\Date; +use Piwik\Db; +use Piwik\Piwik; +use Piwik\Plugins\SegmentEditor\API; +use Piwik\Segment; +use Piwik\Tests\Framework\Fixture; +use Piwik\Tests\Framework\TestCase\IntegrationTestCase; +use Psr\Log\LoggerInterface; + +class QueueConsumerTest extends IntegrationTestCase +{ + public function test_invalidateConsumeOrder() + { + Fixture::createWebsite('2015-02-03'); + Fixture::createWebsite('2020-04-06'); + Fixture::createWebsite('2010-04-06'); + + API::getInstance()->add('testegment', 'browserCode==IE', false, true); + + // force archiving so we don't skip those without visits + Piwik::addAction('Archiving.getIdSitesToArchiveWhenNoVisits', function (&$idSites) { + $idSites[] = 1; + $idSites[] = 2; + }); + + $cronArchive = new CronArchive(); + + $archiveFilter = $this->makeTestArchiveFilter(); + + $queueConsumer = new QueueConsumer( + StaticContainer::get(LoggerInterface::class), + new FixedSiteIds([1,2,3]), + 3, + 24, + new Model(), + new SegmentArchiving('beginning_of_time'), + $cronArchive, + new RequestParser(true), + $archiveFilter + ); + + $segmentHash = (new Segment('browserCode==IE', [1]))->getHash(); + + $invalidations = [ + ['idarchive' => 1, 'name' => 'done', 'idsite' => 1, 'date1' => '2018-03-04', 'date2' => '2018-03-04', 'period' => 1, 'report' => null], + ['idarchive' => 1, 'name' => 'done', 'idsite' => 1, 'date1' => '2018-03-07', 'date2' => '2018-03-07', 'period' => 1, 'report' => null], + ['idarchive' => 1, 'name' => 'done', 'idsite' => 1, 'date1' => '2018-03-08', 'date2' => '2018-03-08', 'period' => 1, 'report' => null], + ['idarchive' => 1, 'name' => 'done', 'idsite' => 1, 'date1' => '2018-03-06', 'date2' => '2018-03-06', 'period' => 1, 'report' => null], + ['idarchive' => 1, 'name' => 'done', 'idsite' => 1, 'date1' => '2018-03-01', 'date2' => '2018-03-31', 'period' => 3, 'report' => null], + ['idarchive' => 1, 'name' => 'done', 'idsite' => 1, 'date1' => '2018-03-04', 'date2' => '2018-03-11', 'period' => 2, 'report' => null], + + ['idarchive' => 1, 'name' => 'done.Actions', 'idsite' => 1, 'date1' => '2018-03-06', 'date2' => '2018-03-06', 'period' => 1, 'report' => 'testReport'], + ['idarchive' => 1, 'name' => 'done.Actions', 'idsite' => 1, 'date1' => '2018-03-01', 'date2' => '2018-03-31', 'period' => 3, 'report' => 'testReport'], + ['idarchive' => 1, 'name' => 'done.Actions', 'idsite' => 1, 'date1' => '2018-03-04', 'date2' => '2018-03-11', 'period' => 2, 'report' => 'testReport'], + + // some or all subperiods before site was created + ['idarchive' => 1, 'name' => 'done', 'idsite' => 2, 'date1' => '2020-04-04', 'date2' => '2020-04-04', 'period' => 1, 'report' => 'testReport'], + ['idarchive' => 1, 'name' => 'done', 'idsite' => 2, 'date1' => '2020-03-30', 'date2' => '2020-04-05', 'period' => 2, 'report' => 'testReport'], + + // segments + ['idarchive' => 1, 'name' => 'done' . $segmentHash, 'idsite' => 1, 'date1' => '2018-03-04', 'date2' => '2018-03-04', 'period' => 1, 'report' => null], + ['idarchive' => 1, 'name' => 'done' . $segmentHash, 'idsite' => 1, 'date1' => '2018-03-07', 'date2' => '2018-03-07', 'period' => 1, 'report' => null], + ['idarchive' => 1, 'name' => 'done' . $segmentHash, 'idsite' => 1, 'date1' => '2018-03-08', 'date2' => '2018-03-08', 'period' => 1, 'report' => null], + ['idarchive' => 1, 'name' => 'done' . $segmentHash, 'idsite' => 1, 'date1' => '2018-03-06', 'date2' => '2018-03-06', 'period' => 1, 'report' => null], + ['idarchive' => 1, 'name' => 'done' . $segmentHash, 'idsite' => 1, 'date1' => '2018-03-01', 'date2' => '2018-03-31', 'period' => 3, 'report' => null], + ['idarchive' => 1, 'name' => 'done' . $segmentHash, 'idsite' => 1, 'date1' => '2018-03-04', 'date2' => '2018-03-11', 'period' => 2, 'report' => null], + + // invalid plugin + ['idarchive' => 1, 'name' => 'done.MyPlugin', 'idsite' => 1, 'date1' => '2018-03-04', 'date2' => '2018-03-11', 'period' => 2, 'report' => 'testReport'], + + // duplicates + ['idarchive' => 1, 'name' => 'done', 'idsite' => 1, 'date1' => '2018-03-06', 'date2' => '2018-03-06', 'period' => 1, 'report' => null], + ['idarchive' => 1, 'name' => 'done', 'idsite' => 1, 'date1' => '2018-03-01', 'date2' => '2018-03-31', 'period' => 3, 'report' => null], + ]; + + shuffle($invalidations); + + $this->insertInvalidations($invalidations); + + $iteratedInvalidations = []; + while (true) { + $next = $queueConsumer->getNextArchivesToProcess(); + if ($next === null) { + break; + } + + foreach ($next as &$item) { + Db::query("UPDATE " . Common::prefixTable('archive_invalidations') . " SET status = 1 WHERE idinvalidation = ?", [$item['idinvalidation']]); + + unset($item['periodObj']); + unset($item['idinvalidation']); + } + + $iteratedInvalidations[] = $next; + } + + $expectedInvalidationsFound = [ + array ( + array ( + 'idarchive' => '1', + 'idsite' => '1', + 'date1' => '2018-03-08', + 'date2' => '2018-03-08', + 'period' => '1', + 'name' => 'done', + 'report' => NULL, + 'plugin' => NULL, + 'segment' => '', + ), + array ( + 'idarchive' => '1', + 'idsite' => '1', + 'date1' => '2018-03-07', + 'date2' => '2018-03-07', + 'period' => '1', + 'name' => 'done', + 'report' => NULL, + 'plugin' => NULL, + 'segment' => '', + ), + array ( + 'idarchive' => '1', + 'idsite' => '1', + 'date1' => '2018-03-06', + 'date2' => '2018-03-06', + 'period' => '1', + 'name' => 'done', + 'report' => NULL, + 'plugin' => NULL, + 'segment' => '', + ), + ), + array ( + array ( + 'idarchive' => '1', + 'idsite' => '1', + 'date1' => '2018-03-08', + 'date2' => '2018-03-08', + 'period' => '1', + 'name' => 'done5f4f9bafeda3443c3c2d4b2ef4dffadc', + 'report' => NULL, + 'plugin' => NULL, + 'segment' => 'browserCode==IE', + ), + array ( + 'idarchive' => '1', + 'idsite' => '1', + 'date1' => '2018-03-07', + 'date2' => '2018-03-07', + 'period' => '1', + 'name' => 'done5f4f9bafeda3443c3c2d4b2ef4dffadc', + 'report' => NULL, + 'plugin' => NULL, + 'segment' => 'browserCode==IE', + ), + array ( + 'idarchive' => '1', + 'idsite' => '1', + 'date1' => '2018-03-06', + 'date2' => '2018-03-06', + 'period' => '1', + 'name' => 'done.Actions', + 'report' => 'testReport', + 'plugin' => 'Actions', + 'segment' => '', + ), + ), + array ( + array ( // duplicate, processed but if in progress or recent should be skipped + 'idarchive' => '1', + 'idsite' => '1', + 'date1' => '2018-03-06', + 'date2' => '2018-03-06', + 'period' => '1', + 'name' => 'done', + 'report' => NULL, + 'plugin' => NULL, + 'segment' => '', + ), + array ( + 'idarchive' => '1', + 'idsite' => '1', + 'date1' => '2018-03-04', + 'date2' => '2018-03-04', + 'period' => '1', + 'name' => 'done', + 'report' => NULL, + 'plugin' => NULL, + 'segment' => '', + ), + ), + array ( + array ( + 'idarchive' => '1', + 'idsite' => '1', + 'date1' => '2018-03-06', + 'date2' => '2018-03-06', + 'period' => '1', + 'name' => 'done5f4f9bafeda3443c3c2d4b2ef4dffadc', + 'report' => NULL, + 'plugin' => NULL, + 'segment' => 'browserCode==IE', + ), + array ( + 'idarchive' => '1', + 'idsite' => '1', + 'date1' => '2018-03-04', + 'date2' => '2018-03-04', + 'period' => '1', + 'name' => 'done5f4f9bafeda3443c3c2d4b2ef4dffadc', + 'report' => NULL, + 'plugin' => NULL, + 'segment' => 'browserCode==IE', + ), + ), + array ( + array ( + 'idarchive' => '1', + 'idsite' => '1', + 'date1' => '2018-03-04', + 'date2' => '2018-03-11', + 'period' => '2', + 'name' => 'done', + 'report' => NULL, + 'plugin' => NULL, + 'segment' => '', + ), + ), + array ( + array ( + 'idarchive' => '1', + 'idsite' => '1', + 'date1' => '2018-03-04', + 'date2' => '2018-03-11', + 'period' => '2', + 'name' => 'done.Actions', + 'report' => 'testReport', + 'plugin' => 'Actions', + 'segment' => '', + ), + ), + array ( + array ( + 'idarchive' => '1', + 'idsite' => '1', + 'date1' => '2018-03-04', + 'date2' => '2018-03-11', + 'period' => '2', + 'name' => 'done5f4f9bafeda3443c3c2d4b2ef4dffadc', + 'report' => NULL, + 'plugin' => NULL, + 'segment' => 'browserCode==IE', + ), + ), + array ( + array ( + 'idarchive' => '1', + 'idsite' => '1', + 'date1' => '2018-03-01', + 'date2' => '2018-03-31', + 'period' => '3', + 'name' => 'done', + 'report' => NULL, + 'plugin' => NULL, + 'segment' => '', + ), + ), + array ( + array ( + 'idarchive' => '1', + 'idsite' => '1', + 'date1' => '2018-03-01', + 'date2' => '2018-03-31', + 'period' => '3', + 'name' => 'done.Actions', + 'report' => 'testReport', + 'plugin' => 'Actions', + 'segment' => '', + ), + ), + array ( + array ( + 'idarchive' => '1', + 'idsite' => '1', + 'date1' => '2018-03-01', + 'date2' => '2018-03-31', + 'period' => '3', + 'name' => 'done5f4f9bafeda3443c3c2d4b2ef4dffadc', + 'report' => NULL, + 'plugin' => NULL, + 'segment' => 'browserCode==IE', + ), + ), + array ( // end of idsite=1 + ), + array ( + array ( + 'idarchive' => '1', + 'idsite' => '2', + 'date1' => '2020-03-30', + 'date2' => '2020-04-05', + 'period' => '2', + 'name' => 'done', + 'report' => 'testReport', + 'plugin' => NULL, + 'segment' => '', + ), + ), + array ( // end of idsite=2 + ), + array ( // end of idsite=3 + ), + ]; + + try { + $this->assertEquals($expectedInvalidationsFound, $iteratedInvalidations); + } catch (\Exception $ex) { + print "\nInvalidations inserted:\n" . var_export($invalidations, true) . "\n"; + throw $ex; + } + } + + private function makeTestArchiveFilter($restrictToDateRange = null, $restrictToPeriods = null, $segmentsToForce = null, $disableSegmentsArchiving = false) + { + $archiveFilter = new CronArchive\ArchiveFilter(); + if ($restrictToDateRange) { + $archiveFilter->setRestrictToDateRange(); + } + $archiveFilter->setDisableSegmentsArchiving($disableSegmentsArchiving); + if ($restrictToPeriods) { + $archiveFilter->setRestrictToPeriods($restrictToPeriods); + } + if ($segmentsToForce) { + $archiveFilter->setSegmentsToForceFromSegmentIds($segmentsToForce); + } + return $archiveFilter; + } + + private function insertInvalidations(array $invalidations) + { + $table = Common::prefixTable('archive_invalidations'); + foreach ($invalidations as $inv) { + $bind = [ + $inv['idarchive'], + $inv['name'], + $inv['idsite'], + $inv['date1'], + $inv['date2'], + $inv['period'], + $inv['report'], + ]; + Db::query("INSERT INTO `$table` (idarchive, name, idsite, date1, date2, period, ts_invalidated, report, status) + VALUES (?, ?, ?, ?, ?, ?, NOW(), ?, 0)", $bind); + } + } + + public function test_canSkipArchiveBecauseNoPoint_returnsTrueIfDateRangeHasNoVisits() + { + Fixture::createWebsite('2010-04-06'); + + Date::$now = strtotime('2020-04-05'); + + $cronArchive = new CronArchive(); + + $archiveFilter = $this->makeTestArchiveFilter(); + + $queueConsumer = new QueueConsumer( + StaticContainer::get(LoggerInterface::class), + new FixedSiteIds([1]), + 3, + 24, + new Model(), + new SegmentArchiving('beginning_of_time'), + $cronArchive, + new RequestParser(true), + $archiveFilter + ); + + $invalidation = [ + 'idsite' => 1, + 'period' => 1, + 'date1' => '2020-04-05', + 'date2' => '2020-04-05', + 'name' => 'done', + 'segment' => '', + ]; + + $result = $queueConsumer->canSkipArchiveBecauseNoPoint($invalidation); + $this->assertTrue($result); + } + + public function test_canSkipArchiveBecauseNoPoint_returnsFalseIfDateRangeHasVisits_AndPeriodDoesNotIncludeToday() + { + $idSite = Fixture::createWebsite('2015-02-03'); + + Date::$now = strtotime('2020-04-05'); + + $t = Fixture::getTracker($idSite, '2020-03-05 10:34:00'); + $t->setUrl('http://whatever.com'); + Fixture::checkResponse($t->doTrackPageView('test title')); + + $cronArchive = new CronArchive(); + + $archiveFilter = $this->makeTestArchiveFilter(); + + $queueConsumer = new QueueConsumer( + StaticContainer::get(LoggerInterface::class), + new FixedSiteIds([1]), + 3, + 24, + new Model(), + new SegmentArchiving('beginning_of_time'), + $cronArchive, + new RequestParser(true), + $archiveFilter + ); + + $invalidation = [ + 'idsite' => 1, + 'period' => 1, + 'date1' => '2020-03-05', + 'date2' => '2020-03-05', + 'name' => 'done', + 'segment' => '', + ]; + + $result = $queueConsumer->canSkipArchiveBecauseNoPoint($invalidation); + $this->assertFalse($result); + } + + public function test_usableArchiveExists_returnsTrueIfDateRangeHasVisits_AndPeriodIncludesToday_AndExistingArchiveIsRecent() + { + $idSite = Fixture::createWebsite('2015-02-03'); + + Date::$now = strtotime('2020-04-05'); + + $t = Fixture::getTracker($idSite, '2020-04-05 10:34:00'); + $t->setUrl('http://whatever.com'); + Fixture::checkResponse($t->doTrackPageView('test title')); + + $cronArchive = new CronArchive(); + + $archiveFilter = $this->makeTestArchiveFilter(); + + $queueConsumer = new QueueConsumer( + StaticContainer::get(LoggerInterface::class), + new FixedSiteIds([1]), + 3, + 24, + new Model(), + new SegmentArchiving('beginning_of_time'), + $cronArchive, + new RequestParser(true), + $archiveFilter + ); + + $invalidation = [ + 'idsite' => 1, + 'period' => 2, + 'date1' => '2020-03-30', + 'date2' => '2020-04-05', + 'name' => 'done', + 'segment' => '', + ]; + + $tsArchived = Date::factory('now')->subSeconds(100)->getDatetime(); + + $archiveTable = ArchiveTableCreator::getNumericTable(Date::factory('2020-03-30')); + Db::query("INSERT INTO $archiveTable (idarchive, idsite, period, date1, date2, name, value, ts_archived) VALUES (?, ?, ?, ?, ?, ?, ?, ?)", [ + 1, 1,2, '2020-03-30', '2020-04-05', 'done', ArchiveWriter::DONE_INVALIDATED, $tsArchived + ]); + + $result = $queueConsumer->usableArchiveExists($invalidation); + $this->assertTrue($result); + } + + public function test_canSkipArchiveBecauseNoPoint_returnsFalseIfDateRangeHasVisits_AndPeriodIncludesToday_AndOnlyExistingArchiveIsRecentButPartial() + { + $idSite = Fixture::createWebsite('2015-02-03'); + + Date::$now = strtotime('2020-04-05'); + + $t = Fixture::getTracker($idSite, '2020-04-05 10:34:00'); + $t->setUrl('http://whatever.com'); + Fixture::checkResponse($t->doTrackPageView('test title')); + + $cronArchive = new CronArchive(); + + $archiveFilter = $this->makeTestArchiveFilter(); + + $queueConsumer = new QueueConsumer( + StaticContainer::get(LoggerInterface::class), + new FixedSiteIds([1]), + 3, + 24, + new Model(), + new SegmentArchiving('beginning_of_time'), + $cronArchive, + new RequestParser(true), + $archiveFilter + ); + + $invalidation = [ + 'idsite' => 1, + 'period' => 2, + 'date1' => '2020-03-30', + 'date2' => '2020-04-05', + 'name' => 'done', + 'segment' => '', + ]; + + $tsArchived = Date::factory('now')->subSeconds(100)->getDatetime(); + + $archiveTable = ArchiveTableCreator::getNumericTable(Date::factory('2020-03-30')); + Db::query("INSERT INTO $archiveTable (idarchive, idsite, period, date1, date2, name, value, ts_archived) VALUES (?, ?, ?, ?, ?, ?, ?, ?)", [ + 1, 1,2, '2020-03-30', '2020-04-05', 'done', ArchiveWriter::DONE_PARTIAL, $tsArchived + ]); + + $result = $queueConsumer->canSkipArchiveBecauseNoPoint($invalidation); + $this->assertFalse($result); + } + + protected static function configureFixture($fixture) + { + parent::configureFixture($fixture); + $fixture->createSuperUser = true; + } +}
\ No newline at end of file diff --git a/tests/PHPUnit/Integration/CronArchiveTest.php b/tests/PHPUnit/Integration/CronArchiveTest.php index 4a18ddb827..0fc0992f4f 100644 --- a/tests/PHPUnit/Integration/CronArchiveTest.php +++ b/tests/PHPUnit/Integration/CronArchiveTest.php @@ -8,18 +8,24 @@ namespace Piwik\Tests\Integration; +use Piwik\ArchiveProcessor\Parameters; use Piwik\Container\StaticContainer; use Piwik\CronArchive; use Piwik\DataAccess\ArchiveTableCreator; +use Piwik\DataAccess\ArchiveWriter; use Piwik\Date; use Piwik\Db; +use Piwik\Period\Factory; use Piwik\Plugins\CoreAdminHome\tests\Framework\Mock\API; use Piwik\Plugins\SegmentEditor\Model; +use Piwik\Segment; +use Piwik\Site; use Piwik\Tests\Framework\Fixture; use Piwik\Tests\Framework\Mock\FakeLogger; use Piwik\Tests\Framework\TestCase\IntegrationTestCase; use Piwik\Plugins\SegmentEditor\API as SegmentAPI; use Piwik\Version; +use Psr\Log\NullLogger; /** * @group Archiver @@ -27,6 +33,111 @@ use Piwik\Version; */ class CronArchiveTest extends IntegrationTestCase { + public function test_isThereExistingValidPeriod_returnsTrueIfPeriodHasToday_AndExistingArchiveIsNewEnough() + { + Fixture::createWebsite('2019-04-04 03:45:45'); + + Date::$now = strtotime('2020-04-05'); + + $archiver = new CronArchive(); + + $params = new Parameters(new Site(1), Factory::build('week', '2020-04-05'), new Segment('', [1])); + + $tsArchived = Date::now()->subSeconds(100)->getDatetime(); + + $archiveTable = ArchiveTableCreator::getNumericTable(Date::factory('2020-03-30')); + Db::query("INSERT INTO $archiveTable (idarchive, idsite, period, date1, date2, name, value, ts_archived) VALUES (?, ?, ?, ?, ?, ?, ?, ?)", [ + 1, 1,2, '2020-03-30', '2020-04-05', 'done', ArchiveWriter::DONE_OK, $tsArchived + ]); + + $actual =$archiver->isThereExistingValidPeriod($params, $isYesterday = false); + $this->assertTrue($actual); + } + + public function test_isThereExistingValidPeriod_returnsTrueIfPeriodHasToday_AndExistingArchiveIsNewEnoughAndInvalidated() + { + Fixture::createWebsite('2019-04-04 03:45:45'); + + Date::$now = strtotime('2020-04-05'); + + $archiver = new CronArchive(); + + $params = new Parameters(new Site(1), Factory::build('week', '2020-04-05'), new Segment('', [1])); + + $tsArchived = Date::now()->subSeconds(100)->getDatetime(); + + $archiveTable = ArchiveTableCreator::getNumericTable(Date::factory('2020-03-30')); + Db::query("INSERT INTO $archiveTable (idarchive, idsite, period, date1, date2, name, value, ts_archived) VALUES (?, ?, ?, ?, ?, ?, ?, ?)", [ + 1, 1,2, '2020-03-30', '2020-04-05', 'done', ArchiveWriter::DONE_INVALIDATED, $tsArchived + ]); + + $actual =$archiver->isThereExistingValidPeriod($params, $isYesterday = false); + $this->assertTrue($actual); + } + + public function test_isThereExistingValidPeriod_returnsTrueIfPeriodDoesNotHaveToday_AndExistingArchiveIsOk() + { + Fixture::createWebsite('2019-04-04 03:45:45'); + + Date::$now = strtotime('2020-04-05'); + + $archiver = new CronArchive(); + + $params = new Parameters(new Site(1), Factory::build('day', '2020-03-05'), new Segment('', [1])); + + $tsArchived = Date::now()->subDay(1)->getDatetime(); + + $archiveTable = ArchiveTableCreator::getNumericTable(Date::factory('2020-03-05')); + Db::query("INSERT INTO $archiveTable (idarchive, idsite, period, date1, date2, name, value, ts_archived) VALUES (?, ?, ?, ?, ?, ?, ?, ?)", [ + 1, 1, 1, '2020-03-05', '2020-03-05', 'done', ArchiveWriter::DONE_OK, $tsArchived + ]); + + $actual =$archiver->isThereExistingValidPeriod($params, $isYesterday = false); + $this->assertTrue($actual); + } + + public function test_isThereExistingValidPeriod_returnsFalseIfDayHasChangedAndDateIsYesterday() + { + Fixture::createWebsite('2019-04-04 03:45:45'); + + Date::$now = strtotime('2020-04-05'); + + $archiver = new CronArchive(); + + $params = new Parameters(new Site(1), Factory::build('day', '2020-04-04'), new Segment('', [1])); + + $tsArchived = Date::now()->subDay(1)->getDatetime(); + + $archiveTable = ArchiveTableCreator::getNumericTable(Date::factory('2020-04-04')); + Db::query("INSERT INTO $archiveTable (idarchive, idsite, period, date1, date2, name, value, ts_archived) VALUES (?, ?, ?, ?, ?, ?, ?, ?)", [ + 1, 1, 1, '2020-04-04', '2020-04-04', 'done', ArchiveWriter::DONE_OK, $tsArchived + ]); + + $actual =$archiver->isThereExistingValidPeriod($params, $isYesterday = true); + $this->assertFalse($actual); + } + + public function test_isThereExistingValidPeriod_returnsTrueIfDayHasNotChangedAndDateIsYesterday() + { + Fixture::createWebsite('2019-04-04 03:45:45'); + + Date::$now = strtotime('2020-04-05 06:23:40'); + + $archiver = new CronArchive(); + + $params = new Parameters(new Site(1), Factory::build('day', '2020-04-04'), new Segment('', [1])); + + $tsArchived = Date::now()->subSeconds(1500)->getDatetime(); + + $archiveTable = ArchiveTableCreator::getNumericTable(Date::factory('2020-04-04')); + Db::query("INSERT INTO $archiveTable (idarchive, idsite, period, date1, date2, name, value, ts_archived) VALUES (?, ?, ?, ?, ?, ?, ?, ?)", [ + 1, 1, 1, '2020-04-04', '2020-04-04', 'done', ArchiveWriter::DONE_OK, $tsArchived + ]); + + $actual = $archiver->isThereExistingValidPeriod($params, $isYesterday = true); + $this->assertTrue($actual); + } + public function test_getColumnNamesFromTable() { Fixture::createWebsite('2014-12-12 00:01:02'); @@ -173,108 +284,135 @@ Checking for queued invalidations... Will invalidate archived reports for 2019-12-11 for following websites ids: 1 Will invalidate archived reports for 2019-12-10 for following websites ids: 1 Will invalidate archived reports for 2019-12-02 for following websites ids: 1 - Today archive can be skipped due to no visits, skipping invalidation... - Yesterday archive can be skipped due to no visits, skipping invalidation... + Today archive can be skipped due to no visits for idSite = 1, skipping invalidation... + Yesterday archive can be skipped due to no visits for idSite = 1, skipping invalidation... Segment "actions>=2" was created or changed recently and will therefore archive today (for site ID = 1) Segment "actions>=4" was created or changed recently and will therefore archive today (for site ID = 1) Done invalidating Start processing archives for site 1. -Found invalidated archive we can skip (no visits or latest archive is not invalidated). [idSite = 1, dates = 2020-01-01 - 2020-01-01, segment = actions>=2] -Skipping invalidated archive : segment 'actions>=4' is not in --force-idsegments -Found invalidated archive we can skip (no visits or latest archive is not invalidated). [idSite = 1, dates = 2020-01-01 - 2020-01-31, segment = actions>=2] -Skipping invalidated archive : segment 'actions>=4' is not in --force-idsegments -Found invalidated archive we can skip (no visits or latest archive is not invalidated). [idSite = 1, dates = 2020-01-01 - 2020-12-31, segment = actions>=2] -Skipping invalidated archive : segment 'actions>=4' is not in --force-idsegments -Found invalidated archive we can skip (no visits or latest archive is not invalidated). [idSite = 1, dates = 2019-12-31 - 2019-12-31, segment = actions>=2] -Skipping invalidated archive : segment 'actions>=4' is not in --force-idsegments -Found invalidated archive we can skip (no visits or latest archive is not invalidated). [idSite = 1, dates = 2019-12-30 - 2019-12-30, segment = actions>=2] -Skipping invalidated archive : segment 'actions>=4' is not in --force-idsegments -Found invalidated archive we can skip (no visits or latest archive is not invalidated). [idSite = 1, dates = 2019-12-30 - 2020-01-05, segment = actions>=2] -Skipping invalidated archive : segment 'actions>=4' is not in --force-idsegments -Found invalidated archive we can skip (no visits or latest archive is not invalidated). [idSite = 1, dates = 2019-12-23 - 2019-12-23, segment = actions>=2] -Skipping invalidated archive : segment 'actions>=4' is not in --force-idsegments -Found invalidated archive we can skip (no visits or latest archive is not invalidated). [idSite = 1, dates = 2019-12-23 - 2019-12-29, segment = actions>=2] -Skipping invalidated archive : segment 'actions>=4' is not in --force-idsegments -Found invalidated archive we can skip (no visits or latest archive is not invalidated). [idSite = 1, dates = 2019-12-16 - 2019-12-16, segment = actions>=2] -Skipping invalidated archive : segment 'actions>=4' is not in --force-idsegments -Found invalidated archive we can skip (no visits or latest archive is not invalidated). [idSite = 1, dates = 2019-12-16 - 2019-12-22, segment = actions>=2] -Skipping invalidated archive : segment 'actions>=4' is not in --force-idsegments -Skipping invalidated archive : segment '' is not in --force-idsegments -Skipping invalidated archive : segment '' is not in --force-idsegments -Skipping invalidated archive : segment '' is not in --force-idsegments -Found invalidated archive we can skip (no visits or latest archive is not invalidated). [idSite = 1, dates = 2019-12-09 - 2019-12-09, segment = actions>=2] -Skipping invalidated archive : segment 'actions>=4' is not in --force-idsegments -Skipping invalidated archive : segment '' is not in --force-idsegments -Skipping invalidated archive : segment '' is not in --force-idsegments -Skipping invalidated archive : segment '' is not in --force-idsegments -Skipping invalidated archive : segment 'actions>=4' is not in --force-idsegments -Found archive with different period than others in concurrent batch, skipping until next batch: 1 -Found archive with different period than others in concurrent batch, skipping until next batch: 1 -Found archive with different period than others in concurrent batch, skipping until next batch: 1 -Found archive with different done flag type (segment vs. no segment) in concurrent batch, skipping until next batch: done -Skipping invalidated archive : segment 'actions>=4' is not in --force-idsegments -Found archive with different period than others in concurrent batch, skipping until next batch: 3 -Found archive with different period than others in concurrent batch, skipping until next batch: 3 -Found archive with different period than others in concurrent batch, skipping until next batch: 3 -Found archive with different period than others in concurrent batch, skipping until next batch: 3 -Found archive with different period than others in concurrent batch, skipping until next batch: 3 -Found archive with different period than others in concurrent batch, skipping until next batch: 3 -Found archive with different period than others in concurrent batch, skipping until next batch: 4 -Found archive with different period than others in concurrent batch, skipping until next batch: 4 -Found archive with different period than others in concurrent batch, skipping until next batch: 4 -Found archive with different period than others in concurrent batch, skipping until next batch: 4 -Found archive with different period than others in concurrent batch, skipping until next batch: 4 -Found archive with different period than others in concurrent batch, skipping until next batch: 4 +Found invalidated archive we can skip (no visits): [idinvalidation = 43, idsite = 1, period = day(2020-02-03 - 2020-02-03), name = donee0512c03f7c20af6ef96a8d792c6bb9f] +Skipping invalidated archive 73, segment 'actions>=4' is not in --force-idsegments: [idinvalidation = 73, idsite = 1, period = day(2020-02-03 - 2020-02-03), name = done49a9440bd6dba4b8850035e09d043c67] +Found invalidated archive we can skip (no visits): [idinvalidation = 44, idsite = 1, period = week(2020-02-03 - 2020-02-09), name = donee0512c03f7c20af6ef96a8d792c6bb9f] +Skipping invalidated archive 74, segment 'actions>=4' is not in --force-idsegments: [idinvalidation = 74, idsite = 1, period = week(2020-02-03 - 2020-02-09), name = done49a9440bd6dba4b8850035e09d043c67] +Found invalidated archive we can skip (no visits): [idinvalidation = 42, idsite = 1, period = day(2020-02-02 - 2020-02-02), name = donee0512c03f7c20af6ef96a8d792c6bb9f] +Skipping invalidated archive 72, segment 'actions>=4' is not in --force-idsegments: [idinvalidation = 72, idsite = 1, period = day(2020-02-02 - 2020-02-02), name = done49a9440bd6dba4b8850035e09d043c67] +Found invalidated archive we can skip (no visits): [idinvalidation = 40, idsite = 1, period = day(2020-02-01 - 2020-02-01), name = donee0512c03f7c20af6ef96a8d792c6bb9f] +Skipping invalidated archive 70, segment 'actions>=4' is not in --force-idsegments: [idinvalidation = 70, idsite = 1, period = day(2020-02-01 - 2020-02-01), name = done49a9440bd6dba4b8850035e09d043c67] +Found invalidated archive we can skip (no visits): [idinvalidation = 41, idsite = 1, period = month(2020-02-01 - 2020-02-29), name = donee0512c03f7c20af6ef96a8d792c6bb9f] +Skipping invalidated archive 71, segment 'actions>=4' is not in --force-idsegments: [idinvalidation = 71, idsite = 1, period = month(2020-02-01 - 2020-02-29), name = done49a9440bd6dba4b8850035e09d043c67] +Found invalidated archive we can skip (no visits): [idinvalidation = 36, idsite = 1, period = week(2020-01-27 - 2020-02-02), name = donee0512c03f7c20af6ef96a8d792c6bb9f] +Skipping invalidated archive 66, segment 'actions>=4' is not in --force-idsegments: [idinvalidation = 66, idsite = 1, period = week(2020-01-27 - 2020-02-02), name = done49a9440bd6dba4b8850035e09d043c67] +Found invalidated archive we can skip (no visits): [idinvalidation = 33, idsite = 1, period = day(2020-01-01 - 2020-01-01), name = donee0512c03f7c20af6ef96a8d792c6bb9f] +Skipping invalidated archive 63, segment 'actions>=4' is not in --force-idsegments: [idinvalidation = 63, idsite = 1, period = day(2020-01-01 - 2020-01-01), name = done49a9440bd6dba4b8850035e09d043c67] +Found invalidated archive we can skip (no visits): [idinvalidation = 34, idsite = 1, period = month(2020-01-01 - 2020-01-31), name = donee0512c03f7c20af6ef96a8d792c6bb9f] +Skipping invalidated archive 64, segment 'actions>=4' is not in --force-idsegments: [idinvalidation = 64, idsite = 1, period = month(2020-01-01 - 2020-01-31), name = done49a9440bd6dba4b8850035e09d043c67] +Found invalidated archive we can skip (no visits): [idinvalidation = 35, idsite = 1, period = year(2020-01-01 - 2020-12-31), name = donee0512c03f7c20af6ef96a8d792c6bb9f] +Skipping invalidated archive 65, segment 'actions>=4' is not in --force-idsegments: [idinvalidation = 65, idsite = 1, period = year(2020-01-01 - 2020-12-31), name = done49a9440bd6dba4b8850035e09d043c67] +Found invalidated archive we can skip (no visits): [idinvalidation = 28, idsite = 1, period = day(2019-12-31 - 2019-12-31), name = donee0512c03f7c20af6ef96a8d792c6bb9f] +Skipping invalidated archive 58, segment 'actions>=4' is not in --force-idsegments: [idinvalidation = 58, idsite = 1, period = day(2019-12-31 - 2019-12-31), name = done49a9440bd6dba4b8850035e09d043c67] +Found invalidated archive we can skip (no visits): [idinvalidation = 26, idsite = 1, period = day(2019-12-30 - 2019-12-30), name = donee0512c03f7c20af6ef96a8d792c6bb9f] +Skipping invalidated archive 56, segment 'actions>=4' is not in --force-idsegments: [idinvalidation = 56, idsite = 1, period = day(2019-12-30 - 2019-12-30), name = done49a9440bd6dba4b8850035e09d043c67] +Found invalidated archive we can skip (no visits): [idinvalidation = 27, idsite = 1, period = week(2019-12-30 - 2020-01-05), name = donee0512c03f7c20af6ef96a8d792c6bb9f] +Skipping invalidated archive 57, segment 'actions>=4' is not in --force-idsegments: [idinvalidation = 57, idsite = 1, period = week(2019-12-30 - 2020-01-05), name = done49a9440bd6dba4b8850035e09d043c67] +Found invalidated archive we can skip (no visits): [idinvalidation = 24, idsite = 1, period = day(2019-12-23 - 2019-12-23), name = donee0512c03f7c20af6ef96a8d792c6bb9f] +Skipping invalidated archive 54, segment 'actions>=4' is not in --force-idsegments: [idinvalidation = 54, idsite = 1, period = day(2019-12-23 - 2019-12-23), name = done49a9440bd6dba4b8850035e09d043c67] +Found invalidated archive we can skip (no visits): [idinvalidation = 25, idsite = 1, period = week(2019-12-23 - 2019-12-29), name = donee0512c03f7c20af6ef96a8d792c6bb9f] +Skipping invalidated archive 55, segment 'actions>=4' is not in --force-idsegments: [idinvalidation = 55, idsite = 1, period = week(2019-12-23 - 2019-12-29), name = done49a9440bd6dba4b8850035e09d043c67] +Found invalidated archive we can skip (no visits): [idinvalidation = 22, idsite = 1, period = day(2019-12-16 - 2019-12-16), name = donee0512c03f7c20af6ef96a8d792c6bb9f] +Skipping invalidated archive 52, segment 'actions>=4' is not in --force-idsegments: [idinvalidation = 52, idsite = 1, period = day(2019-12-16 - 2019-12-16), name = done49a9440bd6dba4b8850035e09d043c67] +Found invalidated archive we can skip (no visits): [idinvalidation = 23, idsite = 1, period = week(2019-12-16 - 2019-12-22), name = donee0512c03f7c20af6ef96a8d792c6bb9f] +Skipping invalidated archive 53, segment 'actions>=4' is not in --force-idsegments: [idinvalidation = 53, idsite = 1, period = week(2019-12-16 - 2019-12-22), name = done49a9440bd6dba4b8850035e09d043c67] +Skipping invalidated archive 1, segment '' is not in --force-idsegments: [idinvalidation = 1, idsite = 1, period = day(2019-12-12 - 2019-12-12), name = done] +Skipping invalidated archive 5, segment '' is not in --force-idsegments: [idinvalidation = 5, idsite = 1, period = day(2019-12-11 - 2019-12-11), name = done] +Skipping invalidated archive 9, segment '' is not in --force-idsegments: [idinvalidation = 9, idsite = 1, period = day(2019-12-10 - 2019-12-10), name = done] +Found invalidated archive we can skip (no visits): [idinvalidation = 20, idsite = 1, period = day(2019-12-09 - 2019-12-09), name = donee0512c03f7c20af6ef96a8d792c6bb9f] +Skipping invalidated archive 50, segment 'actions>=4' is not in --force-idsegments: [idinvalidation = 50, idsite = 1, period = day(2019-12-09 - 2019-12-09), name = done49a9440bd6dba4b8850035e09d043c67] +Skipping invalidated archive 2, segment '' is not in --force-idsegments: [idinvalidation = 2, idsite = 1, period = week(2019-12-09 - 2019-12-15), name = done] +Skipping invalidated archive 6, segment '' is not in --force-idsegments: [idinvalidation = 6, idsite = 1, period = week(2019-12-09 - 2019-12-15), name = done] +Skipping invalidated archive 10, segment '' is not in --force-idsegments: [idinvalidation = 10, idsite = 1, period = week(2019-12-09 - 2019-12-15), name = done] +Processing invalidation: [idinvalidation = 21, idsite = 1, period = week(2019-12-09 - 2019-12-15), name = donee0512c03f7c20af6ef96a8d792c6bb9f]. +Found archive with intersecting period with others in concurrent batch, skipping until next batch: [idinvalidation = 51, idsite = 1, period = week(2019-12-09 - 2019-12-15), name = done49a9440bd6dba4b8850035e09d043c67] +Found archive with different done flag type (segment vs. no segment) in concurrent batch, skipping until next batch: [idinvalidation = 13, idsite = 1, period = day(2019-12-02 - 2019-12-02), name = done] +Processing invalidation: [idinvalidation = 17, idsite = 1, period = day(2019-12-02 - 2019-12-02), name = donee0512c03f7c20af6ef96a8d792c6bb9f]. +Found archive with intersecting period with others in concurrent batch, skipping until next batch: [idinvalidation = 47, idsite = 1, period = day(2019-12-02 - 2019-12-02), name = done49a9440bd6dba4b8850035e09d043c67] +Found archive with different done flag type (segment vs. no segment) in concurrent batch, skipping until next batch: [idinvalidation = 14, idsite = 1, period = week(2019-12-02 - 2019-12-08), name = done] +Found archive with intersecting period with others in concurrent batch, skipping until next batch: [idinvalidation = 18, idsite = 1, period = week(2019-12-02 - 2019-12-08), name = donee0512c03f7c20af6ef96a8d792c6bb9f] +Found archive with intersecting period with others in concurrent batch, skipping until next batch: [idinvalidation = 48, idsite = 1, period = week(2019-12-02 - 2019-12-08), name = done49a9440bd6dba4b8850035e09d043c67] +Found archive with different done flag type (segment vs. no segment) in concurrent batch, skipping until next batch: [idinvalidation = 3, idsite = 1, period = month(2019-12-01 - 2019-12-31), name = done] +Found archive with different done flag type (segment vs. no segment) in concurrent batch, skipping until next batch: [idinvalidation = 7, idsite = 1, period = month(2019-12-01 - 2019-12-31), name = done] +Found archive with different done flag type (segment vs. no segment) in concurrent batch, skipping until next batch: [idinvalidation = 11, idsite = 1, period = month(2019-12-01 - 2019-12-31), name = done] +Found archive with different done flag type (segment vs. no segment) in concurrent batch, skipping until next batch: [idinvalidation = 15, idsite = 1, period = month(2019-12-01 - 2019-12-31), name = done] +Found archive with intersecting period with others in concurrent batch, skipping until next batch: [idinvalidation = 19, idsite = 1, period = month(2019-12-01 - 2019-12-31), name = donee0512c03f7c20af6ef96a8d792c6bb9f] +Found archive with intersecting period with others in concurrent batch, skipping until next batch: [idinvalidation = 49, idsite = 1, period = month(2019-12-01 - 2019-12-31), name = done49a9440bd6dba4b8850035e09d043c67] +Found archive with different done flag type (segment vs. no segment) in concurrent batch, skipping until next batch: [idinvalidation = 4, idsite = 1, period = year(2019-01-01 - 2019-12-31), name = done] +Found archive with different done flag type (segment vs. no segment) in concurrent batch, skipping until next batch: [idinvalidation = 8, idsite = 1, period = year(2019-01-01 - 2019-12-31), name = done] +Found archive with different done flag type (segment vs. no segment) in concurrent batch, skipping until next batch: [idinvalidation = 12, idsite = 1, period = year(2019-01-01 - 2019-12-31), name = done] +Found archive with different done flag type (segment vs. no segment) in concurrent batch, skipping until next batch: [idinvalidation = 16, idsite = 1, period = year(2019-01-01 - 2019-12-31), name = done] +Found archive with intersecting period with others in concurrent batch, skipping until next batch: [idinvalidation = 32, idsite = 1, period = year(2019-01-01 - 2019-12-31), name = donee0512c03f7c20af6ef96a8d792c6bb9f] +Found archive with intersecting period with others in concurrent batch, skipping until next batch: [idinvalidation = 62, idsite = 1, period = year(2019-01-01 - 2019-12-31), name = done49a9440bd6dba4b8850035e09d043c67] No next invalidated archive. -Starting archiving for ?module=API&method=API.get&idSite=1&period=week&date=2019-12-09&format=json&segment=actions%3E%3D2&trigger=archivephp -Starting archiving for ?module=API&method=API.get&idSite=1&period=week&date=2019-12-02&format=json&segment=actions%3E%3D2&trigger=archivephp +Starting archiving for ?module=API&method=CoreAdminHome.archiveReports&idSite=1&period=week&date=2019-12-09&format=json&segment=actions%3E%3D2&trigger=archivephp +Starting archiving for ?module=API&method=CoreAdminHome.archiveReports&idSite=1&period=day&date=2019-12-02&format=json&segment=actions%3E%3D2&trigger=archivephp Archived website id 1, period = week, date = 2019-12-09, segment = 'actions%3E%3D2', 0 visits found. Time elapsed: %fs -Archived website id 1, period = week, date = 2019-12-02, segment = 'actions%3E%3D2', 0 visits found. Time elapsed: %fs -Skipping invalidated archive : segment '' is not in --force-idsegments -Skipping invalidated archive : segment 'actions>=4' is not in --force-idsegments -Found archive with different period than others in concurrent batch, skipping until next batch: 2 -Found archive with different period than others in concurrent batch, skipping until next batch: 3 -Found archive with different period than others in concurrent batch, skipping until next batch: 3 -Found archive with different period than others in concurrent batch, skipping until next batch: 3 -Found archive with different period than others in concurrent batch, skipping until next batch: 3 -Found archive with different period than others in concurrent batch, skipping until next batch: 3 -Found archive with different period than others in concurrent batch, skipping until next batch: 3 -Found archive with different period than others in concurrent batch, skipping until next batch: 4 -Found archive with different period than others in concurrent batch, skipping until next batch: 4 -Found archive with different period than others in concurrent batch, skipping until next batch: 4 -Found archive with different period than others in concurrent batch, skipping until next batch: 4 -Found archive with different period than others in concurrent batch, skipping until next batch: 4 -Found archive with different period than others in concurrent batch, skipping until next batch: 4 -No next invalidated archive. -Starting archiving for ?module=API&method=API.get&idSite=1&period=day&date=2019-12-02&format=json&segment=actions%3E%3D2&trigger=archivephp Archived website id 1, period = day, date = 2019-12-02, segment = 'actions%3E%3D2', 0 visits found. Time elapsed: %fs -Skipping invalidated archive : segment '' is not in --force-idsegments -Skipping invalidated archive : segment '' is not in --force-idsegments -Skipping invalidated archive : segment '' is not in --force-idsegments -Skipping invalidated archive : segment '' is not in --force-idsegments -Skipping invalidated archive : segment '' is not in --force-idsegments -Skipping invalidated archive : segment 'actions>=4' is not in --force-idsegments -Found archive with different period than others in concurrent batch, skipping until next batch: 4 -Found archive with different period than others in concurrent batch, skipping until next batch: 4 -Found archive with different period than others in concurrent batch, skipping until next batch: 4 -Found archive with different period than others in concurrent batch, skipping until next batch: 4 -Found archive with different period than others in concurrent batch, skipping until next batch: 4 -Found archive with different period than others in concurrent batch, skipping until next batch: 4 +Skipping invalidated archive 51, segment 'actions>=4' is not in --force-idsegments: [idinvalidation = 51, idsite = 1, period = week(2019-12-09 - 2019-12-15), name = done49a9440bd6dba4b8850035e09d043c67] +Skipping invalidated archive 13, segment '' is not in --force-idsegments: [idinvalidation = 13, idsite = 1, period = day(2019-12-02 - 2019-12-02), name = done] +Skipping invalidated archive 47, segment 'actions>=4' is not in --force-idsegments: [idinvalidation = 47, idsite = 1, period = day(2019-12-02 - 2019-12-02), name = done49a9440bd6dba4b8850035e09d043c67] +Skipping invalidated archive 14, segment '' is not in --force-idsegments: [idinvalidation = 14, idsite = 1, period = week(2019-12-02 - 2019-12-08), name = done] +Processing invalidation: [idinvalidation = 18, idsite = 1, period = week(2019-12-02 - 2019-12-08), name = donee0512c03f7c20af6ef96a8d792c6bb9f]. +Found archive with intersecting period with others in concurrent batch, skipping until next batch: [idinvalidation = 48, idsite = 1, period = week(2019-12-02 - 2019-12-08), name = done49a9440bd6dba4b8850035e09d043c67] +Found archive with different done flag type (segment vs. no segment) in concurrent batch, skipping until next batch: [idinvalidation = 3, idsite = 1, period = month(2019-12-01 - 2019-12-31), name = done] +Found archive with different done flag type (segment vs. no segment) in concurrent batch, skipping until next batch: [idinvalidation = 7, idsite = 1, period = month(2019-12-01 - 2019-12-31), name = done] +Found archive with different done flag type (segment vs. no segment) in concurrent batch, skipping until next batch: [idinvalidation = 11, idsite = 1, period = month(2019-12-01 - 2019-12-31), name = done] +Found archive with different done flag type (segment vs. no segment) in concurrent batch, skipping until next batch: [idinvalidation = 15, idsite = 1, period = month(2019-12-01 - 2019-12-31), name = done] +Found archive with intersecting period with others in concurrent batch, skipping until next batch: [idinvalidation = 19, idsite = 1, period = month(2019-12-01 - 2019-12-31), name = donee0512c03f7c20af6ef96a8d792c6bb9f] +Found archive with intersecting period with others in concurrent batch, skipping until next batch: [idinvalidation = 49, idsite = 1, period = month(2019-12-01 - 2019-12-31), name = done49a9440bd6dba4b8850035e09d043c67] +Found archive with different done flag type (segment vs. no segment) in concurrent batch, skipping until next batch: [idinvalidation = 4, idsite = 1, period = year(2019-01-01 - 2019-12-31), name = done] +Found archive with different done flag type (segment vs. no segment) in concurrent batch, skipping until next batch: [idinvalidation = 8, idsite = 1, period = year(2019-01-01 - 2019-12-31), name = done] +Found archive with different done flag type (segment vs. no segment) in concurrent batch, skipping until next batch: [idinvalidation = 12, idsite = 1, period = year(2019-01-01 - 2019-12-31), name = done] +Found archive with different done flag type (segment vs. no segment) in concurrent batch, skipping until next batch: [idinvalidation = 16, idsite = 1, period = year(2019-01-01 - 2019-12-31), name = done] +Found archive with intersecting period with others in concurrent batch, skipping until next batch: [idinvalidation = 32, idsite = 1, period = year(2019-01-01 - 2019-12-31), name = donee0512c03f7c20af6ef96a8d792c6bb9f] +Found archive with intersecting period with others in concurrent batch, skipping until next batch: [idinvalidation = 62, idsite = 1, period = year(2019-01-01 - 2019-12-31), name = done49a9440bd6dba4b8850035e09d043c67] +No next invalidated archive. +Starting archiving for ?module=API&method=CoreAdminHome.archiveReports&idSite=1&period=week&date=2019-12-02&format=json&segment=actions%3E%3D2&trigger=archivephp +Archived website id 1, period = week, date = 2019-12-02, segment = 'actions%3E%3D2', 0 visits found. Time elapsed: %fs +Skipping invalidated archive 48, segment 'actions>=4' is not in --force-idsegments: [idinvalidation = 48, idsite = 1, period = week(2019-12-02 - 2019-12-08), name = done49a9440bd6dba4b8850035e09d043c67] +Skipping invalidated archive 3, segment '' is not in --force-idsegments: [idinvalidation = 3, idsite = 1, period = month(2019-12-01 - 2019-12-31), name = done] +Skipping invalidated archive 7, segment '' is not in --force-idsegments: [idinvalidation = 7, idsite = 1, period = month(2019-12-01 - 2019-12-31), name = done] +Skipping invalidated archive 11, segment '' is not in --force-idsegments: [idinvalidation = 11, idsite = 1, period = month(2019-12-01 - 2019-12-31), name = done] +Skipping invalidated archive 15, segment '' is not in --force-idsegments: [idinvalidation = 15, idsite = 1, period = month(2019-12-01 - 2019-12-31), name = done] +Processing invalidation: [idinvalidation = 19, idsite = 1, period = month(2019-12-01 - 2019-12-31), name = donee0512c03f7c20af6ef96a8d792c6bb9f]. +Found archive with intersecting period with others in concurrent batch, skipping until next batch: [idinvalidation = 49, idsite = 1, period = month(2019-12-01 - 2019-12-31), name = done49a9440bd6dba4b8850035e09d043c67] +Found archive with different done flag type (segment vs. no segment) in concurrent batch, skipping until next batch: [idinvalidation = 4, idsite = 1, period = year(2019-01-01 - 2019-12-31), name = done] +Found archive with different done flag type (segment vs. no segment) in concurrent batch, skipping until next batch: [idinvalidation = 8, idsite = 1, period = year(2019-01-01 - 2019-12-31), name = done] +Found archive with different done flag type (segment vs. no segment) in concurrent batch, skipping until next batch: [idinvalidation = 12, idsite = 1, period = year(2019-01-01 - 2019-12-31), name = done] +Found archive with different done flag type (segment vs. no segment) in concurrent batch, skipping until next batch: [idinvalidation = 16, idsite = 1, period = year(2019-01-01 - 2019-12-31), name = done] +Found archive with intersecting period with others in concurrent batch, skipping until next batch: [idinvalidation = 32, idsite = 1, period = year(2019-01-01 - 2019-12-31), name = donee0512c03f7c20af6ef96a8d792c6bb9f] +Found archive with intersecting period with others in concurrent batch, skipping until next batch: [idinvalidation = 62, idsite = 1, period = year(2019-01-01 - 2019-12-31), name = done49a9440bd6dba4b8850035e09d043c67] No next invalidated archive. -Starting archiving for ?module=API&method=API.get&idSite=1&period=month&date=2019-12-01&format=json&segment=actions%3E%3D2&trigger=archivephp +Starting archiving for ?module=API&method=CoreAdminHome.archiveReports&idSite=1&period=month&date=2019-12-01&format=json&segment=actions%3E%3D2&trigger=archivephp Archived website id 1, period = month, date = 2019-12-01, segment = 'actions%3E%3D2', 0 visits found. Time elapsed: %fs -Skipping invalidated archive : segment '' is not in --force-idsegments -Skipping invalidated archive : segment '' is not in --force-idsegments -Skipping invalidated archive : segment '' is not in --force-idsegments -Skipping invalidated archive : segment '' is not in --force-idsegments -Skipping invalidated archive : segment 'actions>=4' is not in --force-idsegments +Skipping invalidated archive 49, segment 'actions>=4' is not in --force-idsegments: [idinvalidation = 49, idsite = 1, period = month(2019-12-01 - 2019-12-31), name = done49a9440bd6dba4b8850035e09d043c67] +Skipping invalidated archive 4, segment '' is not in --force-idsegments: [idinvalidation = 4, idsite = 1, period = year(2019-01-01 - 2019-12-31), name = done] +Skipping invalidated archive 8, segment '' is not in --force-idsegments: [idinvalidation = 8, idsite = 1, period = year(2019-01-01 - 2019-12-31), name = done] +Skipping invalidated archive 12, segment '' is not in --force-idsegments: [idinvalidation = 12, idsite = 1, period = year(2019-01-01 - 2019-12-31), name = done] +Skipping invalidated archive 16, segment '' is not in --force-idsegments: [idinvalidation = 16, idsite = 1, period = year(2019-01-01 - 2019-12-31), name = done] +Processing invalidation: [idinvalidation = 32, idsite = 1, period = year(2019-01-01 - 2019-12-31), name = donee0512c03f7c20af6ef96a8d792c6bb9f]. +Found archive with intersecting period with others in concurrent batch, skipping until next batch: [idinvalidation = 62, idsite = 1, period = year(2019-01-01 - 2019-12-31), name = done49a9440bd6dba4b8850035e09d043c67] No next invalidated archive. -Starting archiving for ?module=API&method=API.get&idSite=1&period=year&date=2019-01-01&format=json&segment=actions%3E%3D2&trigger=archivephp +Starting archiving for ?module=API&method=CoreAdminHome.archiveReports&idSite=1&period=year&date=2019-01-01&format=json&segment=actions%3E%3D2&trigger=archivephp Archived website id 1, period = year, date = 2019-01-01, segment = 'actions%3E%3D2', 0 visits found. Time elapsed: %fs +Skipping invalidated archive 62, segment 'actions>=4' is not in --force-idsegments: [idinvalidation = 62, idsite = 1, period = year(2019-01-01 - 2019-12-31), name = done49a9440bd6dba4b8850035e09d043c67] No next invalidated archive. Finished archiving for site 1, 5 API requests, Time elapsed: %fs [1 / 1 done] No more sites left to archive, stopping. - +Done archiving! +--------------------------- +SUMMARY +Processed 5 archives. +Total API requests: 5 +done: 5 req, %d ms, no error +Time elapsed: %fs LOG; $this->assertStringMatchesFormat($expected, $logger->output); @@ -302,8 +440,8 @@ LOG; START Starting Matomo reports archiving... Checking for queued invalidations... - Today archive can be skipped due to no visits, skipping invalidation... - Yesterday archive can be skipped due to no visits, skipping invalidation... + Today archive can be skipped due to no visits for idSite = 1, skipping invalidation... + Yesterday archive can be skipped due to no visits for idSite = 1, skipping invalidation... Done invalidating Start processing archives for site 1. No next invalidated archive. diff --git a/tests/PHPUnit/Integration/DataAccess/ArchiveInvalidatorTest.php b/tests/PHPUnit/Integration/DataAccess/ArchiveInvalidatorTest.php index dedce82e58..e49a2c9699 100644 --- a/tests/PHPUnit/Integration/DataAccess/ArchiveInvalidatorTest.php +++ b/tests/PHPUnit/Integration/DataAccess/ArchiveInvalidatorTest.php @@ -11,6 +11,7 @@ namespace Piwik\Tests\Integration\DataAccess; use Piwik\ArchiveProcessor\ArchivingStatus; use Piwik\ArchiveProcessor\Rules; use Piwik\Common; +use Piwik\Config; use Piwik\Container\StaticContainer; use Piwik\DataAccess\ArchiveTableCreator; use Piwik\DataAccess\ArchiveWriter; @@ -21,6 +22,7 @@ use Piwik\Option; use Piwik\Piwik; use Piwik\Plugins\CoreAdminHome\Tasks\ArchivesToPurgeDistributedList; use Piwik\Plugins\PrivacyManager\PrivacyManager; +use Piwik\Plugins\SegmentEditor\API; use Piwik\Tests\Framework\Fixture; use Piwik\Tests\Framework\TestCase\IntegrationTestCase; use Piwik\Archive\ArchiveInvalidator; @@ -67,6 +69,10 @@ class ArchiveInvalidatorTest extends IntegrationTestCase for ($i = 0; $i != 10; ++$i) { Fixture::createWebsite('2012-03-04'); } + + self::addVisitToEachSite(); + + Option::deleteLike('%report_to_invalidate_%'); // test w/ a blank slate } public function setUp(): void @@ -314,6 +320,7 @@ class ArchiveInvalidatorTest extends IntegrationTestCase 'date2' => '2016-12-31', 'period' => '4', 'name' => 'done', + 'report' => null, ], [ 'idarchive' => null, @@ -322,6 +329,7 @@ class ArchiveInvalidatorTest extends IntegrationTestCase 'date2' => '2016-03-06', 'period' => '2', 'name' => 'done', + 'report' => null, ], [ 'idarchive' => null, @@ -330,6 +338,7 @@ class ArchiveInvalidatorTest extends IntegrationTestCase 'date2' => '2016-03-31', 'period' => '3', 'name' => 'done', + 'report' => null, ], [ 'idarchive' => null, @@ -338,6 +347,7 @@ class ArchiveInvalidatorTest extends IntegrationTestCase 'date2' => '2016-03-04', 'period' => '1', 'name' => 'done', + 'report' => null, ], ]; @@ -349,7 +359,7 @@ class ArchiveInvalidatorTest extends IntegrationTestCase * @dataProvider getTestDataForMarkArchivesAsInvalidated */ public function test_markArchivesAsInvalidated_MarksCorrectArchivesAsInvalidated($idSites, $dates, $period, $segment, $cascadeDown, $expectedIdArchives, - $expectedInvalidatedArchives) + $expectedInvalidatedArchives, $name = null) { $this->insertArchiveRowsForTest(); @@ -359,7 +369,8 @@ class ArchiveInvalidatorTest extends IntegrationTestCase /** @var ArchiveInvalidator $archiveInvalidator */ $archiveInvalidator = self::$fixture->piwikEnvironment->getContainer()->get('Piwik\Archive\ArchiveInvalidator'); - $result = $archiveInvalidator->markArchivesAsInvalidated($idSites, $dates, $period, $segment, $cascadeDown); + + $result = $archiveInvalidator->markArchivesAsInvalidated($idSites, $dates, $period, $segment, $cascadeDown, false, $name); $this->assertEquals($dates, $result->processedDates); @@ -367,11 +378,11 @@ class ArchiveInvalidatorTest extends IntegrationTestCase // Remove empty values (some new empty entries may be added each month) $idArchives = array_filter($idArchives); - $expectedIdArchives = array_filter($expectedIdArchives); $this->assertEquals($expectedIdArchives, $idArchives); $invalidatedIdArchives = $this->getInvalidatedArchiveTableEntries(); + $this->assertEqualsSorted($expectedInvalidatedArchives, $invalidatedIdArchives); } @@ -418,26 +429,26 @@ class ArchiveInvalidatorTest extends IntegrationTestCase ], ), [ - ['idarchive' => NULL, 'idsite' => '1', 'date1' => '2014-12-29', 'date2' => '2015-01-04', 'period' => '2', 'name' => 'done'], - ['idarchive' => NULL, 'idsite' => '1', 'date1' => '2015-01-01', 'date2' => '2015-01-01', 'period' => '1', 'name' => 'done'], - ['idarchive' => NULL, 'idsite' => '1', 'date1' => '2015-01-01', 'date2' => '2015-01-31', 'period' => '3', 'name' => 'done'], - ['idarchive' => NULL, 'idsite' => '1', 'date1' => '2015-01-01', 'date2' => '2015-12-31', 'period' => '4', 'name' => 'done'], - ['idarchive' => NULL, 'idsite' => '1', 'date1' => '2015-02-01', 'date2' => '2015-02-28', 'period' => '3', 'name' => 'done'], - ['idarchive' => '85', 'idsite' => '1', 'date1' => '2015-02-02', 'date2' => '2015-02-08', 'period' => '2', 'name' => 'done'], - ['idarchive' => NULL, 'idsite' => '1', 'date1' => '2015-02-05', 'date2' => '2015-02-05', 'period' => '1', 'name' => 'done'], - ['idarchive' => NULL, 'idsite' => '1', 'date1' => '2015-04-01', 'date2' => '2015-04-30', 'period' => '3', 'name' => 'done'], - ['idarchive' => '100', 'idsite' => '1', 'date1' => '2015-04-27', 'date2' => '2015-05-03', 'period' => '2', 'name' => 'done'], - ['idarchive' => NULL, 'idsite' => '1', 'date1' => '2015-04-30', 'date2' => '2015-04-30', 'period' => '1', 'name' => 'done'], - ['idarchive' => NULL, 'idsite' => '2', 'date1' => '2014-12-29', 'date2' => '2015-01-04', 'period' => '2', 'name' => 'done'], - ['idarchive' => NULL, 'idsite' => '2', 'date1' => '2015-01-01', 'date2' => '2015-01-01', 'period' => '1', 'name' => 'done'], - ['idarchive' => NULL, 'idsite' => '2', 'date1' => '2015-01-01', 'date2' => '2015-01-31', 'period' => '3', 'name' => 'done'], - ['idarchive' => '110', 'idsite' => '2', 'date1' => '2015-01-01', 'date2' => '2015-12-31', 'period' => '4', 'name' => 'done'], - ['idarchive' => NULL, 'idsite' => '2', 'date1' => '2015-02-01', 'date2' => '2015-02-28', 'period' => '3', 'name' => 'done'], - ['idarchive' => NULL, 'idsite' => '2', 'date1' => '2015-02-02', 'date2' => '2015-02-08', 'period' => '2', 'name' => 'done'], - ['idarchive' => NULL, 'idsite' => '2', 'date1' => '2015-02-05', 'date2' => '2015-02-05', 'period' => '1', 'name' => 'done'], - ['idarchive' => NULL, 'idsite' => '2', 'date1' => '2015-04-01', 'date2' => '2015-04-30', 'period' => '3', 'name' => 'done'], - ['idarchive' => NULL, 'idsite' => '2', 'date1' => '2015-04-27', 'date2' => '2015-05-03', 'period' => '2', 'name' => 'done'], - ['idarchive' => NULL, 'idsite' => '2', 'date1' => '2015-04-30', 'date2' => '2015-04-30', 'period' => '1', 'name' => 'done'], + ['idarchive' => NULL, 'idsite' => '1', 'date1' => '2014-12-29', 'date2' => '2015-01-04', 'period' => '2', 'name' => 'done', 'report' => null], + ['idarchive' => NULL, 'idsite' => '1', 'date1' => '2015-01-01', 'date2' => '2015-01-01', 'period' => '1', 'name' => 'done', 'report' => null], + ['idarchive' => NULL, 'idsite' => '1', 'date1' => '2015-01-01', 'date2' => '2015-01-31', 'period' => '3', 'name' => 'done', 'report' => null], + ['idarchive' => NULL, 'idsite' => '1', 'date1' => '2015-01-01', 'date2' => '2015-12-31', 'period' => '4', 'name' => 'done', 'report' => null], + ['idarchive' => NULL, 'idsite' => '1', 'date1' => '2015-02-01', 'date2' => '2015-02-28', 'period' => '3', 'name' => 'done', 'report' => null], + ['idarchive' => '85', 'idsite' => '1', 'date1' => '2015-02-02', 'date2' => '2015-02-08', 'period' => '2', 'name' => 'done', 'report' => null], + ['idarchive' => NULL, 'idsite' => '1', 'date1' => '2015-02-05', 'date2' => '2015-02-05', 'period' => '1', 'name' => 'done', 'report' => null], + ['idarchive' => NULL, 'idsite' => '1', 'date1' => '2015-04-01', 'date2' => '2015-04-30', 'period' => '3', 'name' => 'done', 'report' => null], + ['idarchive' => '100', 'idsite' => '1', 'date1' => '2015-04-27', 'date2' => '2015-05-03', 'period' => '2', 'name' => 'done', 'report' => null], + ['idarchive' => NULL, 'idsite' => '1', 'date1' => '2015-04-30', 'date2' => '2015-04-30', 'period' => '1', 'name' => 'done', 'report' => null], + ['idarchive' => NULL, 'idsite' => '2', 'date1' => '2014-12-29', 'date2' => '2015-01-04', 'period' => '2', 'name' => 'done', 'report' => null], + ['idarchive' => NULL, 'idsite' => '2', 'date1' => '2015-01-01', 'date2' => '2015-01-01', 'period' => '1', 'name' => 'done', 'report' => null], + ['idarchive' => NULL, 'idsite' => '2', 'date1' => '2015-01-01', 'date2' => '2015-01-31', 'period' => '3', 'name' => 'done', 'report' => null], + ['idarchive' => '110', 'idsite' => '2', 'date1' => '2015-01-01', 'date2' => '2015-12-31', 'period' => '4', 'name' => 'done', 'report' => null], + ['idarchive' => NULL, 'idsite' => '2', 'date1' => '2015-02-01', 'date2' => '2015-02-28', 'period' => '3', 'name' => 'done', 'report' => null], + ['idarchive' => NULL, 'idsite' => '2', 'date1' => '2015-02-02', 'date2' => '2015-02-08', 'period' => '2', 'name' => 'done', 'report' => null], + ['idarchive' => NULL, 'idsite' => '2', 'date1' => '2015-02-05', 'date2' => '2015-02-05', 'period' => '1', 'name' => 'done', 'report' => null], + ['idarchive' => NULL, 'idsite' => '2', 'date1' => '2015-04-01', 'date2' => '2015-04-30', 'period' => '3', 'name' => 'done', 'report' => null], + ['idarchive' => NULL, 'idsite' => '2', 'date1' => '2015-04-27', 'date2' => '2015-05-03', 'period' => '2', 'name' => 'done', 'report' => null], + ['idarchive' => NULL, 'idsite' => '2', 'date1' => '2015-04-30', 'date2' => '2015-04-30', 'period' => '1', 'name' => 'done', 'report' => null], ], ), @@ -455,8 +466,8 @@ class ArchiveInvalidatorTest extends IntegrationTestCase ), ), [ - ['idarchive' => NULL, 'idsite' => '1', 'date1' => '2015-01-01', 'date2' => '2015-01-31', 'period' => '3', 'name' => 'done'], - ['idarchive' => NULL, 'idsite' => '1', 'date1' => '2015-01-01', 'date2' => '2015-12-31', 'period' => '4', 'name' => 'done'], + ['idarchive' => NULL, 'idsite' => '1', 'date1' => '2015-01-01', 'date2' => '2015-01-31', 'period' => '3', 'name' => 'done', 'report' => null], + ['idarchive' => NULL, 'idsite' => '1', 'date1' => '2015-01-01', 'date2' => '2015-12-31', 'period' => '4', 'name' => 'done', 'report' => null], ], ), @@ -513,44 +524,44 @@ class ArchiveInvalidatorTest extends IntegrationTestCase ), ), [ - ['idarchive' => NULL, 'idsite' => '1', 'date1' => '2014-12-29', 'date2' => '2015-01-04', 'period' => '2', 'name' => 'done'], - ['idarchive' => NULL, 'idsite' => '1', 'date1' => '2015-01-01', 'date2' => '2015-01-01', 'period' => '1', 'name' => 'done'], - ['idarchive' => NULL, 'idsite' => '1', 'date1' => '2015-01-01', 'date2' => '2015-01-31', 'period' => '3', 'name' => 'done'], - ['idarchive' => NULL, 'idsite' => '1', 'date1' => '2015-01-01', 'date2' => '2015-12-31', 'period' => '4', 'name' => 'done'], - ['idarchive' => NULL, 'idsite' => '1', 'date1' => '2015-01-02', 'date2' => '2015-01-02', 'period' => '1', 'name' => 'done'], - ['idarchive' => NULL, 'idsite' => '1', 'date1' => '2015-01-03', 'date2' => '2015-01-03', 'period' => '1', 'name' => 'done'], - ['idarchive' => '10', 'idsite' => '1', 'date1' => '2015-01-04', 'date2' => '2015-01-04', 'period' => '1', 'name' => 'done'], - ['idarchive' => NULL, 'idsite' => '1', 'date1' => '2015-01-05', 'date2' => '2015-01-05', 'period' => '1', 'name' => 'done'], - ['idarchive' => NULL, 'idsite' => '1', 'date1' => '2015-01-05', 'date2' => '2015-01-11', 'period' => '2', 'name' => 'done'], - ['idarchive' => NULL, 'idsite' => '1', 'date1' => '2015-01-06', 'date2' => '2015-01-06', 'period' => '1', 'name' => 'done'], - ['idarchive' => NULL, 'idsite' => '1', 'date1' => '2015-01-07', 'date2' => '2015-01-07', 'period' => '1', 'name' => 'done'], - ['idarchive' => NULL, 'idsite' => '1', 'date1' => '2015-01-08', 'date2' => '2015-01-08', 'period' => '1', 'name' => 'done'], - ['idarchive' => '25', 'idsite' => '1', 'date1' => '2015-01-09', 'date2' => '2015-01-09', 'period' => '1', 'name' => 'done'], - ['idarchive' => NULL, 'idsite' => '1', 'date1' => '2015-01-10', 'date2' => '2015-01-10', 'period' => '1', 'name' => 'done'], - ['idarchive' => NULL, 'idsite' => '1', 'date1' => '2015-01-11', 'date2' => '2015-01-11', 'period' => '1', 'name' => 'done'], - ['idarchive' => NULL, 'idsite' => '1', 'date1' => '2015-01-12', 'date2' => '2015-01-12', 'period' => '1', 'name' => 'done'], - ['idarchive' => NULL, 'idsite' => '1', 'date1' => '2015-01-12', 'date2' => '2015-01-18', 'period' => '2', 'name' => 'done'], - ['idarchive' => NULL, 'idsite' => '1', 'date1' => '2015-01-13', 'date2' => '2015-01-13', 'period' => '1', 'name' => 'done'], - ['idarchive' => '40', 'idsite' => '1', 'date1' => '2015-01-14', 'date2' => '2015-01-14', 'period' => '1', 'name' => 'done'], - ['idarchive' => NULL, 'idsite' => '1', 'date1' => '2015-01-15', 'date2' => '2015-01-15', 'period' => '1', 'name' => 'done'], - ['idarchive' => NULL, 'idsite' => '1', 'date1' => '2015-01-16', 'date2' => '2015-01-16', 'period' => '1', 'name' => 'done'], - ['idarchive' => NULL, 'idsite' => '1', 'date1' => '2015-01-17', 'date2' => '2015-01-17', 'period' => '1', 'name' => 'done'], - ['idarchive' => NULL, 'idsite' => '1', 'date1' => '2015-01-18', 'date2' => '2015-01-18', 'period' => '1', 'name' => 'done'], - ['idarchive' => '55', 'idsite' => '1', 'date1' => '2015-01-19', 'date2' => '2015-01-19', 'period' => '1', 'name' => 'done'], - ['idarchive' => '100', 'idsite' => '1', 'date1' => '2015-01-19', 'date2' => '2015-01-25', 'period' => '2', 'name' => 'done'], - ['idarchive' => NULL, 'idsite' => '1', 'date1' => '2015-01-20', 'date2' => '2015-01-20', 'period' => '1', 'name' => 'done'], - ['idarchive' => NULL, 'idsite' => '1', 'date1' => '2015-01-21', 'date2' => '2015-01-21', 'period' => '1', 'name' => 'done'], - ['idarchive' => NULL, 'idsite' => '1', 'date1' => '2015-01-22', 'date2' => '2015-01-22', 'period' => '1', 'name' => 'done'], - ['idarchive' => NULL, 'idsite' => '1', 'date1' => '2015-01-23', 'date2' => '2015-01-23', 'period' => '1', 'name' => 'done'], - ['idarchive' => '70', 'idsite' => '1', 'date1' => '2015-01-24', 'date2' => '2015-01-24', 'period' => '1', 'name' => 'done'], - ['idarchive' => NULL, 'idsite' => '1', 'date1' => '2015-01-25', 'date2' => '2015-01-25', 'period' => '1', 'name' => 'done'], - ['idarchive' => NULL, 'idsite' => '1', 'date1' => '2015-01-26', 'date2' => '2015-01-26', 'period' => '1', 'name' => 'done'], - ['idarchive' => NULL, 'idsite' => '1', 'date1' => '2015-01-26', 'date2' => '2015-02-01', 'period' => '2', 'name' => 'done'], - ['idarchive' => NULL, 'idsite' => '1', 'date1' => '2015-01-27', 'date2' => '2015-01-27', 'period' => '1', 'name' => 'done'], - ['idarchive' => NULL, 'idsite' => '1', 'date1' => '2015-01-28', 'date2' => '2015-01-28', 'period' => '1', 'name' => 'done'], - ['idarchive' => '85', 'idsite' => '1', 'date1' => '2015-01-29', 'date2' => '2015-01-29', 'period' => '1', 'name' => 'done'], - ['idarchive' => NULL, 'idsite' => '1', 'date1' => '2015-01-30', 'date2' => '2015-01-30', 'period' => '1', 'name' => 'done'], - ['idarchive' => NULL, 'idsite' => '1', 'date1' => '2015-01-31', 'date2' => '2015-01-31', 'period' => '1', 'name' => 'done'], + ['idarchive' => NULL, 'idsite' => '1', 'date1' => '2014-12-29', 'date2' => '2015-01-04', 'period' => '2', 'name' => 'done', 'report' => null], + ['idarchive' => NULL, 'idsite' => '1', 'date1' => '2015-01-01', 'date2' => '2015-01-01', 'period' => '1', 'name' => 'done', 'report' => null], + ['idarchive' => NULL, 'idsite' => '1', 'date1' => '2015-01-01', 'date2' => '2015-01-31', 'period' => '3', 'name' => 'done', 'report' => null], + ['idarchive' => NULL, 'idsite' => '1', 'date1' => '2015-01-01', 'date2' => '2015-12-31', 'period' => '4', 'name' => 'done', 'report' => null], + ['idarchive' => NULL, 'idsite' => '1', 'date1' => '2015-01-02', 'date2' => '2015-01-02', 'period' => '1', 'name' => 'done', 'report' => null], + ['idarchive' => NULL, 'idsite' => '1', 'date1' => '2015-01-03', 'date2' => '2015-01-03', 'period' => '1', 'name' => 'done', 'report' => null], + ['idarchive' => '10', 'idsite' => '1', 'date1' => '2015-01-04', 'date2' => '2015-01-04', 'period' => '1', 'name' => 'done', 'report' => null], + ['idarchive' => NULL, 'idsite' => '1', 'date1' => '2015-01-05', 'date2' => '2015-01-05', 'period' => '1', 'name' => 'done', 'report' => null], + ['idarchive' => NULL, 'idsite' => '1', 'date1' => '2015-01-05', 'date2' => '2015-01-11', 'period' => '2', 'name' => 'done', 'report' => null], + ['idarchive' => NULL, 'idsite' => '1', 'date1' => '2015-01-06', 'date2' => '2015-01-06', 'period' => '1', 'name' => 'done', 'report' => null], + ['idarchive' => NULL, 'idsite' => '1', 'date1' => '2015-01-07', 'date2' => '2015-01-07', 'period' => '1', 'name' => 'done', 'report' => null], + ['idarchive' => NULL, 'idsite' => '1', 'date1' => '2015-01-08', 'date2' => '2015-01-08', 'period' => '1', 'name' => 'done', 'report' => null], + ['idarchive' => '25', 'idsite' => '1', 'date1' => '2015-01-09', 'date2' => '2015-01-09', 'period' => '1', 'name' => 'done', 'report' => null], + ['idarchive' => NULL, 'idsite' => '1', 'date1' => '2015-01-10', 'date2' => '2015-01-10', 'period' => '1', 'name' => 'done', 'report' => null], + ['idarchive' => NULL, 'idsite' => '1', 'date1' => '2015-01-11', 'date2' => '2015-01-11', 'period' => '1', 'name' => 'done', 'report' => null], + ['idarchive' => NULL, 'idsite' => '1', 'date1' => '2015-01-12', 'date2' => '2015-01-12', 'period' => '1', 'name' => 'done', 'report' => null], + ['idarchive' => NULL, 'idsite' => '1', 'date1' => '2015-01-12', 'date2' => '2015-01-18', 'period' => '2', 'name' => 'done', 'report' => null], + ['idarchive' => NULL, 'idsite' => '1', 'date1' => '2015-01-13', 'date2' => '2015-01-13', 'period' => '1', 'name' => 'done', 'report' => null], + ['idarchive' => '40', 'idsite' => '1', 'date1' => '2015-01-14', 'date2' => '2015-01-14', 'period' => '1', 'name' => 'done', 'report' => null], + ['idarchive' => NULL, 'idsite' => '1', 'date1' => '2015-01-15', 'date2' => '2015-01-15', 'period' => '1', 'name' => 'done', 'report' => null], + ['idarchive' => NULL, 'idsite' => '1', 'date1' => '2015-01-16', 'date2' => '2015-01-16', 'period' => '1', 'name' => 'done', 'report' => null], + ['idarchive' => NULL, 'idsite' => '1', 'date1' => '2015-01-17', 'date2' => '2015-01-17', 'period' => '1', 'name' => 'done', 'report' => null], + ['idarchive' => NULL, 'idsite' => '1', 'date1' => '2015-01-18', 'date2' => '2015-01-18', 'period' => '1', 'name' => 'done', 'report' => null], + ['idarchive' => '55', 'idsite' => '1', 'date1' => '2015-01-19', 'date2' => '2015-01-19', 'period' => '1', 'name' => 'done', 'report' => null], + ['idarchive' => '100', 'idsite' => '1', 'date1' => '2015-01-19', 'date2' => '2015-01-25', 'period' => '2', 'name' => 'done', 'report' => null], + ['idarchive' => NULL, 'idsite' => '1', 'date1' => '2015-01-20', 'date2' => '2015-01-20', 'period' => '1', 'name' => 'done', 'report' => null], + ['idarchive' => NULL, 'idsite' => '1', 'date1' => '2015-01-21', 'date2' => '2015-01-21', 'period' => '1', 'name' => 'done', 'report' => null], + ['idarchive' => NULL, 'idsite' => '1', 'date1' => '2015-01-22', 'date2' => '2015-01-22', 'period' => '1', 'name' => 'done', 'report' => null], + ['idarchive' => NULL, 'idsite' => '1', 'date1' => '2015-01-23', 'date2' => '2015-01-23', 'period' => '1', 'name' => 'done', 'report' => null], + ['idarchive' => '70', 'idsite' => '1', 'date1' => '2015-01-24', 'date2' => '2015-01-24', 'period' => '1', 'name' => 'done', 'report' => null], + ['idarchive' => NULL, 'idsite' => '1', 'date1' => '2015-01-25', 'date2' => '2015-01-25', 'period' => '1', 'name' => 'done', 'report' => null], + ['idarchive' => NULL, 'idsite' => '1', 'date1' => '2015-01-26', 'date2' => '2015-01-26', 'period' => '1', 'name' => 'done', 'report' => null], + ['idarchive' => NULL, 'idsite' => '1', 'date1' => '2015-01-26', 'date2' => '2015-02-01', 'period' => '2', 'name' => 'done', 'report' => null], + ['idarchive' => NULL, 'idsite' => '1', 'date1' => '2015-01-27', 'date2' => '2015-01-27', 'period' => '1', 'name' => 'done', 'report' => null], + ['idarchive' => NULL, 'idsite' => '1', 'date1' => '2015-01-28', 'date2' => '2015-01-28', 'period' => '1', 'name' => 'done', 'report' => null], + ['idarchive' => '85', 'idsite' => '1', 'date1' => '2015-01-29', 'date2' => '2015-01-29', 'period' => '1', 'name' => 'done', 'report' => null], + ['idarchive' => NULL, 'idsite' => '1', 'date1' => '2015-01-30', 'date2' => '2015-01-30', 'period' => '1', 'name' => 'done', 'report' => null], + ['idarchive' => NULL, 'idsite' => '1', 'date1' => '2015-01-31', 'date2' => '2015-01-31', 'period' => '1', 'name' => 'done', 'report' => null], ], ), @@ -595,27 +606,27 @@ class ArchiveInvalidatorTest extends IntegrationTestCase ], ), [ - ['idarchive' => NULL, 'idsite' => '1', 'date1' => '2014-01-01', 'date2' => '2014-12-31', 'period' => '4', 'name' => 'done'], - ['idarchive' => NULL, 'idsite' => '1', 'date1' => '2014-12-01', 'date2' => '2014-12-31', 'period' => '3', 'name' => 'done'], - ['idarchive' => '85', 'idsite' => '1', 'date1' => '2014-12-29', 'date2' => '2014-12-29', 'period' => '1', 'name' => 'done'], - ['idarchive' => NULL, 'idsite' => '1', 'date1' => '2014-12-29', 'date2' => '2015-01-04', 'period' => '2', 'name' => 'done'], - ['idarchive' => NULL, 'idsite' => '1', 'date1' => '2014-12-30', 'date2' => '2014-12-30', 'period' => '1', 'name' => 'done'], - ['idarchive' => NULL, 'idsite' => '1', 'date1' => '2014-12-31', 'date2' => '2014-12-31', 'period' => '1', 'name' => 'done'], - ['idarchive' => NULL, 'idsite' => '1', 'date1' => '2015-01-01', 'date2' => '2015-01-01', 'period' => '1', 'name' => 'done'], - ['idarchive' => NULL, 'idsite' => '1', 'date1' => '2015-01-01', 'date2' => '2015-01-31', 'period' => '3', 'name' => 'done'], - ['idarchive' => NULL, 'idsite' => '1', 'date1' => '2015-01-01', 'date2' => '2015-12-31', 'period' => '4', 'name' => 'done'], - ['idarchive' => NULL, 'idsite' => '1', 'date1' => '2015-01-02', 'date2' => '2015-01-02', 'period' => '1', 'name' => 'done'], - ['idarchive' => NULL, 'idsite' => '1', 'date1' => '2015-01-03', 'date2' => '2015-01-03', 'period' => '1', 'name' => 'done'], - ['idarchive' => '10', 'idsite' => '1', 'date1' => '2015-01-04', 'date2' => '2015-01-04', 'period' => '1', 'name' => 'done'], - ['idarchive' => NULL, 'idsite' => '1', 'date1' => '2015-01-26', 'date2' => '2015-01-26', 'period' => '1', 'name' => 'done'], - ['idarchive' => NULL, 'idsite' => '1', 'date1' => '2015-01-26', 'date2' => '2015-02-01', 'period' => '2', 'name' => 'done'], - ['idarchive' => NULL, 'idsite' => '1', 'date1' => '2015-01-27', 'date2' => '2015-01-27', 'period' => '1', 'name' => 'done'], - ['idarchive' => NULL, 'idsite' => '1', 'date1' => '2015-01-28', 'date2' => '2015-01-28', 'period' => '1', 'name' => 'done'], - ['idarchive' => '85', 'idsite' => '1', 'date1' => '2015-01-29', 'date2' => '2015-01-29', 'period' => '1', 'name' => 'done'], - ['idarchive' => NULL, 'idsite' => '1', 'date1' => '2015-01-30', 'date2' => '2015-01-30', 'period' => '1', 'name' => 'done'], - ['idarchive' => NULL, 'idsite' => '1', 'date1' => '2015-01-31', 'date2' => '2015-01-31', 'period' => '1', 'name' => 'done'], - ['idarchive' => NULL, 'idsite' => '1', 'date1' => '2015-02-01', 'date2' => '2015-02-01', 'period' => '1', 'name' => 'done'], - ['idarchive' => NULL, 'idsite' => '1', 'date1' => '2015-02-01', 'date2' => '2015-02-28', 'period' => '3', 'name' => 'done'], + ['idarchive' => NULL, 'idsite' => '1', 'date1' => '2014-01-01', 'date2' => '2014-12-31', 'period' => '4', 'name' => 'done', 'report' => null], + ['idarchive' => NULL, 'idsite' => '1', 'date1' => '2014-12-01', 'date2' => '2014-12-31', 'period' => '3', 'name' => 'done', 'report' => null], + ['idarchive' => '85', 'idsite' => '1', 'date1' => '2014-12-29', 'date2' => '2014-12-29', 'period' => '1', 'name' => 'done', 'report' => null], + ['idarchive' => NULL, 'idsite' => '1', 'date1' => '2014-12-29', 'date2' => '2015-01-04', 'period' => '2', 'name' => 'done', 'report' => null], + ['idarchive' => NULL, 'idsite' => '1', 'date1' => '2014-12-30', 'date2' => '2014-12-30', 'period' => '1', 'name' => 'done', 'report' => null], + ['idarchive' => NULL, 'idsite' => '1', 'date1' => '2014-12-31', 'date2' => '2014-12-31', 'period' => '1', 'name' => 'done', 'report' => null], + ['idarchive' => NULL, 'idsite' => '1', 'date1' => '2015-01-01', 'date2' => '2015-01-01', 'period' => '1', 'name' => 'done', 'report' => null], + ['idarchive' => NULL, 'idsite' => '1', 'date1' => '2015-01-01', 'date2' => '2015-01-31', 'period' => '3', 'name' => 'done', 'report' => null], + ['idarchive' => NULL, 'idsite' => '1', 'date1' => '2015-01-01', 'date2' => '2015-12-31', 'period' => '4', 'name' => 'done', 'report' => null], + ['idarchive' => NULL, 'idsite' => '1', 'date1' => '2015-01-02', 'date2' => '2015-01-02', 'period' => '1', 'name' => 'done', 'report' => null], + ['idarchive' => NULL, 'idsite' => '1', 'date1' => '2015-01-03', 'date2' => '2015-01-03', 'period' => '1', 'name' => 'done', 'report' => null], + ['idarchive' => '10', 'idsite' => '1', 'date1' => '2015-01-04', 'date2' => '2015-01-04', 'period' => '1', 'name' => 'done', 'report' => null], + ['idarchive' => NULL, 'idsite' => '1', 'date1' => '2015-01-26', 'date2' => '2015-01-26', 'period' => '1', 'name' => 'done', 'report' => null], + ['idarchive' => NULL, 'idsite' => '1', 'date1' => '2015-01-26', 'date2' => '2015-02-01', 'period' => '2', 'name' => 'done', 'report' => null], + ['idarchive' => NULL, 'idsite' => '1', 'date1' => '2015-01-27', 'date2' => '2015-01-27', 'period' => '1', 'name' => 'done', 'report' => null], + ['idarchive' => NULL, 'idsite' => '1', 'date1' => '2015-01-28', 'date2' => '2015-01-28', 'period' => '1', 'name' => 'done', 'report' => null], + ['idarchive' => '85', 'idsite' => '1', 'date1' => '2015-01-29', 'date2' => '2015-01-29', 'period' => '1', 'name' => 'done', 'report' => null], + ['idarchive' => NULL, 'idsite' => '1', 'date1' => '2015-01-30', 'date2' => '2015-01-30', 'period' => '1', 'name' => 'done', 'report' => null], + ['idarchive' => NULL, 'idsite' => '1', 'date1' => '2015-01-31', 'date2' => '2015-01-31', 'period' => '1', 'name' => 'done', 'report' => null], + ['idarchive' => NULL, 'idsite' => '1', 'date1' => '2015-02-01', 'date2' => '2015-02-01', 'period' => '1', 'name' => 'done', 'report' => null], + ['idarchive' => NULL, 'idsite' => '1', 'date1' => '2015-02-01', 'date2' => '2015-02-28', 'period' => '3', 'name' => 'done', 'report' => null], ], ), @@ -688,44 +699,44 @@ class ArchiveInvalidatorTest extends IntegrationTestCase ), [ // TODO: super strange, there are two idarchive = 106 values here - ['idarchive' => '106', 'idsite' => '1', 'date1' => '2015-01-01', 'date2' => '2015-01-31', 'period' => '3', 'name' => 'done3736b708e4d20cfc10610e816a1b2341'], - ['idarchive' => '1', 'idsite' => '1', 'date1' => '2015-01-01', 'date2' => '2015-01-01', 'period' => '1', 'name' => 'done3736b708e4d20cfc10610e816a1b2341'], - ['idarchive' => NULL, 'idsite' => '1', 'date1' => '2015-01-01', 'date2' => '2015-12-31', 'period' => '4', 'name' => 'done3736b708e4d20cfc10610e816a1b2341'], - ['idarchive' => NULL, 'idsite' => '1', 'date1' => '2015-01-02', 'date2' => '2015-01-02', 'period' => '1', 'name' => 'done3736b708e4d20cfc10610e816a1b2341'], - ['idarchive' => NULL, 'idsite' => '1', 'date1' => '2015-01-03', 'date2' => '2015-01-03', 'period' => '1', 'name' => 'done3736b708e4d20cfc10610e816a1b2341'], - ['idarchive' => NULL, 'idsite' => '1', 'date1' => '2015-01-04', 'date2' => '2015-01-04', 'period' => '1', 'name' => 'done3736b708e4d20cfc10610e816a1b2341'], - ['idarchive' => NULL, 'idsite' => '1', 'date1' => '2015-01-05', 'date2' => '2015-01-11', 'period' => '2', 'name' => 'done3736b708e4d20cfc10610e816a1b2341'], - ['idarchive' => NULL, 'idsite' => '1', 'date1' => '2015-01-05', 'date2' => '2015-01-05', 'period' => '1', 'name' => 'done3736b708e4d20cfc10610e816a1b2341'], - ['idarchive' => '16', 'idsite' => '1', 'date1' => '2015-01-06', 'date2' => '2015-01-06', 'period' => '1', 'name' => 'done3736b708e4d20cfc10610e816a1b2341'], - ['idarchive' => NULL, 'idsite' => '1', 'date1' => '2015-01-07', 'date2' => '2015-01-07', 'period' => '1', 'name' => 'done3736b708e4d20cfc10610e816a1b2341'], - ['idarchive' => NULL, 'idsite' => '1', 'date1' => '2015-01-08', 'date2' => '2015-01-08', 'period' => '1', 'name' => 'done3736b708e4d20cfc10610e816a1b2341'], - ['idarchive' => NULL, 'idsite' => '1', 'date1' => '2015-01-09', 'date2' => '2015-01-09', 'period' => '1', 'name' => 'done3736b708e4d20cfc10610e816a1b2341'], - ['idarchive' => NULL, 'idsite' => '1', 'date1' => '2015-01-10', 'date2' => '2015-01-10', 'period' => '1', 'name' => 'done3736b708e4d20cfc10610e816a1b2341'], - ['idarchive' => '31', 'idsite' => '1', 'date1' => '2015-01-11', 'date2' => '2015-01-11', 'period' => '1', 'name' => 'done3736b708e4d20cfc10610e816a1b2341'], - ['idarchive' => NULL, 'idsite' => '1', 'date1' => '2015-01-12', 'date2' => '2015-01-18', 'period' => '2', 'name' => 'done3736b708e4d20cfc10610e816a1b2341'], - ['idarchive' => NULL, 'idsite' => '1', 'date1' => '2015-01-12', 'date2' => '2015-01-12', 'period' => '1', 'name' => 'done3736b708e4d20cfc10610e816a1b2341'], - ['idarchive' => NULL, 'idsite' => '1', 'date1' => '2015-01-13', 'date2' => '2015-01-13', 'period' => '1', 'name' => 'done3736b708e4d20cfc10610e816a1b2341'], - ['idarchive' => NULL, 'idsite' => '1', 'date1' => '2015-01-14', 'date2' => '2015-01-14', 'period' => '1', 'name' => 'done3736b708e4d20cfc10610e816a1b2341'], - ['idarchive' => NULL, 'idsite' => '1', 'date1' => '2015-01-15', 'date2' => '2015-01-15', 'period' => '1', 'name' => 'done3736b708e4d20cfc10610e816a1b2341'], - ['idarchive' => '46', 'idsite' => '1', 'date1' => '2015-01-16', 'date2' => '2015-01-16', 'period' => '1', 'name' => 'done3736b708e4d20cfc10610e816a1b2341'], - ['idarchive' => NULL, 'idsite' => '1', 'date1' => '2015-01-17', 'date2' => '2015-01-17', 'period' => '1', 'name' => 'done3736b708e4d20cfc10610e816a1b2341'], - ['idarchive' => NULL, 'idsite' => '1', 'date1' => '2015-01-18', 'date2' => '2015-01-18', 'period' => '1', 'name' => 'done3736b708e4d20cfc10610e816a1b2341'], - ['idarchive' => NULL, 'idsite' => '1', 'date1' => '2015-01-19', 'date2' => '2015-01-25', 'period' => '2', 'name' => 'done3736b708e4d20cfc10610e816a1b2341'], - ['idarchive' => NULL, 'idsite' => '1', 'date1' => '2015-01-19', 'date2' => '2015-01-19', 'period' => '1', 'name' => 'done3736b708e4d20cfc10610e816a1b2341'], - ['idarchive' => NULL, 'idsite' => '1', 'date1' => '2015-01-20', 'date2' => '2015-01-20', 'period' => '1', 'name' => 'done3736b708e4d20cfc10610e816a1b2341'], - ['idarchive' => '61', 'idsite' => '1', 'date1' => '2015-01-21', 'date2' => '2015-01-21', 'period' => '1', 'name' => 'done3736b708e4d20cfc10610e816a1b2341'], - ['idarchive' => NULL, 'idsite' => '1', 'date1' => '2015-01-22', 'date2' => '2015-01-22', 'period' => '1', 'name' => 'done3736b708e4d20cfc10610e816a1b2341'], - ['idarchive' => NULL, 'idsite' => '1', 'date1' => '2015-01-23', 'date2' => '2015-01-23', 'period' => '1', 'name' => 'done3736b708e4d20cfc10610e816a1b2341'], - ['idarchive' => NULL, 'idsite' => '1', 'date1' => '2015-01-24', 'date2' => '2015-01-24', 'period' => '1', 'name' => 'done3736b708e4d20cfc10610e816a1b2341'], - ['idarchive' => NULL, 'idsite' => '1', 'date1' => '2015-01-25', 'date2' => '2015-01-25', 'period' => '1', 'name' => 'done3736b708e4d20cfc10610e816a1b2341'], - ['idarchive' => '76', 'idsite' => '1', 'date1' => '2015-01-26', 'date2' => '2015-01-26', 'period' => '1', 'name' => 'done3736b708e4d20cfc10610e816a1b2341'], - ['idarchive' => NULL, 'idsite' => '1', 'date1' => '2015-01-26', 'date2' => '2015-02-01', 'period' => '2', 'name' => 'done3736b708e4d20cfc10610e816a1b2341'], - ['idarchive' => NULL, 'idsite' => '1', 'date1' => '2015-01-27', 'date2' => '2015-01-27', 'period' => '1', 'name' => 'done3736b708e4d20cfc10610e816a1b2341'], - ['idarchive' => NULL, 'idsite' => '1', 'date1' => '2015-01-28', 'date2' => '2015-01-28', 'period' => '1', 'name' => 'done3736b708e4d20cfc10610e816a1b2341'], - ['idarchive' => NULL, 'idsite' => '1', 'date1' => '2015-01-29', 'date2' => '2015-01-29', 'period' => '1', 'name' => 'done3736b708e4d20cfc10610e816a1b2341'], - ['idarchive' => NULL, 'idsite' => '1', 'date1' => '2015-01-30', 'date2' => '2015-01-30', 'period' => '1', 'name' => 'done3736b708e4d20cfc10610e816a1b2341'], - ['idarchive' => '106', 'idsite' => '1', 'date1' => '2014-12-29', 'date2' => '2015-01-04', 'period' => '2', 'name' => 'done3736b708e4d20cfc10610e816a1b2341'], - ['idarchive' => '91', 'idsite' => '1', 'date1' => '2015-01-31', 'date2' => '2015-01-31', 'period' => '1', 'name' => 'done3736b708e4d20cfc10610e816a1b2341'], + ['idarchive' => '106', 'idsite' => '1', 'date1' => '2015-01-01', 'date2' => '2015-01-31', 'period' => '3', 'name' => 'done3736b708e4d20cfc10610e816a1b2341', 'report' => null], + ['idarchive' => '1', 'idsite' => '1', 'date1' => '2015-01-01', 'date2' => '2015-01-01', 'period' => '1', 'name' => 'done3736b708e4d20cfc10610e816a1b2341', 'report' => null], + ['idarchive' => NULL, 'idsite' => '1', 'date1' => '2015-01-01', 'date2' => '2015-12-31', 'period' => '4', 'name' => 'done3736b708e4d20cfc10610e816a1b2341', 'report' => null], + ['idarchive' => NULL, 'idsite' => '1', 'date1' => '2015-01-02', 'date2' => '2015-01-02', 'period' => '1', 'name' => 'done3736b708e4d20cfc10610e816a1b2341', 'report' => null], + ['idarchive' => NULL, 'idsite' => '1', 'date1' => '2015-01-03', 'date2' => '2015-01-03', 'period' => '1', 'name' => 'done3736b708e4d20cfc10610e816a1b2341', 'report' => null], + ['idarchive' => NULL, 'idsite' => '1', 'date1' => '2015-01-04', 'date2' => '2015-01-04', 'period' => '1', 'name' => 'done3736b708e4d20cfc10610e816a1b2341', 'report' => null], + ['idarchive' => NULL, 'idsite' => '1', 'date1' => '2015-01-05', 'date2' => '2015-01-11', 'period' => '2', 'name' => 'done3736b708e4d20cfc10610e816a1b2341', 'report' => null], + ['idarchive' => NULL, 'idsite' => '1', 'date1' => '2015-01-05', 'date2' => '2015-01-05', 'period' => '1', 'name' => 'done3736b708e4d20cfc10610e816a1b2341', 'report' => null], + ['idarchive' => '16', 'idsite' => '1', 'date1' => '2015-01-06', 'date2' => '2015-01-06', 'period' => '1', 'name' => 'done3736b708e4d20cfc10610e816a1b2341', 'report' => null], + ['idarchive' => NULL, 'idsite' => '1', 'date1' => '2015-01-07', 'date2' => '2015-01-07', 'period' => '1', 'name' => 'done3736b708e4d20cfc10610e816a1b2341', 'report' => null], + ['idarchive' => NULL, 'idsite' => '1', 'date1' => '2015-01-08', 'date2' => '2015-01-08', 'period' => '1', 'name' => 'done3736b708e4d20cfc10610e816a1b2341', 'report' => null], + ['idarchive' => NULL, 'idsite' => '1', 'date1' => '2015-01-09', 'date2' => '2015-01-09', 'period' => '1', 'name' => 'done3736b708e4d20cfc10610e816a1b2341', 'report' => null], + ['idarchive' => NULL, 'idsite' => '1', 'date1' => '2015-01-10', 'date2' => '2015-01-10', 'period' => '1', 'name' => 'done3736b708e4d20cfc10610e816a1b2341', 'report' => null], + ['idarchive' => '31', 'idsite' => '1', 'date1' => '2015-01-11', 'date2' => '2015-01-11', 'period' => '1', 'name' => 'done3736b708e4d20cfc10610e816a1b2341', 'report' => null], + ['idarchive' => NULL, 'idsite' => '1', 'date1' => '2015-01-12', 'date2' => '2015-01-18', 'period' => '2', 'name' => 'done3736b708e4d20cfc10610e816a1b2341', 'report' => null], + ['idarchive' => NULL, 'idsite' => '1', 'date1' => '2015-01-12', 'date2' => '2015-01-12', 'period' => '1', 'name' => 'done3736b708e4d20cfc10610e816a1b2341', 'report' => null], + ['idarchive' => NULL, 'idsite' => '1', 'date1' => '2015-01-13', 'date2' => '2015-01-13', 'period' => '1', 'name' => 'done3736b708e4d20cfc10610e816a1b2341', 'report' => null], + ['idarchive' => NULL, 'idsite' => '1', 'date1' => '2015-01-14', 'date2' => '2015-01-14', 'period' => '1', 'name' => 'done3736b708e4d20cfc10610e816a1b2341', 'report' => null], + ['idarchive' => NULL, 'idsite' => '1', 'date1' => '2015-01-15', 'date2' => '2015-01-15', 'period' => '1', 'name' => 'done3736b708e4d20cfc10610e816a1b2341', 'report' => null], + ['idarchive' => '46', 'idsite' => '1', 'date1' => '2015-01-16', 'date2' => '2015-01-16', 'period' => '1', 'name' => 'done3736b708e4d20cfc10610e816a1b2341', 'report' => null], + ['idarchive' => NULL, 'idsite' => '1', 'date1' => '2015-01-17', 'date2' => '2015-01-17', 'period' => '1', 'name' => 'done3736b708e4d20cfc10610e816a1b2341', 'report' => null], + ['idarchive' => NULL, 'idsite' => '1', 'date1' => '2015-01-18', 'date2' => '2015-01-18', 'period' => '1', 'name' => 'done3736b708e4d20cfc10610e816a1b2341', 'report' => null], + ['idarchive' => NULL, 'idsite' => '1', 'date1' => '2015-01-19', 'date2' => '2015-01-25', 'period' => '2', 'name' => 'done3736b708e4d20cfc10610e816a1b2341', 'report' => null], + ['idarchive' => NULL, 'idsite' => '1', 'date1' => '2015-01-19', 'date2' => '2015-01-19', 'period' => '1', 'name' => 'done3736b708e4d20cfc10610e816a1b2341', 'report' => null], + ['idarchive' => NULL, 'idsite' => '1', 'date1' => '2015-01-20', 'date2' => '2015-01-20', 'period' => '1', 'name' => 'done3736b708e4d20cfc10610e816a1b2341', 'report' => null], + ['idarchive' => '61', 'idsite' => '1', 'date1' => '2015-01-21', 'date2' => '2015-01-21', 'period' => '1', 'name' => 'done3736b708e4d20cfc10610e816a1b2341', 'report' => null], + ['idarchive' => NULL, 'idsite' => '1', 'date1' => '2015-01-22', 'date2' => '2015-01-22', 'period' => '1', 'name' => 'done3736b708e4d20cfc10610e816a1b2341', 'report' => null], + ['idarchive' => NULL, 'idsite' => '1', 'date1' => '2015-01-23', 'date2' => '2015-01-23', 'period' => '1', 'name' => 'done3736b708e4d20cfc10610e816a1b2341', 'report' => null], + ['idarchive' => NULL, 'idsite' => '1', 'date1' => '2015-01-24', 'date2' => '2015-01-24', 'period' => '1', 'name' => 'done3736b708e4d20cfc10610e816a1b2341', 'report' => null], + ['idarchive' => NULL, 'idsite' => '1', 'date1' => '2015-01-25', 'date2' => '2015-01-25', 'period' => '1', 'name' => 'done3736b708e4d20cfc10610e816a1b2341', 'report' => null], + ['idarchive' => '76', 'idsite' => '1', 'date1' => '2015-01-26', 'date2' => '2015-01-26', 'period' => '1', 'name' => 'done3736b708e4d20cfc10610e816a1b2341', 'report' => null], + ['idarchive' => NULL, 'idsite' => '1', 'date1' => '2015-01-26', 'date2' => '2015-02-01', 'period' => '2', 'name' => 'done3736b708e4d20cfc10610e816a1b2341', 'report' => null], + ['idarchive' => NULL, 'idsite' => '1', 'date1' => '2015-01-27', 'date2' => '2015-01-27', 'period' => '1', 'name' => 'done3736b708e4d20cfc10610e816a1b2341', 'report' => null], + ['idarchive' => NULL, 'idsite' => '1', 'date1' => '2015-01-28', 'date2' => '2015-01-28', 'period' => '1', 'name' => 'done3736b708e4d20cfc10610e816a1b2341', 'report' => null], + ['idarchive' => NULL, 'idsite' => '1', 'date1' => '2015-01-29', 'date2' => '2015-01-29', 'period' => '1', 'name' => 'done3736b708e4d20cfc10610e816a1b2341', 'report' => null], + ['idarchive' => NULL, 'idsite' => '1', 'date1' => '2015-01-30', 'date2' => '2015-01-30', 'period' => '1', 'name' => 'done3736b708e4d20cfc10610e816a1b2341', 'report' => null], + ['idarchive' => '106', 'idsite' => '1', 'date1' => '2014-12-29', 'date2' => '2015-01-04', 'period' => '2', 'name' => 'done3736b708e4d20cfc10610e816a1b2341', 'report' => null], + ['idarchive' => '91', 'idsite' => '1', 'date1' => '2015-01-31', 'date2' => '2015-01-31', 'period' => '1', 'name' => 'done3736b708e4d20cfc10610e816a1b2341', 'report' => null], ], ), @@ -747,12 +758,30 @@ class ArchiveInvalidatorTest extends IntegrationTestCase ), ), [ - ['idarchive' => NULL, 'idsite' => '1', 'date1' => '2015-05-05', 'date2' => '2015-05-05', 'period' => '1', 'name' => 'done'], - ['idarchive' => NULL, 'idsite' => '1', 'date1' => '2015-05-04', 'date2' => '2015-05-10', 'period' => '2', 'name' => 'done'], - ['idarchive' => NULL, 'idsite' => '1', 'date1' => '2015-05-01', 'date2' => '2015-05-31', 'period' => '3', 'name' => 'done'], - ['idarchive' => NULL, 'idsite' => '1', 'date1' => '2015-01-01', 'date2' => '2015-12-31', 'period' => '4', 'name' => 'done'], + ['idarchive' => NULL, 'idsite' => '1', 'date1' => '2015-05-05', 'date2' => '2015-05-05', 'period' => '1', 'name' => 'done', 'report' => null], + ['idarchive' => NULL, 'idsite' => '1', 'date1' => '2015-05-04', 'date2' => '2015-05-10', 'period' => '2', 'name' => 'done', 'report' => null], + ['idarchive' => NULL, 'idsite' => '1', 'date1' => '2015-05-01', 'date2' => '2015-05-31', 'period' => '3', 'name' => 'done', 'report' => null], + ['idarchive' => NULL, 'idsite' => '1', 'date1' => '2015-01-01', 'date2' => '2015-12-31', 'period' => '4', 'name' => 'done', 'report' => null], ], ), + + // period before site creation date + [ + [1], + ['2012-03-02'], + '', + null, + false, + [ + // empty + ], + [ + // month week and year exist, but not day since it is before the site was created + ['idarchive' => null, 'idsite' => 1, 'date1' => '2012-03-01', 'date2' => '2012-03-31', 'period' => 3, 'name' => 'done', 'report' => null], + ['idarchive' => null, 'idsite' => 1, 'date1' => '2012-02-27', 'date2' => '2012-03-04', 'period' => 2, 'name' => 'done', 'report' => null], + ['idarchive' => null, 'idsite' => 1, 'date1' => '2012-01-01', 'date2' => '2012-12-31', 'period' => 4, 'name' => 'done', 'report' => null], + ], + ], ); } @@ -916,14 +945,257 @@ class ArchiveInvalidatorTest extends IntegrationTestCase $this->assertEquals([], $archives); $expectedInvalidatedTableEntries = [ - ['idarchive' => NULL, 'idsite' => '1', 'date1' => '2015-03-04', 'date2' => '2015-03-06', 'period' => '5', 'name' => 'done'], - ['idarchive' => NULL, 'idsite' => '1', 'date1' => '2016-04-03', 'date2' => '2016-05-12', 'period' => '5', 'name' => 'done'], + ['idarchive' => NULL, 'idsite' => '1', 'date1' => '2015-03-04', 'date2' => '2015-03-06', 'period' => '5', 'name' => 'done', 'report' => null], + ['idarchive' => NULL, 'idsite' => '1', 'date1' => '2016-04-03', 'date2' => '2016-05-12', 'period' => '5', 'name' => 'done', 'report' => null], ]; $invalidatedTableEntries = $this->getInvalidatedArchiveTableEntries(); $this->assertEquals($expectedInvalidatedTableEntries, $invalidatedTableEntries); } + public function test_markArchivesAsInvalidated_invalidatesIndividualPluginNames() + { + $idSites = [1]; + $dates = ['2015-01-11']; + $period = 'day'; + $segment = new Segment('', [1]); + $cascadeDown = false; + $expectedIdArchives = []; + $expectedInvalidatedArchives = [ + ['idarchive' => NULL, 'idsite' => '1', 'date1' => '2015-01-01', 'date2' => '2015-01-31', 'period' => '3', 'name' => 'done.ExamplePlugin', 'report' => null], + ['idarchive' => NULL, 'idsite' => '1', 'date1' => '2015-01-01', 'date2' => '2015-12-31', 'period' => '4', 'name' => 'done.ExamplePlugin', 'report' => null], + ['idarchive' => NULL, 'idsite' => '1', 'date1' => '2015-01-05', 'date2' => '2015-01-11', 'period' => '2', 'name' => 'done.ExamplePlugin', 'report' => null], + ['idarchive' => NULL, 'idsite' => '1', 'date1' => '2015-01-11', 'date2' => '2015-01-11', 'period' => '1', 'name' => 'done.ExamplePlugin', 'report' => null], + ]; + $plugin = 'ExamplePlugin'; + + $this->test_markArchivesAsInvalidated_MarksCorrectArchivesAsInvalidated($idSites, $dates, $period, $segment, $cascadeDown, $expectedIdArchives, + $expectedInvalidatedArchives, $plugin); + } + + public function test_markArchivesAsInvalidated_invalidatesIndividualReports() + { + $idSites = [1]; + $dates = ['2015-01-11']; + $period = 'day'; + $segment = new Segment('', [1]); + $cascadeDown = false; + $expectedIdArchives = []; + $expectedInvalidatedArchives = [ + ['idarchive' => NULL, 'idsite' => '1', 'date1' => '2015-01-01', 'date2' => '2015-01-31', 'period' => '3', 'name' => 'done.ExamplePlugin', 'report' => 'someReport'], + ['idarchive' => NULL, 'idsite' => '1', 'date1' => '2015-01-01', 'date2' => '2015-12-31', 'period' => '4', 'name' => 'done.ExamplePlugin', 'report' => 'someReport'], + ['idarchive' => NULL, 'idsite' => '1', 'date1' => '2015-01-05', 'date2' => '2015-01-11', 'period' => '2', 'name' => 'done.ExamplePlugin', 'report' => 'someReport'], + ['idarchive' => NULL, 'idsite' => '1', 'date1' => '2015-01-11', 'date2' => '2015-01-11', 'period' => '1', 'name' => 'done.ExamplePlugin', 'report' => 'someReport'], + ]; + $report = 'ExamplePlugin.someReport'; + + $this->test_markArchivesAsInvalidated_MarksCorrectArchivesAsInvalidated($idSites, $dates, $period, $segment, $cascadeDown, $expectedIdArchives, + $expectedInvalidatedArchives, $report); + } + + public function test_reArchiveReport_createsCorrectInvalidationEntries_ifNoReportSpecified() + { + Date::$now = strtotime('2020-06-16 12:00:00'); + + Config::getInstance()->General['rearchive_reports_in_past_last_n_months'] = 'last1'; + + $this->invalidator->reArchiveReport([1], 'VisitsSummary'); + + $expectedInvalidations = [ + array ( + 'idsite' => '1', + 'period' => '1', + 'name' => 'done.VisitsSummary', + 'report' => NULL, + 'dates' => '2020-05-01,2020-05-01|2020-05-02,2020-05-02|2020-05-03,2020-05-03|2020-05-04,2020-05-04|2020-05-05,2020-05-05|2020-05-06,2020-05-06' + . '|2020-05-07,2020-05-07|2020-05-08,2020-05-08|2020-05-09,2020-05-09|2020-05-10,2020-05-10|2020-05-11,2020-05-11|2020-05-12,2020-05-12' + . '|2020-05-13,2020-05-13|2020-05-14,2020-05-14|2020-05-15,2020-05-15|2020-05-16,2020-05-16|2020-05-17,2020-05-17|2020-05-18,2020-05-18' + . '|2020-05-19,2020-05-19|2020-05-20,2020-05-20|2020-05-21,2020-05-21|2020-05-22,2020-05-22|2020-05-23,2020-05-23|2020-05-24,2020-05-24' + . '|2020-05-25,2020-05-25|2020-05-26,2020-05-26|2020-05-27,2020-05-27|2020-05-28,2020-05-28|2020-05-29,2020-05-29|2020-05-30,2020-05-30' + . '|2020-05-31,2020-05-31|2020-06-01,2020-06-01|2020-06-02,2020-06-02|2020-06-03,2020-06-03|2020-06-04,2020-06-04|2020-06-05,2020-06-05' + . '|2020-06-06,2020-06-06|2020-06-07,2020-06-07|2020-06-08,2020-06-08|2020-06-09,2020-06-09|2020-06-10,2020-06-10|2020-06-11,2020-06-11' + . '|2020-06-12,2020-06-12|2020-06-13,2020-06-13|2020-06-14,2020-06-14', + 'count' => '45', + ), + array ( + 'idsite' => '1', + 'period' => '2', + 'name' => 'done.VisitsSummary', + 'report' => NULL, + 'dates' => '2020-05-04,2020-05-10|2020-05-11,2020-05-17|2020-05-18,2020-05-24|2020-05-25,2020-05-31|2020-04-27,2020-05-03|2020-06-01,2020-06-07' + . '|2020-06-08,2020-06-14', + 'count' => '7', + ), + array ( + 'idsite' => '1', + 'period' => '3', + 'name' => 'done.VisitsSummary', + 'report' => NULL, + 'dates' => '2020-05-01,2020-05-31|2020-06-01,2020-06-30', + 'count' => '2', + ), + array ( + 'idsite' => '1', + 'period' => '4', + 'name' => 'done.VisitsSummary', + 'report' => NULL, + 'dates' => '2020-01-01,2020-12-31', + 'count' => '1', + ), + ]; + + $actualInvalidations = $this->getInvalidatedArchiveTableEntriesSummary(); + + $this->assertEquals($expectedInvalidations, $actualInvalidations); + } + + public function test_reArchiveReport_createsCorrectInvalidationEntries_ifReportSpecified() + { + Date::$now = strtotime('2020-06-16 12:00:00'); + + Config::getInstance()->General['rearchive_reports_in_past_last_n_months'] = 'last1'; + + $this->invalidator->reArchiveReport([1], 'VisitsSummary', 'some.Report'); + + $expectedInvalidations = [ + array ( + 'idsite' => '1', + 'period' => '1', + 'name' => 'done.VisitsSummary', + 'report' => 'some.Report', + 'dates' => '2020-05-01,2020-05-01|2020-05-02,2020-05-02|2020-05-03,2020-05-03|2020-05-04,2020-05-04|2020-05-05,2020-05-05' + . '|2020-05-06,2020-05-06|2020-05-07,2020-05-07|2020-05-08,2020-05-08|2020-05-09,2020-05-09|2020-05-10,2020-05-10' + . '|2020-05-11,2020-05-11|2020-05-12,2020-05-12|2020-05-13,2020-05-13|2020-05-14,2020-05-14|2020-05-15,2020-05-15' + . '|2020-05-16,2020-05-16|2020-05-17,2020-05-17|2020-05-18,2020-05-18|2020-05-19,2020-05-19|2020-05-20,2020-05-20' + . '|2020-05-21,2020-05-21|2020-05-22,2020-05-22|2020-05-23,2020-05-23|2020-05-24,2020-05-24|2020-05-25,2020-05-25' + . '|2020-05-26,2020-05-26|2020-05-27,2020-05-27|2020-05-28,2020-05-28|2020-05-29,2020-05-29|2020-05-30,2020-05-30' + . '|2020-05-31,2020-05-31|2020-06-01,2020-06-01|2020-06-02,2020-06-02|2020-06-03,2020-06-03|2020-06-04,2020-06-04' + . '|2020-06-05,2020-06-05|2020-06-06,2020-06-06|2020-06-07,2020-06-07|2020-06-08,2020-06-08|2020-06-09,2020-06-09' + . '|2020-06-10,2020-06-10|2020-06-11,2020-06-11|2020-06-12,2020-06-12|2020-06-13,2020-06-13|2020-06-14,2020-06-14', + 'count' => '45', + ), + array ( + 'idsite' => '1', + 'period' => '2', + 'name' => 'done.VisitsSummary', + 'report' => 'some.Report', + 'dates' => '2020-05-04,2020-05-10|2020-05-11,2020-05-17|2020-05-18,2020-05-24|2020-05-25,2020-05-31|2020-04-27,2020-05-03' + . '|2020-06-01,2020-06-07|2020-06-08,2020-06-14', + 'count' => '7', + ), + array ( + 'idsite' => '1', + 'period' => '3', + 'name' => 'done.VisitsSummary', + 'report' => 'some.Report', + 'dates' => '2020-05-01,2020-05-31|2020-06-01,2020-06-30', + 'count' => '2', + ), + array ( + 'idsite' => '1', + 'period' => '4', + 'name' => 'done.VisitsSummary', + 'report' => 'some.Report', + 'dates' => '2020-01-01,2020-12-31', + 'count' => '1', + ), + ]; + + $actualInvalidations = $this->getInvalidatedArchiveTableEntriesSummary(); + + $this->assertEquals($expectedInvalidations, $actualInvalidations); + } + + public function test_reArchive_alsoInvalidatesSegments() + { + Date::$now = strtotime('2020-06-16 12:00:00'); + + Config::getInstance()->General['rearchive_reports_in_past_last_n_months'] = 'last2'; + Config::getInstance()->General['process_new_segments_from'] = 'beginning_of_time'; + + $idSite = Fixture::createWebsite(Date::today()->subMonth(1)->getDatetime()); + + $t = Fixture::getTracker($idSite, '2020-05-04 03:45:45'); + $t->setUrl('http://test.com/test'); + Fixture::checkResponse($t->doTrackPageView('test page')); + + API::getInstance()->add('autoArchiveSegment', 'browserCode==IE', false, true); + API::getInstance()->add('browserArchiveSegment', 'browserCode==IE', false, false); + + $this->invalidator->reArchiveReport([$idSite], 'VisitsSummary', 'some.Report'); + + $expectedInvalidations = [ + array ( + 'idsite' => '11', + 'period' => '1', + 'name' => 'done.VisitsSummary', + 'report' => 'some.Report', + 'dates' => '2020-04-30,2020-04-30|2020-05-01,2020-05-01|2020-05-02,2020-05-02|2020-05-03,2020-05-03|2020-05-04,2020-05-04|2020-05-05,2020-05-05|2020-05-06,2020-05-06|2020-05-07,2020-05-07|2020-05-08,2020-05-08|2020-05-09,2020-05-09|2020-05-10,2020-05-10|2020-05-11,2020-05-11|2020-05-12,2020-05-12|2020-05-13,2020-05-13|2020-05-14,2020-05-14|2020-05-15,2020-05-15|2020-05-16,2020-05-16|2020-05-17,2020-05-17|2020-05-18,2020-05-18|2020-05-19,2020-05-19|2020-05-20,2020-05-20|2020-05-21,2020-05-21|2020-05-22,2020-05-22|2020-05-23,2020-05-23|2020-05-24,2020-05-24|2020-05-25,2020-05-25|2020-05-26,2020-05-26|2020-05-27,2020-05-27|2020-05-28,2020-05-28|2020-05-29,2020-05-29|2020-05-30,2020-05-30|2020-05-31,2020-05-31|2020-06-01,2020-06-01|2020-06-02,2020-06-02|2020-06-03,2020-06-03|2020-06-04,2020-06-04|2020-06-05,2020-06-05|2020-06-06,2020-06-06|2020-06-07,2020-06-07|2020-06-08,2020-06-08|2020-06-09,2020-06-09|2020-06-10,2020-06-10|2020-06-11,2020-06-11|2020-06-12,2020-06-12|2020-06-13,2020-06-13|2020-06-14,2020-06-14', + 'count' => '46', + ), + array ( + 'idsite' => '11', + 'period' => '1', + 'name' => 'done5f4f9bafeda3443c3c2d4b2ef4dffadc.VisitsSummary', + 'report' => 'some.Report', + 'dates' => '2020-04-30,2020-04-30|2020-05-01,2020-05-01|2020-05-02,2020-05-02|2020-05-03,2020-05-03|2020-05-04,2020-05-04|2020-05-05,2020-05-05|2020-05-06,2020-05-06|2020-05-07,2020-05-07|2020-05-08,2020-05-08|2020-05-09,2020-05-09|2020-05-10,2020-05-10|2020-05-11,2020-05-11|2020-05-12,2020-05-12|2020-05-13,2020-05-13|2020-05-14,2020-05-14|2020-05-15,2020-05-15|2020-05-16,2020-05-16|2020-05-17,2020-05-17|2020-05-18,2020-05-18|2020-05-19,2020-05-19|2020-05-20,2020-05-20|2020-05-21,2020-05-21|2020-05-22,2020-05-22|2020-05-23,2020-05-23|2020-05-24,2020-05-24|2020-05-25,2020-05-25|2020-05-26,2020-05-26|2020-05-27,2020-05-27|2020-05-28,2020-05-28|2020-05-29,2020-05-29|2020-05-30,2020-05-30|2020-05-31,2020-05-31|2020-06-01,2020-06-01|2020-06-02,2020-06-02|2020-06-03,2020-06-03|2020-06-04,2020-06-04|2020-06-05,2020-06-05|2020-06-06,2020-06-06|2020-06-07,2020-06-07|2020-06-08,2020-06-08|2020-06-09,2020-06-09|2020-06-10,2020-06-10|2020-06-11,2020-06-11|2020-06-12,2020-06-12|2020-06-13,2020-06-13|2020-06-14,2020-06-14', + 'count' => '46', + ), + array ( + 'idsite' => '11', + 'period' => '2', + 'name' => 'done.VisitsSummary', + 'report' => 'some.Report', + 'dates' => '2020-04-27,2020-05-03|2020-05-04,2020-05-10|2020-05-11,2020-05-17|2020-05-18,2020-05-24|2020-05-25,2020-05-31|2020-06-01,2020-06-07|2020-06-08,2020-06-14', + 'count' => '7', + ), + array ( + 'idsite' => '11', + 'period' => '2', + 'name' => 'done5f4f9bafeda3443c3c2d4b2ef4dffadc.VisitsSummary', + 'report' => 'some.Report', + 'dates' => '2020-04-27,2020-05-03|2020-05-04,2020-05-10|2020-05-11,2020-05-17|2020-05-18,2020-05-24|2020-05-25,2020-05-31|2020-06-01,2020-06-07|2020-06-08,2020-06-14', + 'count' => '7', + ), + array ( + 'idsite' => '11', + 'period' => '3', + 'name' => 'done.VisitsSummary', + 'report' => 'some.Report', + 'dates' => '2020-04-01,2020-04-30|2020-05-01,2020-05-31|2020-06-01,2020-06-30', + 'count' => '3', + ), + array ( + 'idsite' => '11', + 'period' => '3', + 'name' => 'done5f4f9bafeda3443c3c2d4b2ef4dffadc.VisitsSummary', + 'report' => 'some.Report', + 'dates' => '2020-04-01,2020-04-30|2020-05-01,2020-05-31|2020-06-01,2020-06-30', + 'count' => '3', + ), + array ( + 'idsite' => '11', + 'period' => '4', + 'name' => 'done.VisitsSummary', + 'report' => 'some.Report', + 'dates' => '2020-01-01,2020-12-31', + 'count' => '1', + ), + array ( + 'idsite' => '11', + 'period' => '4', + 'name' => 'done5f4f9bafeda3443c3c2d4b2ef4dffadc.VisitsSummary', + 'report' => 'some.Report', + 'dates' => '2020-01-01,2020-12-31', + 'count' => '1', + ), + ]; + + $actualInvalidations = $this->getInvalidatedArchiveTableEntriesSummary(); + + $this->assertEquals($expectedInvalidations, $actualInvalidations); + } + private function getInvalidatedIdArchives() { $result = array(); @@ -978,9 +1250,9 @@ class ArchiveInvalidatorTest extends IntegrationTestCase } $rangePeriods = array( - '2015-03-04,2015-03-05', - '2014-12-05,2015-01-01', - '2015-03-05,2015-03-10', + '2015-03-04,2015-03-05', + '2014-12-05,2015-01-01', + '2015-03-05,2015-03-10', '2015-01-01,2015-01-10', '2014-10-15,2014-10-20' ); @@ -1021,7 +1293,7 @@ class ArchiveInvalidatorTest extends IntegrationTestCase private function getInvalidatedArchiveTableEntries() { - return Db::fetchAll("SELECT idarchive, idsite, date1, date2, period, name FROM " . Common::prefixTable('archive_invalidations')); + return Db::fetchAll("SELECT idarchive, idsite, date1, date2, period, name, report FROM " . Common::prefixTable('archive_invalidations')); } private function assertEqualsSorted(array $expectedEntries, array $invalidatedArchiveTableEntries) @@ -1038,4 +1310,28 @@ class ArchiveInvalidatorTest extends IntegrationTestCase return strcmp(json_encode($lhs), json_encode($rhs)); }); } + + private function getInvalidatedArchiveTableEntriesSummary() + { + $table = Common::prefixTable('archive_invalidations'); + return Db::fetchAll("SELECT idsite, period, name, report, GROUP_CONCAT(CONCAT(date1, ',', date2) SEPARATOR '|') as dates, COUNT(*) as count FROM $table GROUP BY idsite, period, name, report"); + } + + private static function addVisitToEachSite() + { + $t = Fixture::getTracker(1, '2012-04-05 00:00:00'); + $t->enableBulkTracking(); + for ($i = 0; $i < 10; ++$i) { + $t->setIdSite($i + 1); + $t->setUrl('http://test.com'); + self::assertTrue($t->doTrackPageView('test page')); + } + Fixture::checkBulkTrackingResponse($t->doBulkTrack()); + } + + protected static function configureFixture($fixture) + { + parent::configureFixture($fixture); + $fixture->createSuperUser = true; + } } diff --git a/tests/PHPUnit/Integration/DataAccess/ArchiveSelectorTest.php b/tests/PHPUnit/Integration/DataAccess/ArchiveSelectorTest.php index 408ba7910a..6eaf1d9a8a 100644 --- a/tests/PHPUnit/Integration/DataAccess/ArchiveSelectorTest.php +++ b/tests/PHPUnit/Integration/DataAccess/ArchiveSelectorTest.php @@ -52,8 +52,13 @@ class ArchiveSelectorTest extends IntegrationTestCase private function insertArchiveData($archiveRows) { - $table = ArchiveTableCreator::getNumericTable(Date::factory('2019-10-01 12:13:14')); foreach ($archiveRows as $row) { + if (!empty($row['is_blob_data'])) { + $row['value'] = gzcompress($row['value']); + } + + $d = Date::factory($row['date1']); + $table = !empty($row['is_blob_data']) ? ArchiveTableCreator::getBlobTable($d) : ArchiveTableCreator::getNumericTable($d); $tsArchived = isset($row['ts_archived']) ? $row['ts_archived'] : Date::now()->getDatetime(); Db::query("INSERT INTO `$table` (idarchive, idsite, period, date1, date2, `name`, `value`, ts_archived) VALUES (?, ?, ?, ?, ?, ?, ?, ?)", [$row['idarchive'], $row['idsite'], $row['period'], $row['date1'], $row['date2'], $row['name'], $row['value'], $tsArchived]); @@ -143,7 +148,7 @@ class ArchiveSelectorTest extends IntegrationTestCase '', $minDateProcessed, false, - [false, 0, 0, true], + [false, false, false, true], ], // archive is too old @@ -201,7 +206,7 @@ class ArchiveSelectorTest extends IntegrationTestCase '', $minDateProcessed, false, - [1, false, false, true], + [[1], 0, 0, true], ], [ [ @@ -212,7 +217,218 @@ class ArchiveSelectorTest extends IntegrationTestCase '', $minDateProcessed, false, - [1, 5, 10, true], + [[1], 5, 10, true], + ], + ]; + } + + /** + * @dataProvider getTestDataForGetArchiveData + */ + public function test_getArchiveData_returnsCorrectData($archiveRows, $dataType, $idArchives, $recordNames, $idSubtable, + $expectedData) + { + Fixture::createWebsite('2010-02-02 00:00:00'); + + $this->insertArchiveData($archiveRows); + + $data = ArchiveSelector::getArchiveData($idArchives, $recordNames, $dataType, $idSubtable); + + $this->assertEquals($expectedData, $data); + } + + public function getTestDataForGetArchiveData() + { + // $blobArray1 + $blobArray1 = [ + 1 => 'blobvalue1', + 2 => 'blobvalue2', + 3 => 'blobvalue3', + ]; + $blobArray2 = [ + 1 => 'blobvalue4', + 2 => 'blobvalue5', + 3 => 'blobvalue6', + ]; + $blobArray3 = [ + 1 => 'blobvalue7', + 2 => 'blobvalue8', + 4 => 'blobvalue9', + ]; + $blobArray4 = [ + 1 => 'blobvalue10', + 2 => 'blobvalue11', + 3 => 'blobvalue12', + ]; + + return [ + // numeric data + [ + [ + ['idarchive' => 1, 'idsite' => 1, 'period' => 1, 'date1' => '2019-10-05', 'date2' => '2019-10-05', 'name' => 'nb_visits', 'value' => 5, 'ts_archived' => '2020-06-13 09:04:56'], + ['idarchive' => 1, 'idsite' => 1, 'period' => 1, 'date1' => '2019-10-05', 'date2' => '2019-10-05', 'name' => 'nb_visits_converted', 'value' => 10, 'ts_archived' => '2020-06-12 02:04:56'], + ['idarchive' => 2, 'idsite' => 1, 'period' => 1, 'date1' => '2019-10-05', 'date2' => '2019-10-05', 'name' => 'nb_visits', 'value' => 15, 'ts_archived' => '2020-06-13 04:04:56'], + ['idarchive' => 2, 'idsite' => 1, 'period' => 1, 'date1' => '2019-10-05', 'date2' => '2019-10-05', 'name' => 'nb_visits_converted', 'value' => 20, 'ts_archived' => '2020-06-13 04:04:56'], + ['idarchive' => 3, 'idsite' => 1, 'period' => 1, 'date1' => '2019-10-05', 'date2' => '2019-10-05', 'name' => 'nb_visits', 'value' => 30, 'ts_archived' => '2020-06-13 04:04:56'], + ], + 'numeric', + [ + '2019-10-05,2019-10-05' => [1,2,3], + ], + ['nb_visits', 'nb_visits_converted'], + null, + array ( + array ( + 'value' => '10', + 'name' => 'nb_visits_converted', + 'idsite' => '1', + 'date1' => '2019-10-05', + 'date2' => '2019-10-05', + 'ts_archived' => '2020-06-12 02:04:56', + ), + array ( + 'value' => '15', + 'name' => 'nb_visits', + 'idsite' => '1', + 'date1' => '2019-10-05', + 'date2' => '2019-10-05', + 'ts_archived' => '2020-06-13 04:04:56', + ), + array ( + 'value' => '20', + 'name' => 'nb_visits_converted', + 'idsite' => '1', + 'date1' => '2019-10-05', + 'date2' => '2019-10-05', + 'ts_archived' => '2020-06-13 04:04:56', + ), + array ( + 'value' => '30', + 'name' => 'nb_visits', + 'idsite' => '1', + 'date1' => '2019-10-05', + 'date2' => '2019-10-05', + 'ts_archived' => '2020-06-13 04:04:56', + ), + array ( + 'value' => '5', + 'name' => 'nb_visits', + 'idsite' => '1', + 'date1' => '2019-10-05', + 'date2' => '2019-10-05', + 'ts_archived' => '2020-06-13 09:04:56', + ), + ), + ], + + // blob data + [ + [ + ['idarchive' => 1, 'idsite' => 1, 'period' => 1, 'date1' => '2019-10-05', 'date2' => '2019-10-05', 'name' => 'blob1', 'value' => 'nop', 'ts_archived' => '2020-06-13 09:04:56', 'is_blob_data' => true], + ['idarchive' => 1, 'idsite' => 1, 'period' => 1, 'date1' => '2019-10-05', 'date2' => '2019-10-05', 'name' => 'blob2', 'value' => 'klm', 'ts_archived' => '2020-06-12 02:04:56', 'is_blob_data' => true], + ['idarchive' => 2, 'idsite' => 1, 'period' => 1, 'date1' => '2019-10-05', 'date2' => '2019-10-05', 'name' => 'blob1', 'value' => 'hij', 'ts_archived' => '2020-06-13 04:04:56', 'is_blob_data' => true], + ['idarchive' => 2, 'idsite' => 1, 'period' => 1, 'date1' => '2019-10-05', 'date2' => '2019-10-05', 'name' => 'blob2', 'value' => 'ghi', 'ts_archived' => '2020-06-13 04:04:56', 'is_blob_data' => true], + ['idarchive' => 3, 'idsite' => 1, 'period' => 1, 'date1' => '2019-10-05', 'date2' => '2019-10-05', 'name' => 'blob2', 'value' => 'abcd', 'ts_archived' => '2020-06-13 04:04:56', 'is_blob_data' => true], + ['idarchive' => 4, 'idsite' => 1, 'period' => 1, 'date1' => '2019-10-05', 'date2' => '2019-10-05', 'name' => 'blob2', 'value' => 'abcd', 'ts_archived' => '2020-08-13 04:04:56', 'is_blob_data' => true], + ], + 'blob', + [ + '2019-10-05,2019-10-05' => [1,2,3], + ], + ['blob1', 'blob2'], + null, + array ( + array ( + 'value' => 'klm', + 'name' => 'blob2', + 'idsite' => '1', + 'date1' => '2019-10-05', + 'date2' => '2019-10-05', + 'ts_archived' => '2020-06-12 02:04:56', + ), + array ( + 'value' => 'hij', + 'name' => 'blob1', + 'idsite' => '1', + 'date1' => '2019-10-05', + 'date2' => '2019-10-05', + 'ts_archived' => '2020-06-13 04:04:56', + ), + array ( + 'value' => 'ghi', + 'name' => 'blob2', + 'idsite' => '1', + 'date1' => '2019-10-05', + 'date2' => '2019-10-05', + 'ts_archived' => '2020-06-13 04:04:56', + ), + array ( + 'value' => 'abcd', + 'name' => 'blob2', + 'idsite' => '1', + 'date1' => '2019-10-05', + 'date2' => '2019-10-05', + 'ts_archived' => '2020-06-13 04:04:56', + ), + array ( + 'value' => 'nop', + 'name' => 'blob1', + 'idsite' => '1', + 'date1' => '2019-10-05', + 'date2' => '2019-10-05', + 'ts_archived' => '2020-06-13 09:04:56', + ), + ), + ], + + // blub data w/ subtable + [ + [ + ['idarchive' => 1, 'idsite' => 1, 'period' => 1, 'date1' => '2019-10-05', 'date2' => '2019-10-05', 'name' => 'blob1_chunk_0_99', 'value' => serialize($blobArray1), 'ts_archived' => '2020-06-13 09:04:56', 'is_blob_data' => true], + ['idarchive' => 1, 'idsite' => 1, 'period' => 1, 'date1' => '2019-10-05', 'date2' => '2019-10-05', 'name' => 'blob2_chunk_0_99', 'value' => serialize($blobArray2), 'ts_archived' => '2020-06-12 02:04:56', 'is_blob_data' => true], + ['idarchive' => 2, 'idsite' => 1, 'period' => 1, 'date1' => '2019-10-05', 'date2' => '2019-10-05', 'name' => 'blob1_chunk_0_99', 'value' => serialize($blobArray3), 'ts_archived' => '2020-06-13 04:04:56', 'is_blob_data' => true], + ['idarchive' => 2, 'idsite' => 1, 'period' => 1, 'date1' => '2019-10-05', 'date2' => '2019-10-05', 'name' => 'blob2_chunk_0_99', 'value' => serialize($blobArray4), 'ts_archived' => '2020-06-13 04:04:56', 'is_blob_data' => true], + ], + 'blob', + [ + '2019-10-05,2019-10-05' => [1,2,3], + ], + ['blob1', 'blob2'], + 2, + array ( + array ( + 'value' => 'blobvalue5', + 'name' => 'blob2_2', + 'idsite' => '1', + 'date1' => '2019-10-05', + 'date2' => '2019-10-05', + 'ts_archived' => '2020-06-12 02:04:56', + ), + array ( + 'value' => 'blobvalue8', + 'name' => 'blob1_2', + 'idsite' => '1', + 'date1' => '2019-10-05', + 'date2' => '2019-10-05', + 'ts_archived' => '2020-06-13 04:04:56', + ), + array ( + 'value' => 'blobvalue11', + 'name' => 'blob2_2', + 'idsite' => '1', + 'date1' => '2019-10-05', + 'date2' => '2019-10-05', + 'ts_archived' => '2020-06-13 04:04:56', + ), + array ( + 'value' => 'blobvalue2', + 'name' => 'blob1_2', + 'idsite' => '1', + 'date1' => '2019-10-05', + 'date2' => '2019-10-05', + 'ts_archived' => '2020-06-13 09:04:56', + ), + ), ], ]; } diff --git a/tests/PHPUnit/Integration/DataAccess/ArchiveWriterTest.php b/tests/PHPUnit/Integration/DataAccess/ArchiveWriterTest.php index d227e6f91e..d8493d14ad 100644 --- a/tests/PHPUnit/Integration/DataAccess/ArchiveWriterTest.php +++ b/tests/PHPUnit/Integration/DataAccess/ArchiveWriterTest.php @@ -18,9 +18,10 @@ use Piwik\Db; use Piwik\Period\Day; use Piwik\Period\Factory as PeriodFactory; use Piwik\Segment; +use Piwik\Sequence; use Piwik\Site; use Piwik\Tests\Framework\Fixture; -use Piwik\Tests\Integration\Settings\IntegrationTestCase; +use Piwik\Tests\Framework\TestCase\IntegrationTestCase; class TestArchiveWriter extends ArchiveWriter { public function flushSpools() @@ -40,24 +41,70 @@ class ArchiveWriterTest extends IntegrationTestCase public function setUp(): void { + parent::setUp(); + Access::getInstance()->setSuperUserAccess(true); $this->idSite = Fixture::createWebsite('2019-08-29'); } - public function test_initNewArchive_doesNotWiteNewArchiveStatusToFileRightAway() + public function test_finalizeArchive_removesOldArchivesIfNotPartial() { + Date::$now = strtotime('2020-04-05 03:00:00'); + $period = 'day'; $date = '2019-08-29'; + $initialArchiveData = [ + ['idarchive' => 1, 'idsite' => $this->idSite, 'date1' => '2019-08-29', 'date2' => '2019-08-29', 'period' => 1, 'name' => 'done', 'value' => ArchiveWriter::DONE_PARTIAL, 'ts_archived' => '2020-02-02 03:44:44'], + ['idarchive' => 2, 'idsite' => $this->idSite, 'date1' => '2019-08-29', 'date2' => '2019-08-29', 'period' => 1, 'name' => 'done', 'value' => ArchiveWriter::DONE_OK, 'ts_archived' => '2020-02-04 03:44:44'], + ]; + + $this->insertArchiveData($initialArchiveData); + $writer = $this->buildWriter($period, $date); $writer->initNewArchive(); + $writer->insertRecord('nb_visits', 5); - $this->assertEquals(array(), $this->getAllNumericRows($date)); + $this->assertEquals($initialArchiveData, $this->getAllColsOfAllNumericRows($date)); - // now we flush and it should be written - $writer->flushSpools(); - $this->assertCount(1, $this->getAllNumericRows($date)); - $this->assertNumericArchiveExists(Day::PERIOD_ID, $date, 'done', ArchiveWriter::DONE_ERROR); + $writer->finalizeArchive(); + + $expected = [ + ['idarchive' => 3, 'idsite' => 1, 'date1' => '2019-08-29', 'date2' => '2019-08-29', 'period' => 1, 'name' => 'done', 'value' => 1, 'ts_archived' => '2020-04-05 03:00:00'], + ['idarchive' => 3, 'idsite' => 1, 'date1' => '2019-08-29', 'date2' => '2019-08-29', 'period' => 1, 'name' => 'nb_visits', 'value' => 5, 'ts_archived' => '2020-04-05 03:00:00'], + ]; + $this->assertEquals($expected, $this->getAllColsOfAllNumericRows($date)); + } + + public function test_finalizeArchive_doesNotRemoveOldArchivesIfPartial() + { + Date::$now = strtotime('2020-04-05 03:00:00'); + + $period = 'day'; + $date = '2019-08-29'; + + $initialArchiveData = [ + ['idarchive' => 1, 'idsite' => $this->idSite, 'date1' => '2019-08-29', 'date2' => '2019-08-29', 'period' => 1, 'name' => 'done', 'value' => ArchiveWriter::DONE_OK, 'ts_archived' => '2020-02-02 03:44:44'], + ['idarchive' => 2, 'idsite' => $this->idSite, 'date1' => '2019-08-29', 'date2' => '2019-08-29', 'period' => 1, 'name' => 'done', 'value' => ArchiveWriter::DONE_PARTIAL, 'ts_archived' => '2020-02-04 03:44:44'], + ]; + + $this->insertArchiveData($initialArchiveData); + + $writer = $this->buildWriter($period, $date, $isPartial = true); + $writer->initNewArchive(); + $writer->insertRecord('nb_visits', 5); + + $this->assertEquals($initialArchiveData, $this->getAllColsOfAllNumericRows($date)); + + $writer->finalizeArchive(); + + $expected = [ + ['idarchive' => 1, 'idsite' => $this->idSite, 'date1' => '2019-08-29', 'date2' => '2019-08-29', 'period' => 1, 'name' => 'done', 'value' => ArchiveWriter::DONE_OK, 'ts_archived' => '2020-02-02 03:44:44'], + ['idarchive' => 2, 'idsite' => $this->idSite, 'date1' => '2019-08-29', 'date2' => '2019-08-29', 'period' => 1, 'name' => 'done', 'value' => ArchiveWriter::DONE_PARTIAL, 'ts_archived' => '2020-02-04 03:44:44'], + ['idarchive' => 3, 'idsite' => 1, 'date1' => '2019-08-29', 'date2' => '2019-08-29', 'period' => 1, 'name' => 'done', 'value' => 5, 'ts_archived' => '2020-04-05 03:00:00'], + ['idarchive' => 3, 'idsite' => 1, 'date1' => '2019-08-29', 'date2' => '2019-08-29', 'period' => 1, 'name' => 'nb_visits', 'value' => 5, 'ts_archived' => '2020-04-05 03:00:00'], + ]; + $this->assertEquals($expected, $this->getAllColsOfAllNumericRows($date)); } public function test_finaliseArchive_writesArchiveStatusToFile() @@ -72,6 +119,22 @@ class ArchiveWriterTest extends IntegrationTestCase $this->assertNumericArchiveExists(Day::PERIOD_ID, $date, 'done', ArchiveWriter::DONE_OK); } + public function test_initNewArchive_doesNotWiteNewArchiveStatusToFileRightAway() + { + $period = 'day'; + $date = '2019-08-29'; + + $writer = $this->buildWriter($period, $date); + $writer->initNewArchive(); + + $this->assertEquals(array(), $this->getAllNumericRows($date)); + + // now we flush and it should be written + $writer->flushSpools(); + $this->assertCount(1, $this->getAllNumericRows($date)); + $this->assertNumericArchiveExists(Day::PERIOD_ID, $date, 'done', ArchiveWriter::DONE_ERROR); + } + public function test_insertRecord_notFlushedUntilFinaliseCalled() { $period = 'day'; @@ -163,11 +226,15 @@ class ArchiveWriterTest extends IntegrationTestCase } } - private function buildWriter($period, $date) + private function buildWriter($period, $date, $isPartial = false) { $oPeriod = PeriodFactory::makePeriodFromQueryParams('UTC', $period, $date); $segment = new Segment('', []); $params = new Parameters(new Site($this->idSite), $oPeriod, $segment); + if ($isPartial) { + $params->setRequestedPlugin('ExamplePlugin'); + $params->setIsPartialArchive(true); + } $writer = new TestArchiveWriter($params); return $writer; } @@ -218,4 +285,34 @@ class ArchiveWriterTest extends IntegrationTestCase $result = Db::get()->query($sql); return $result->fetch(); } + + private function insertArchiveData($archiveRows) + { + if (!empty($archiveRows)) { + $idarchives = array_column($archiveRows, 'idarchive'); + $max = max($idarchives); + + $d = Date::factory($archiveRows[0]['date1']); + $tableName = Common::prefixTable('archive_numeric_' . $d->toString('Y_m')); + $seq = new Sequence($tableName); + $seq->create($max); + } + + foreach ($archiveRows as $row) { + $d = Date::factory($row['date1']); + $table = ArchiveTableCreator::getNumericTable($d); + $tsArchived = isset($row['ts_archived']) ? $row['ts_archived'] : Date::now()->getDatetime(); + + Db::query("INSERT INTO `$table` (idarchive, idsite, period, date1, date2, `name`, `value`, ts_archived) VALUES (?, ?, ?, ?, ?, ?, ?, ?)", + [$row['idarchive'], $row['idsite'], $row['period'], $row['date1'], $row['date2'], $row['name'], $row['value'], $tsArchived]); + } + } + + private function getAllColsOfAllNumericRows(string $date) + { + $archiveTableName = ArchiveTableCreator::getNumericTable(Date::factory($date)); + $sql = 'SELECT idarchive, idsite, date1, date2, period, name, value, ts_archived FROM ' . $archiveTableName; + + return Db::fetchAll($sql); + } } diff --git a/tests/PHPUnit/Integration/DataAccess/ModelTest.php b/tests/PHPUnit/Integration/DataAccess/ModelTest.php index b2377bbf8d..42f67fea92 100644 --- a/tests/PHPUnit/Integration/DataAccess/ModelTest.php +++ b/tests/PHPUnit/Integration/DataAccess/ModelTest.php @@ -263,6 +263,7 @@ class ModelTest extends IntegrationTestCase 'date2' => '2015-04-24', 'period' => '5', 'name' => 'done', + 'report' => null, ), array ( 'idinvalidation' => '12', @@ -272,6 +273,7 @@ class ModelTest extends IntegrationTestCase 'date2' => '2015-04-06', 'period' => '1', 'name' => 'done', + 'report' => null, ), array ( 'idinvalidation' => '13', @@ -281,6 +283,7 @@ class ModelTest extends IntegrationTestCase 'date2' => '2015-04-06', 'period' => '1', 'name' => 'done764644a7142bdcbedaab92f9dedef5e5', + 'report' => null, ), array ( 'idinvalidation' => '19', @@ -290,6 +293,7 @@ class ModelTest extends IntegrationTestCase 'date2' => '2015-04-12', 'period' => '2', 'name' => 'done', + 'report' => null, ), array ( 'idinvalidation' => '5', @@ -299,6 +303,7 @@ class ModelTest extends IntegrationTestCase 'date2' => '2015-04-12', 'period' => '2', 'name' => 'done764644a7142bdcbedaab92f9dedef5e5', + 'report' => null, ), array ( 'idinvalidation' => '15', @@ -308,6 +313,7 @@ class ModelTest extends IntegrationTestCase 'date2' => '2015-04-05', 'period' => '1', 'name' => 'done', + 'report' => null, ), array ( 'idinvalidation' => '8', @@ -317,6 +323,7 @@ class ModelTest extends IntegrationTestCase 'date2' => '2015-04-04', 'period' => '1', 'name' => 'done', + 'report' => null, ), array ( 'idinvalidation' => '14', @@ -326,6 +333,7 @@ class ModelTest extends IntegrationTestCase 'date2' => '2015-04-03', 'period' => '1', 'name' => 'done', + 'report' => null, ), array ( 'idinvalidation' => '20', @@ -335,6 +343,7 @@ class ModelTest extends IntegrationTestCase 'date2' => '2015-04-02', 'period' => '1', 'name' => 'done', + 'report' => null, ), array ( 'idinvalidation' => '3', @@ -344,6 +353,7 @@ class ModelTest extends IntegrationTestCase 'date2' => '2015-04-02', 'period' => '1', 'name' => 'done67564f109e3f4bba6b185a5343ff2bb0', + 'report' => null, ), array ( 'idinvalidation' => '2', @@ -353,6 +363,7 @@ class ModelTest extends IntegrationTestCase 'date2' => '2015-04-01', 'period' => '1', 'name' => 'done', + 'report' => null, ), array ( 'idinvalidation' => '10', @@ -362,6 +373,7 @@ class ModelTest extends IntegrationTestCase 'date2' => '2015-04-30', 'period' => '3', 'name' => 'done', + 'report' => null, ), array ( 'idinvalidation' => '17', @@ -371,6 +383,7 @@ class ModelTest extends IntegrationTestCase 'date2' => '2015-04-30', 'period' => '3', 'name' => 'done67564f109e3f4bba6b185a5343ff2bb0', + 'report' => null, ), array ( 'idinvalidation' => '22', @@ -380,6 +393,7 @@ class ModelTest extends IntegrationTestCase 'date2' => '2015-03-31', 'period' => '1', 'name' => 'done', + 'report' => null, ), array ( 'idinvalidation' => '7', @@ -389,6 +403,7 @@ class ModelTest extends IntegrationTestCase 'date2' => '2015-03-30', 'period' => '1', 'name' => 'done', + 'report' => null, ), array ( 'idinvalidation' => '1', @@ -398,6 +413,7 @@ class ModelTest extends IntegrationTestCase 'date2' => '2015-03-30', 'period' => '1', 'name' => 'done0bb102ea2ac682a578480dd184736607', + 'report' => null, ), array ( 'idinvalidation' => '16', @@ -407,6 +423,7 @@ class ModelTest extends IntegrationTestCase 'date2' => '2015-04-05', 'period' => '2', 'name' => 'done', + 'report' => null, ), array ( 'idinvalidation' => '6', @@ -416,6 +433,7 @@ class ModelTest extends IntegrationTestCase 'date2' => '2015-03-29', 'period' => '1', 'name' => 'done', + 'report' => null, ), array ( 'idinvalidation' => '9', @@ -425,6 +443,7 @@ class ModelTest extends IntegrationTestCase 'date2' => '2015-03-29', 'period' => '1', 'name' => 'doneb321434abb5a139c17dadf08c9d2e315', + 'report' => null, ), array ( 'idinvalidation' => '18', @@ -434,6 +453,7 @@ class ModelTest extends IntegrationTestCase 'date2' => '2015-03-24', 'period' => '1', 'name' => 'done', + 'report' => null, ), array ( 'idinvalidation' => '21', @@ -443,6 +463,7 @@ class ModelTest extends IntegrationTestCase 'date2' => '2015-03-31', 'period' => '3', 'name' => 'done', + 'report' => null, ), array ( 'idinvalidation' => '4', @@ -452,6 +473,7 @@ class ModelTest extends IntegrationTestCase 'date2' => '2015-12-31', 'period' => '4', 'name' => 'done', + 'report' => null, ), ); diff --git a/tests/PHPUnit/System/OneVisitorOneWebsiteSeveralDaysDateRangeArchivingTest.php b/tests/PHPUnit/System/OneVisitorOneWebsiteSeveralDaysDateRangeArchivingTest.php index efce6d829c..f5be1dd491 100644 --- a/tests/PHPUnit/System/OneVisitorOneWebsiteSeveralDaysDateRangeArchivingTest.php +++ b/tests/PHPUnit/System/OneVisitorOneWebsiteSeveralDaysDateRangeArchivingTest.php @@ -121,9 +121,9 @@ class OneVisitorOneWebsiteSeveralDaysDateRangeArchivingTest extends SystemTestCa $expectedActionsBlobsWhenFlattened = $expectedActionsBlobs + 1; $tests = array( - 'archive_blob_2010_12' => ( ($expectedActionsBlobs+1) /*Actions*/ + 'archive_blob_2010_12' => ( ($expectedActionsBlobs+3) /*Actions*/ + 2 /* Resolution */ - + 2 /* VisitTime */) * 3, + + 2 /* VisitTime */) * 3 + 1, /** * In Each "Period=range" Archive, we expect following non zero numeric entries: @@ -217,6 +217,12 @@ class OneVisitorOneWebsiteSeveralDaysDateRangeArchivingTest extends SystemTestCa protected function printDebugWhenTestFails($table) { $data = Db::get()->fetchAll("SELECT * FROM " . Common::prefixTable($table) . " WHERE period = " . Piwik::$idPeriods['range'] . " ORDER BY idarchive ASC"); + if (strpos($table, 'blob') !== false) { + $data = array_map(function ($r) { + unset($r['value']); + return $r; + }, $data); + } var_export($data); $idArchives = array(); diff --git a/tests/PHPUnit/System/PeriodIsRangeDateIsLastNMetadataAndNormalAPITest.php b/tests/PHPUnit/System/PeriodIsRangeDateIsLastNMetadataAndNormalAPITest.php index d805428d19..5b0f56b109 100644 --- a/tests/PHPUnit/System/PeriodIsRangeDateIsLastNMetadataAndNormalAPITest.php +++ b/tests/PHPUnit/System/PeriodIsRangeDateIsLastNMetadataAndNormalAPITest.php @@ -27,6 +27,13 @@ class PeriodIsRangeDateIsLastNMetadataAndNormalAPITest extends SystemTestCase parent::setUpBeforeClass(); } + protected function setUp(): void + { + // test results change based on time of day for some reason + Date::$now = strtotime(date('Y-m-d') . ' 20:00:00'); + parent::setUp(); + } + /** * @dataProvider getApiForTesting */ diff --git a/tests/PHPUnit/System/expected/test_ImportLogs__ExamplePlugin.getExampleArchivedMetric_month.xml b/tests/PHPUnit/System/expected/test_ImportLogs__ExamplePlugin.getExampleArchivedMetric_month.xml new file mode 100644 index 0000000000..043911003e --- /dev/null +++ b/tests/PHPUnit/System/expected/test_ImportLogs__ExamplePlugin.getExampleArchivedMetric_month.xml @@ -0,0 +1,5 @@ +<?xml version="1.0" encoding="utf-8" ?> +<result> + <ExamplePlugin_example_metric>12814</ExamplePlugin_example_metric> + <ExamplePlugin_example_metric2>520</ExamplePlugin_example_metric2> +</result>
\ No newline at end of file diff --git a/tests/PHPUnit/System/expected/test_OneVisitorTwoVisits__ExamplePlugin.getExampleArchivedMetric_day.xml b/tests/PHPUnit/System/expected/test_OneVisitorTwoVisits__ExamplePlugin.getExampleArchivedMetric_day.xml new file mode 100644 index 0000000000..7422ab593f --- /dev/null +++ b/tests/PHPUnit/System/expected/test_OneVisitorTwoVisits__ExamplePlugin.getExampleArchivedMetric_day.xml @@ -0,0 +1,5 @@ +<?xml version="1.0" encoding="utf-8" ?> +<result> + <ExamplePlugin_example_metric>2316</ExamplePlugin_example_metric> + <ExamplePlugin_example_metric2>55</ExamplePlugin_example_metric2> +</result>
\ No newline at end of file diff --git a/tests/PHPUnit/System/expected/test_noVisit_PeriodIsLast__ExamplePlugin.getExampleArchivedMetric_day.xml b/tests/PHPUnit/System/expected/test_noVisit_PeriodIsLast__ExamplePlugin.getExampleArchivedMetric_day.xml new file mode 100644 index 0000000000..106f23f16b --- /dev/null +++ b/tests/PHPUnit/System/expected/test_noVisit_PeriodIsLast__ExamplePlugin.getExampleArchivedMetric_day.xml @@ -0,0 +1,10 @@ +<?xml version="1.0" encoding="utf-8" ?> +<results> + <result date="2009-01-04" /> + <result date="2009-01-05" /> + <result date="2009-01-06" /> + <result date="2009-01-07" /> + <result date="2009-01-08" /> + <result date="2009-01-09" /> + <result date="2009-01-10" /> +</results>
\ No newline at end of file diff --git a/tests/PHPUnit/System/expected/test_noVisit_PeriodIsLast__ExamplePlugin.getExampleArchivedMetric_week.xml b/tests/PHPUnit/System/expected/test_noVisit_PeriodIsLast__ExamplePlugin.getExampleArchivedMetric_week.xml new file mode 100644 index 0000000000..867400a47a --- /dev/null +++ b/tests/PHPUnit/System/expected/test_noVisit_PeriodIsLast__ExamplePlugin.getExampleArchivedMetric_week.xml @@ -0,0 +1,10 @@ +<?xml version="1.0" encoding="utf-8" ?> +<results> + <result date="2008-12-29,2009-01-04" /> + <result date="2009-01-05,2009-01-11" /> + <result date="2009-01-12,2009-01-18" /> + <result date="2009-01-19,2009-01-25" /> + <result date="2009-01-26,2009-02-01" /> + <result date="2009-02-02,2009-02-08" /> + <result date="2009-02-09,2009-02-15" /> +</results>
\ No newline at end of file diff --git a/tests/PHPUnit/System/expected/test_noVisit__ExamplePlugin.getExampleArchivedMetric_day.xml b/tests/PHPUnit/System/expected/test_noVisit__ExamplePlugin.getExampleArchivedMetric_day.xml new file mode 100644 index 0000000000..c82a27e763 --- /dev/null +++ b/tests/PHPUnit/System/expected/test_noVisit__ExamplePlugin.getExampleArchivedMetric_day.xml @@ -0,0 +1,5 @@ +<?xml version="1.0" encoding="utf-8" ?> +<result> + <ExamplePlugin_example_metric>0</ExamplePlugin_example_metric> + <ExamplePlugin_example_metric2>0</ExamplePlugin_example_metric2> +</result>
\ No newline at end of file diff --git a/tests/UI/expected-screenshots/UIIntegrationTest_admin_diagnostics_configfile.png b/tests/UI/expected-screenshots/UIIntegrationTest_admin_diagnostics_configfile.png index dfae4a5e04..183d50f357 100644 --- a/tests/UI/expected-screenshots/UIIntegrationTest_admin_diagnostics_configfile.png +++ b/tests/UI/expected-screenshots/UIIntegrationTest_admin_diagnostics_configfile.png @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:ede1813eeb82cc042fc4f8244a8c2ce0210eba52b37a3c9de130e68067414b1a -size 4319181 +oid sha256:e48f0ab68a157e15ad5f27cea5c1b263ce9fb5f434e9d80f2fb190b31f2952e2 +size 4335592 diff --git a/tests/UI/expected-screenshots/UIIntegrationTest_admin_plugins.png b/tests/UI/expected-screenshots/UIIntegrationTest_admin_plugins.png index a9a0ced4ed..52afc9cd82 100644 --- a/tests/UI/expected-screenshots/UIIntegrationTest_admin_plugins.png +++ b/tests/UI/expected-screenshots/UIIntegrationTest_admin_plugins.png @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:27e7041385dfc642c951ad647ad056ccfcb52726e602fb05f30c7a74564e73c3 -size 1054626 +oid sha256:947f16f424732a8418f1977e8f50358100f553a100c3a46d25c2fca9c0966f41 +size 1063376 diff --git a/tests/UI/expected-screenshots/UIIntegrationTest_api_listing.png b/tests/UI/expected-screenshots/UIIntegrationTest_api_listing.png index 384bf2ed73..2449ec4c08 100644 --- a/tests/UI/expected-screenshots/UIIntegrationTest_api_listing.png +++ b/tests/UI/expected-screenshots/UIIntegrationTest_api_listing.png @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:1a816e74a8d2033456d840d9a96c71a6ad721d900ad74c3572ae07a25d5fdab4 -size 4924205 +oid sha256:7e711ea4c2de88449879eef219e308403c05eb850e2772c96e63a21dde4d6209 +size 4936276 |
