Move Archive.php archive invalidation to Loader… (#15616)3.13.4-b1

* Move Archive.php archive invalidation to Loader so we only invalidate when about to launch archiving.

* Attempt to handle more cases when invalidating before launching archiving.

* fix possible sql error

* fix possible error

* fixing some tests

* remove test code

* Only invalidate specific archive being requested.

* Do not invalidate on today in tracker and avoid existing valid archive check in CronArchive.

* more test fixes

* Attempt to fix more tests.

* Fixing last tests.

* another test fix

* Invalidate in scheduled task if browser triggered archiving is enabled.

* deal with TODO

* Get ArchiveSelectorTest to pass.

* applying review feedback including new tests

* apply review feedback & fix tests

* fix couple more tests

Co-authored-by: Thomas Steur <tsteur@users.noreply.github.com>

10 files changed, 285 insertions, 179 deletions

diff --git a/core/Archive.php b/core/Archive.php
index 1f56ef6062..141d477af4 100644
--- a/core/Archive.php
+++ b/core/Archive.php
@@ -12,7 +12,6 @@ use Piwik\Archive\ArchiveQuery;
 use Piwik\Archive\ArchiveQueryFactory;
 use Piwik\Archive\Parameters;
 use Piwik\ArchiveProcessor\Rules;
-use Piwik\Archive\ArchiveInvalidator;
 use Piwik\Container\StaticContainer;
 use Piwik\DataAccess\ArchiveSelector;
 
@@ -168,11 +167,6 @@ class Archive implements ArchiveQuery
     private static $cache;
 
     /**
-     * @var ArchiveInvalidator
-     */
-    private $invalidator;
-
-    /**
      * @param Parameters $params
      * @param bool $forceIndexedBySite Whether to force index the result of a query by site ID.
      * @param bool $forceIndexedByDate Whether to force index the result of a query by period.
@@ -183,8 +177,6 @@ class Archive implements ArchiveQuery
         $this->params = $params;
         $this->forceIndexedBySite = $forceIndexedBySite;
         $this->forceIndexedByDate = $forceIndexedByDate;
-
-        $this->invalidator = StaticContainer::get('Piwik\Archive\ArchiveInvalidator');
     }
 
     /**
@@ -453,67 +445,6 @@ class Archive implements ArchiveQuery
         return $dataTable;
     }
 
-    private function getSiteIdsThatAreRequestedInThisArchiveButWereNotInvalidatedYet()
-    {
-        if (is_null(self::$cache)) {
-            self::$cache = Cache::getTransientCache();
-        }
-
-        $id = 'Archive.SiteIdsOfRememberedReportsInvalidated';
-
-        if (!self::$cache->contains($id)) {
-            self::$cache->save($id, array());
-        }
-
-        $siteIdsAlreadyHandled = self::$cache->fetch($id);
-        $siteIdsRequested      = $this->params->getIdSites();
-
-        foreach ($siteIdsRequested as $index => $siteIdRequested) {
-            $siteIdRequested = (int) $siteIdRequested;
-
-            if (in_array($siteIdRequested, $siteIdsAlreadyHandled)) {
-                unset($siteIdsRequested[$index]); // was already handled previously, do not do it again
-            } else {
-                $siteIdsAlreadyHandled[] = $siteIdRequested; // we will handle this id this time
-            }
-        }
-
-        self::$cache->save($id, $siteIdsAlreadyHandled);
-
-        return $siteIdsRequested;
-    }
-
-    private function invalidatedReportsIfNeeded()
-    {
-        $siteIdsRequested = $this->getSiteIdsThatAreRequestedInThisArchiveButWereNotInvalidatedYet();
-
-        if (empty($siteIdsRequested)) {
-            return; // all requested site ids were already handled
-        }
-
-        $sitesPerDays = $this->invalidator->getRememberedArchivedReportsThatShouldBeInvalidated();
-
-        foreach ($sitesPerDays as $date => $siteIds) {
-            if (empty($siteIds)) {
-                continue;
-            }
-
-            $siteIdsToActuallyInvalidate = array_intersect($siteIds, $siteIdsRequested);
-
-            if (empty($siteIdsToActuallyInvalidate)) {
-                continue; // all site ids that should be handled are already handled
-            }
-
-            try {
-                $this->invalidator->markArchivesAsInvalidated($siteIdsToActuallyInvalidate, array(Date::factory($date)), false);
-            } catch (\Exception $e) {
-                Site::clearCache();
-                throw $e;
-            }
-        }
-
-        Site::clearCache();
-    }
 
     /**
      * Queries archive tables for data and returns the result.
@@ -638,8 +569,6 @@ class Archive implements ArchiveQuery
      */
     private function cacheArchiveIdsAfterLaunching($archiveGroups, $plugins)
     {
-        $this->invalidatedReportsIfNeeded();
-
         $today = Date::today();
 
         foreach ($this->params->getPeriods() as $period) {
@@ -856,8 +785,11 @@ class Archive implements ArchiveQuery
      */
     private function prepareArchive(array $archiveGroups, Site $site, Period $period)
     {
+        // if cron archiving is running, we will invalidate in CronArchive, not here
+        $invalidateBeforeArchiving = !SettingsServer::isArchivePhpTriggered();
+
         $parameters = new ArchiveProcessor\Parameters($site, $period, $this->params->getSegment());
-        $archiveLoader = new ArchiveProcessor\Loader($parameters);
+        $archiveLoader = new ArchiveProcessor\Loader($parameters, $invalidateBeforeArchiving);
 
         $periodString = $period->getRangeString();
 
diff --git a/core/Archive/ArchiveInvalidator.php b/core/Archive/ArchiveInvalidator.php
index 68274c9857..ad449921f2 100644
--- a/core/Archive/ArchiveInvalidator.php
+++ b/core/Archive/ArchiveInvalidator.php
@@ -147,6 +147,10 @@ class ArchiveInvalidator
             $siteId = (int) $report[0];
             $date   = $report[1];
 
+            if (empty($siteId)) {
+                continue;
+            }
+
             if (empty($sitesPerDay[$date])) {
                 $sitesPerDay[$date] = array();
             }
diff --git a/core/ArchiveProcessor/Loader.php b/core/ArchiveProcessor/Loader.php
index 4564146bd9..20627e98fd 100644
--- a/core/ArchiveProcessor/Loader.php
+++ b/core/ArchiveProcessor/Loader.php
@@ -8,13 +8,18 @@
  */
 namespace Piwik\ArchiveProcessor;
 
+use Piwik\Archive\ArchiveInvalidator;
 use Piwik\Cache;
 use Piwik\Config;
 use Piwik\Container\StaticContainer;
 use Piwik\Context;
 use Piwik\DataAccess\ArchiveSelector;
+use Piwik\DataAccess\ArchiveTableCreator;
 use Piwik\Date;
+use Piwik\Db;
 use Piwik\Piwik;
+use Piwik\Site;
+use Psr\Log\LoggerInterface;
 
 /**
  * This class uses PluginsArchiver class to trigger data aggregation and create archives.
@@ -33,9 +38,28 @@ class Loader
      */
     protected $params;
 
-    public function __construct(Parameters $params)
+    /**
+     * @var ArchiveInvalidator
+     */
+    private $invalidator;
+
+    /**
+     * @var \Piwik\Cache\Cache
+     */
+    private $cache;
+
+    /**
+     * @var LoggerInterface
+     */
+    private $logger;
+
+    public function __construct(Parameters $params, $invalidateBeforeArchiving = false)
     {
         $this->params = $params;
+        $this->invalidateBeforeArchiving = $invalidateBeforeArchiving;
+        $this->invalidator = StaticContainer::get(ArchiveInvalidator::class);
+        $this->cache = Cache::getTransientCache();
+        $this->logger = StaticContainer::get(LoggerInterface::class);
     }
 
     /**
@@ -65,11 +89,19 @@ class Loader
     {
         $this->params->setRequestedPlugin($pluginName);
 
-        list($idArchive, $visits, $visitsConverted) = $this->loadExistingArchiveIdFromDb();
-        if (!empty($idArchive)) {
+        list($idArchive, $visits, $visitsConverted, $isAnyArchiveExists) = $this->loadExistingArchiveIdFromDb();
+        if (!empty($idArchive)) { // we have a usable idarchive (it's not invalidated and it's new enough)
             return $idArchive;
         }
 
+        // if there is an archive, but we can't use it for some reason, invalidate existing archives before
+        // we start archiving. if the archive is made invalid, we will correctly re-archive below.
+        if ($this->invalidateBeforeArchiving
+            && $isAnyArchiveExists
+        ) {
+            $this->invalidatedReportsIfNeeded();
+        }
+
         /** @var ArchivingStatus $archivingStatus */
         $archivingStatus = StaticContainer::get(ArchivingStatus::class);
         $archivingStatus->archiveStarted($this->params);
@@ -170,20 +202,17 @@ class Loader
      */
     public function loadExistingArchiveIdFromDb()
     {
-        $noArchiveFound = array(false, false, false);
-
         if ($this->isArchivingForcedToTrigger()) {
-            return $noArchiveFound;
-        }
+            $this->logger->debug("Archiving forced to trigger for {$this->params}.");
 
-        $minDatetimeArchiveProcessedUTC = $this->getMinTimeArchiveProcessed();
-        $idAndVisits = ArchiveSelector::getArchiveIdAndVisits($this->params, $minDatetimeArchiveProcessedUTC);
-
-        if (!$idAndVisits) {
-            return $noArchiveFound;
+            // return no usable archive found, and no existing archive. this will skip invalidation, which should
+            // be fine since we just force archiving.
+            return [false, false, false, false];
         }
 
-        return $idAndVisits;
+        $minDatetimeArchiveProcessedUTC = $this->getMinTimeArchiveProcessed();
+        $result = ArchiveSelector::getArchiveIdAndVisits($this->params, $minDatetimeArchiveProcessedUTC);
+        return $result;
     }
 
     /**
@@ -242,4 +271,46 @@ class Loader
 
         return $cache->fetch($cacheKey);
     }
+
+    // public for tests
+    public function getReportsToInvalidate()
+    {
+        $sitesPerDays = $this->invalidator->getRememberedArchivedReportsThatShouldBeInvalidated();
+
+        foreach ($sitesPerDays as $dateStr => $siteIds) {
+            if (empty($siteIds)
+                || !in_array($this->params->getSite()->getId(), $siteIds)
+            ) {
+                unset($sitesPerDays[$dateStr]);
+            }
+
+            $date = Date::factory($dateStr);
+            if ($date->isEarlier($this->params->getPeriod()->getDateStart())
+                || $date->isLater($this->params->getPeriod()->getDateEnd())
+            ) { // date in list is not the current date, so ignore it
+                unset($sitesPerDays[$dateStr]);
+            }
+        }
+
+        return $sitesPerDays;
+    }
+
+    private function invalidatedReportsIfNeeded()
+    {
+        $sitesPerDays = $this->getReportsToInvalidate();
+        if (empty($sitesPerDays)) {
+            return;
+        }
+
+        foreach ($sitesPerDays as $date => $siteIds) {
+            try {
+                $this->invalidator->markArchivesAsInvalidated([$this->params->getSite()->getId()], array(Date::factory($date)), false, $this->params->getSegment());
+            } catch (\Exception $e) {
+                Site::clearCache();
+                throw $e;
+            }
+        }
+
+        Site::clearCache();
+    }
 }
diff --git a/core/ArchiveProcessor/Parameters.php b/core/ArchiveProcessor/Parameters.php
index e765787920..954be0a87d 100644
--- a/core/ArchiveProcessor/Parameters.php
+++ b/core/ArchiveProcessor/Parameters.php
@@ -258,4 +258,9 @@ class Parameters
     {
         $this->isRootArchiveRequest = $isRootArchiveRequest;
     }
+
+    public function __toString()
+    {
+        return "[idSite = {$this->getSite()->getId()}, period = {$this->getPeriod()->getLabel()} {$this->getPeriod()->getRangeString()}, segment = {$this->getSegment()->getString()}]";
+    }
 }
diff --git a/core/CronArchive.php b/core/CronArchive.php
index 99bd257c43..9d8801fcbd 100644
--- a/core/CronArchive.php
+++ b/core/CronArchive.php
@@ -898,6 +898,8 @@ class CronArchive
 
         $visitsLastDays = 0;
 
+        $this->invalidateArchivedReportsForSitesThatNeedToBeArchivedAgain();
+
         list($isThereArchive, $newDate) = $this->isThereAValidArchiveForPeriod($idSite, 'day', $date, $segment = '');
         if ($isThereArchive) {
             $visitsToday = Archive::build($idSite, 'day', $date)->getNumeric('nb_visits');
@@ -972,7 +974,8 @@ class CronArchive
         return $dayArchiveWasSuccessful;
     }
 
-    private function isThereAValidArchiveForPeriod($idSite, $period, $date, $segment = '')
+    // public for tests
+    public function isThereAValidArchiveForPeriod($idSite, $period, $date, $segment = '')
     {
         if (Range::isMultiplePeriod($date, $period)) {
             $rangePeriod = Factory::build($period, $date, Site::getTimezoneFor($idSite));
@@ -981,9 +984,17 @@ class CronArchive
             $periodsToCheck = [Factory::build($period, $date, Site::getTimezoneFor($idSite))];
         }
 
-        $periodsToCheckRanges = array_map(function (Period $p) { return $p->getRangeString(); }, $periodsToCheck);
+        $isTodayIncluded = $this->isTodayIncludedInPeriod($idSite, $periodsToCheck);
+        $isLast = preg_match('/^last([0-9]+)/', $date, $matches);
 
-        $this->invalidateArchivedReportsForSitesThatNeedToBeArchivedAgain();
+        // don't do this check for a single period that includes today
+        if ($isTodayIncluded
+            && !$isLast
+        ) {
+            return [false, null];
+        }
+
+        $periodsToCheckRanges = array_map(function (Period $p) { return $p->getRangeString(); }, $periodsToCheck);
 
         $archiveIds = ArchiveSelector::getArchiveIds(
             [$idSite], $periodsToCheck, new Segment($segment, [$idSite]), $plugins = [], // empty plugins param since we only check for an 'all' archive
@@ -1003,37 +1014,60 @@ class CronArchive
         // if there is an invalidated archive within the range, find out the oldest one and how far it is from today,
         // and change the lastN $date to be value so it is correctly re-processed.
         $newDate = $date;
-        if (!$isThereArchiveForAllPeriods
-            && preg_match('/^last([0-9]+)/', $date, $matches)
-        ) {
-            $lastNValue = (int) $matches[1];
-
-            usort($diff, function ($lhs, $rhs) {
-                $lhsDate = explode(',', $lhs)[0];
-                $rhsDate = explode(',', $rhs)[0];
-
-                if ($lhsDate == $rhsDate) {
-                    return 1;
-                } else if (Date::factory($lhsDate)->isEarlier(Date::factory($rhsDate))) {
-                    return -1;
-                } else {
-                    return 1;
-                }
-            });
+        if ($isLast) {
+            if (!$isThereArchiveForAllPeriods) {
+                $lastNValue = (int)$matches[1];
+
+                usort($diff, function ($lhs, $rhs) {
+                    $lhsDate = explode(',', $lhs)[0];
+                    $rhsDate = explode(',', $rhs)[0];
+
+                    if ($lhsDate == $rhsDate) {
+                        return 1;
+                    } else if (Date::factory($lhsDate)->isEarlier(Date::factory($rhsDate))) {
+                        return -1;
+                    } else {
+                        return 1;
+                    }
+                });
 
-            $oldestDateWithoutArchive = explode(',', reset($diff))[0];
-            $todayInTimezone = Date::factoryInTimezone('today', Site::getTimezoneFor($idSite));
+                $oldestDateWithoutArchive = explode(',', reset($diff))[0];
+                $todayInTimezone = Date::factoryInTimezone('today', Site::getTimezoneFor($idSite));
 
-            /** @var Range $newRangePeriod */
-            $newRangePeriod = PeriodFactory::build($period, $oldestDateWithoutArchive . ',' . $todayInTimezone);
+                /** @var Range $newRangePeriod */
+                $newRangePeriod = PeriodFactory::build($period, $oldestDateWithoutArchive . ',' . $todayInTimezone);
 
-            $newDate = 'last' . min($lastNValue, $newRangePeriod->getNumberOfSubperiods());
+                $newDate = 'last' . max(min($lastNValue, $newRangePeriod->getNumberOfSubperiods()), 2);
+            } else if ($isTodayIncluded) {
+                $isThereArchiveForAllPeriods = false;
+                $newDate = 'last2';
+            }
         }
 
         return [$isThereArchiveForAllPeriods, $newDate];
     }
 
     /**
+     * @param int $idSite
+     * @param Period[] $periods
+     * @return bool
+     * @throws Exception
+     */
+    private function isTodayIncludedInPeriod($idSite, $periods)
+    {
+        $timezone = Site::getTimezoneFor($idSite);
+        $today = Date::factoryInTimezone('today', $timezone);
+
+        foreach ($periods as $period) {
+            if ($period->isDateInPeriod($today)) {
+                return true;
+            }
+        }
+
+        return false;
+    }
+
+    /**
      * @param $idSite
      * @return array
      */
@@ -1102,6 +1136,8 @@ class CronArchive
                     return Request::ABORT;
                 }
 
+                $this->invalidateArchivedReportsForSitesThatNeedToBeArchivedAgain();
+
                 list($isThereArchive, $newDate) = $this->isThereAValidArchiveForPeriod($idSite, $period, $date, $segment);
                 if ($isThereArchive) {
                     $this->logArchiveWebsiteSkippedValidArchiveExists($idSite, $period, $date);
@@ -1979,6 +2015,8 @@ class CronArchive
                     return Request::ABORT;
                 }
 
+                $this->invalidateArchivedReportsForSitesThatNeedToBeArchivedAgain();
+
                 list($isThereArchive, $newDate) = $this->isThereAValidArchiveForPeriod($idSite, $period, $date, $segment);
                 if ($isThereArchive) {
                     $this->logArchiveWebsiteSkippedValidArchiveExists($idSite, $period, $date, $segment);
diff --git a/core/DataAccess/ArchiveSelector.php b/core/DataAccess/ArchiveSelector.php
index 4232404141..daaeb7b2a6 100644
--- a/core/DataAccess/ArchiveSelector.php
+++ b/core/DataAccess/ArchiveSelector.php
@@ -48,7 +48,12 @@ class ArchiveSelector
     /**
      * @param ArchiveProcessor\Parameters $params
      * @param bool $minDatetimeArchiveProcessedUTC deprecated. will be removed in Matomo 4.
-     * @return array|bool
+     * @return array An array with four values: \
+     *               - the latest archive ID or false if none
+     *               - the latest visits value for the latest archive, regardless of whether the archive is invalidated or not
+     *               - the latest visits converted value for the latest archive, regardless of whether the archive is invalidated or not
+     *               - whether there is an archive that exists or not. if this is true and the latest archive is false, it means
+     *                 the archive found was not usable (for example, it was invalidated and we are not looking for invalidated archives)
      * @throws Exception
      */
     public static function getArchiveIdAndVisits(ArchiveProcessor\Parameters $params, $minDatetimeArchiveProcessedUTC = false, $includeInvalidated = true)
@@ -61,80 +66,41 @@ class ArchiveSelector
 
         $numericTable = ArchiveTableCreator::getNumericTable($dateStart);
 
-        $minDatetimeIsoArchiveProcessedUTC = null;
-        if ($minDatetimeArchiveProcessedUTC) {
-            $minDatetimeIsoArchiveProcessedUTC = Date::factory($minDatetimeArchiveProcessedUTC)->getDatetime();
-        }
-
         $requestedPlugin = $params->getRequestedPlugin();
         $segment         = $params->getSegment();
         $plugins = array("VisitsSummary", $requestedPlugin);
 
         $doneFlags      = Rules::getDoneFlags($plugins, $segment);
+        $requestedPluginDoneFlags = Rules::getDoneFlags([$requestedPlugin], $segment);
         $doneFlagValues = Rules::getSelectableDoneFlagValues($includeInvalidated, $params);
 
-        $results = self::getModel()->getArchiveIdAndVisits($numericTable, $idSite, $period, $dateStartIso, $dateEndIso, $minDatetimeIsoArchiveProcessedUTC, $doneFlags, $doneFlagValues);
-
-        if (empty($results)) {
-            return false;
-        }
-
-        $idArchive = self::getMostRecentIdArchiveFromResults($segment, $requestedPlugin, $results);
-
-        $idArchiveVisitsSummary = self::getMostRecentIdArchiveFromResults($segment, "VisitsSummary", $results);
-
-        list($visits, $visitsConverted) = self::getVisitsMetricsFromResults($idArchive, $idArchiveVisitsSummary, $results);
-
-        if (false === $visits && false === $idArchive) {
-            return false;
+        $results = self::getModel()->getArchiveIdAndVisits($numericTable, $idSite, $period, $dateStartIso, $dateEndIso, null, $doneFlags);
+        if (empty($results)) { // no archive found
+            return [false, false, false, false];
         }
 
-        return array($idArchive, $visits, $visitsConverted);
-    }
+        $result = self::findArchiveDataWithLatestTsArchived($results, $requestedPluginDoneFlags);
 
-    protected static function getVisitsMetricsFromResults($idArchive, $idArchiveVisitsSummary, $results)
-    {
-        $visits = $visitsConverted = false;
-        $archiveWithVisitsMetricsWasFound = ($idArchiveVisitsSummary !== false);
+        $visits = isset($result['nb_visits']) ? $result['nb_visits'] : false;
+        $visitsConverted = isset($result['nb_visits_converted']) ? $result['nb_visits_converted'] : false;
 
-        if ($archiveWithVisitsMetricsWasFound) {
-            $visits = $visitsConverted = 0;
+        if (isset($result['value'])
+            && !in_array($result['value'], $doneFlagValues)
+        ) { // the archive cannot be considered valid for this request (has wrong done flag value)
+            return [false, $visits, $visitsConverted, true];
         }
 
-        foreach ($results as $result) {
-            if (in_array($result['idarchive'], array($idArchive, $idArchiveVisitsSummary))) {
-                $value = (int)$result['value'];
-                if (empty($visits)
-                    && $result['name'] == self::NB_VISITS_RECORD_LOOKED_UP
-                ) {
-                    $visits = $value;
-                }
-                if (empty($visitsConverted)
-                    && $result['name'] == self::NB_VISITS_CONVERTED_RECORD_LOOKED_UP
-                ) {
-                    $visitsConverted = $value;
-                }
-            }
+        // the archive is too old
+        if ($minDatetimeArchiveProcessedUTC
+            && isset($result['idarchive'])
+            && Date::factory($result['ts_archived'])->isEarlier(Date::factory($minDatetimeArchiveProcessedUTC))
+        ) {
+            return [false, $visits, $visitsConverted, true];
         }
 
-        return array($visits, $visitsConverted);
-    }
-
-    protected static function getMostRecentIdArchiveFromResults(Segment $segment, $requestedPlugin, $results)
-    {
-        $idArchive = false;
-        $namesRequestedPlugin = Rules::getDoneFlags(array($requestedPlugin), $segment);
-
-        foreach ($results as $result) {
-            if ($idArchive === false
-                && in_array($result['name'], $namesRequestedPlugin)
-            ) {
-                $idArchive = $result['idarchive'];
-                break;
-            }
-        }
+        $idArchive = isset($result['idarchive']) ? $result['idarchive'] : false;
 
-        return $idArchive;
+        return array($idArchive, $visits, $visitsConverted, true);
     }
 
     /**
@@ -213,7 +179,6 @@ class ArchiveSelector
 
             $sql = sprintf($getArchiveIdsSql, $table, $dateCondition);
 
-
             $archiveIds = $db->fetchAll($sql, $bind);
 
             // get the archive IDs
@@ -381,4 +346,71 @@ class ArchiveSelector
         // create the SQL to find archives that are DONE
         return "((name IN ($allDoneFlags)) AND (value IN (" . implode(',', $possibleValues) . ")))";
     }
+
+    /**
+     * This method takes the output of Model::getArchiveIdAndVisits() and selects data from the
+     * latest archives.
+     *
+     * This includes:
+     * - the idarchive with the latest ts_archived ($results will be ordered by ts_archived desc)
+     * - the visits/converted visits of the latest archive, which includes archives for VisitsSummary alone
+     *   ($requestedPluginDoneFlags will have the done flag for the overall archive plus a done flag for
+     *   VisitsSummary by itself)
+     * - the ts_archived for the latest idarchive
+     * - the doneFlag value for the latest archive
+     *
+     * @param $results
+     * @param $requestedPluginDoneFlags
+     * @return array
+     */
+    private static function findArchiveDataWithLatestTsArchived($results, $requestedPluginDoneFlags)
+    {
+        // find latest idarchive for each done flag
+        $idArchives = [];
+        foreach ($results as $row) {
+            $doneFlag = $row['name'];
+            if (preg_match('/^done/', $doneFlag)
+                && !isset($idArchives[$doneFlag])
+            ) {
+                $idArchives[$doneFlag] = $row['idarchive'];
+            }
+        }
+
+        $archiveData = [];
+
+        // gather the latest visits/visits_converted metrics
+        foreach ($results as $row) {
+            $name = $row['name'];
+            if (!isset($archiveData[$name])
+                && in_array($name, [self::NB_VISITS_RECORD_LOOKED_UP, self::NB_VISITS_CONVERTED_RECORD_LOOKED_UP])
+                && in_array($row['idarchive'], $idArchives)
+            ) {
+                $archiveData[$name] = $row['value'];
+            }
+        }
+
+        // if an  archive is found, but the metric data isn't found, we set the value to 0,
+        // so it won't get returned as false. this is here because the code used to do this before this change
+        // and we didn't want to introduce any side effects. it may be removable in the future.
+        foreach ([self::NB_VISITS_RECORD_LOOKED_UP, self::NB_VISITS_CONVERTED_RECORD_LOOKED_UP] as $metric) {
+            if (!empty($idArchives)
+                && !isset($archiveData[$metric])
+            ) {
+                $archiveData[$metric] = 0;
+            }
+        }
+
+        // set the idarchive & ts_archived for the archive we're looking for
+        foreach ($results as $row) {
+            $name = $row['name'];
+            if (in_array($name, $requestedPluginDoneFlags)) {
+                $archiveData['idarchive'] = $row['idarchive'];
+                $archiveData['ts_archived'] = $row['ts_archived'];
+                $archiveData['value'] = $row['value'];
+                break;
+            }
+        }
+
+        return $archiveData;
+    }
 }
diff --git a/core/DataAccess/Model.php b/core/DataAccess/Model.php
index ab32487f8d..4e72698269 100644
--- a/core/DataAccess/Model.php
+++ b/core/DataAccess/Model.php
@@ -215,7 +215,8 @@ class Model
         return $deletedRows;
     }
 
-    public function getArchiveIdAndVisits($numericTable, $idSite, $period, $dateStartIso, $dateEndIso, $minDatetimeIsoArchiveProcessedUTC, $doneFlags, $doneFlagValues)
+    public function getArchiveIdAndVisits($numericTable, $idSite, $period, $dateStartIso, $dateEndIso, $minDatetimeIsoArchiveProcessedUTC,
+                                          $doneFlags, $doneFlagValues = null)
     {
         $bindSQL = array($idSite,
             $dateStartIso,
@@ -231,7 +232,8 @@ class Model
             $bindSQL[]      = $minDatetimeIsoArchiveProcessedUTC;
         }
 
-        $sqlQuery = "SELECT idarchive, value, name, date1 as startDate FROM $numericTable
+        // NOTE: we can't predict how many segments there will be so there could be lots of nb_visits/nb_visits_converted rows... have to select everything.
+        $sqlQuery = "SELECT idarchive, value, name, ts_archived, date1 as startDate FROM $numericTable
                      WHERE idsite = ?
                          AND date1 = ?
                          AND date2 = ?
@@ -240,7 +242,7 @@ class Model
                                OR name = '" . ArchiveSelector::NB_VISITS_RECORD_LOOKED_UP . "'
                                OR name = '" . ArchiveSelector::NB_VISITS_CONVERTED_RECORD_LOOKED_UP . "')
                          $timeStampWhere
-                     ORDER BY idarchive DESC";
+                     ORDER BY ts_archived DESC, idarchive DESC";
         $results = Db::fetchAll($sqlQuery, $bindSQL);
 
         return $results;
@@ -428,7 +430,14 @@ class Model
         $allDoneFlags = "'" . implode("','", $doneFlags) . "'";
 
         // create the SQL to find archives that are DONE
-        return "((name IN ($allDoneFlags)) AND (value IN (" . implode(',', $possibleValues) . ")))";
+        $result = "((name IN ($allDoneFlags))";
+
+        if (!empty($possibleValues)) {
+            $result .= " AND (value IN (" . implode(',', $possibleValues) . ")))";
+        }
+        $result .= ')';
+
+        return $result;
     }
 
 }
diff --git a/core/Period.php b/core/Period.php
index 6e3ef0f327..e717995f28 100644
--- a/core/Period.php
+++ b/core/Period.php
@@ -248,6 +248,21 @@ abstract class Period
     }
 
     /**
+     * Returns whether the date `$date` is within the current period or not.
+     *
+     * Note: the time component of the period's dates and `$date` is ignored.
+     *
+     * @param Date $today
+     * @return bool
+     */
+    public function isDateInPeriod(Date $date)
+    {
+        $ts = $date->getStartOfDay()->getTimestamp();
+        return $ts >= $this->getDateStart()->getStartOfDay()->getTimestamp()
+            && $ts < $this->getDateEnd()->addDay(1)->getStartOfDay()->getTimestamp();
+    }
+
+    /**
      * Add a date to the period.
      *
      * Protected because adding periods after initialization is not supported.
diff --git a/core/Tracker/Visit.php b/core/Tracker/Visit.php
index 9c76967adc..d3de0015ca 100644
--- a/core/Tracker/Visit.php
+++ b/core/Tracker/Visit.php
@@ -572,10 +572,10 @@ class Visit implements VisitInterface
         $date = Date::factory((int)$time, $timezone);
 
         // $date->isToday() is buggy when server and website timezones don't match - so we'll do our own checking
-        $startOfTomorrow = Date::factoryInTimezone('today', $timezone)->addDay(1);
-        $isLaterThanToday = $date->getTimestamp() >= $startOfTomorrow->getTimestamp();
-        if ($isLaterThanToday) {
-            return;
+        $startOfToday = Date::factoryInTimezone('yesterday', $timezone)->addDay(1);
+        $isLaterThanYesterday = $date->getTimestamp() >= $startOfToday->getTimestamp();
+        if ($isLaterThanYesterday) {
+            return; // don't try to invalidate archives for today or later
         }
 
         $this->invalidator->rememberToInvalidateArchivedReportsLater($idSite, $date);
diff --git a/core/Twig.php b/core/Twig.php
index afb6a1cdc2..687e67be75 100644
--- a/core/Twig.php
+++ b/core/Twig.php
@@ -362,7 +362,7 @@ class Twig
             if (!empty($options['raw'])) {
                 $template .= piwik_fix_lbrace($message);
             } else {
-                $template .= twig_escape_filter($twigEnv, $message, 'html');
+                $template .= piwik_escape_filter($twigEnv, $message, 'html');
             }
 
             $template .= '</div>';