Initiate range archiving if an archive is invalidated, the request is from the browser, and browser archiving is authorized (#17379)

* Initiate archiving if an archive is invalidated, the request is from the browser, and browser archiving for the current request is authorized.

* add test that was strangely removed and add tests for period = range

* Add test for tracking data in the past and fix invalidation issue in core:archive.

* Add test for default use case of core:archive w/ browser initiated range archiving.

* actually just dont perform the check if the period does not include today

* undo isPeriodIncludesToday change

* Fix ArchiveInvalidationTest and get to work, fix bug in Loader causing it to fail

* fix tests

* add comment to test

* Fix CronArchiveTest, make sure we do not use a TTL when running invalidateRecentDate("yesterday").

* tweak comment

* make sure invalid ranges only rearchive when authorized to rearchive child archive or when all child archives are usable while still respecting ttl for periods that include today

* instead of previous change, make range ttl get respected if range period is used and archiving is enabled for the current request/period

* remove dead code + tweak test

* add check for invalidated archive

* move new invalidation check to Loader from ArchiveSelector since getArchiveIdAndVisits is used in multiple code paths now

* remove TODO

* remove use

* apply pr feedback

* get tests to pass

* Fix strange test failure on travis-ci (class was loaded before mock file methods used in next test were loaded)

1 files changed, 19 insertions, 6 deletions

diff --git a/core/CronArchive.php b/core/CronArchive.php
index 0475067ce9..ddbc74e743 100644
--- a/core/CronArchive.php
+++ b/core/CronArchive.php
@@ -857,10 +857,14 @@ class CronArchive
             'date' => $date->getDatetime(),
         ]);
 
-        $this->invalidateWithSegments([$idSite], $date->toString(), 'day');
+        // if we are invalidating yesterday here, we are only interested in checking if there is no archive for yesterday, or the day has changed since
+        // the last archive was archived (in which there may have been more visits before midnight). so we disable the ttl check, since any archive
+        // will be good enough, if the date hasn't changed.
+        $isYesterday = $dateStr == 'yesterday';
+        $this->invalidateWithSegments([$idSite], $date->toString(), 'day', false, $doNotIncludeTtlInExistingArchiveCheck = $isYesterday);
     }
 
-    private function invalidateWithSegments($idSites, $date, $period, $_forceInvalidateNonexistant = false)
+    private function invalidateWithSegments($idSites, $date, $period, $_forceInvalidateNonexistant = false, $doNotIncludeTtlInExistingArchiveCheck = false)
     {
         if ($date instanceof Date) {
             $date = $date->toString();
@@ -878,7 +882,7 @@ class CronArchive
 
         foreach ($idSites as $idSite) {
             $params = new Parameters(new Site($idSite), $periodObj, new Segment('', [$idSite], $periodObj->getDateStart(), $periodObj->getDateEnd()));
-            if ($this->isThereExistingValidPeriod($params)) {
+            if ($this->canWeSkipInvalidatingBecauseThereIsAUsablePeriod($params, $doNotIncludeTtlInExistingArchiveCheck)) {
                 $this->logger->debug('  Found usable archive for {archive}, skipping invalidation.', ['archive' => $params]);
             } else {
                 $this->getApiToInvalidateArchivedReport()->invalidateArchivedReports($idSite, $date, $period, $segment = false, $cascadeDown = false,
@@ -887,7 +891,7 @@ class CronArchive
 
             foreach ($this->segmentArchiving->getAllSegmentsToArchive($idSite) as $segmentDefinition) {
                 $params = new Parameters(new Site($idSite), $periodObj, new Segment(urlencode($segmentDefinition), [$idSite], $periodObj->getDateStart(), $periodObj->getDateEnd()));
-                if ($this->isThereExistingValidPeriod($params)) {
+                if ($this->canWeSkipInvalidatingBecauseThereIsAUsablePeriod($params, $doNotIncludeTtlInExistingArchiveCheck)) {
                     $this->logger->debug('  Found usable archive for {archive}, skipping invalidation.', ['archive' => $params]);
                 } else {
                     $this->getApiToInvalidateArchivedReport()->invalidateArchivedReports($idSite, $date, $period, urlencode($segmentDefinition),
@@ -897,14 +901,23 @@ class CronArchive
         }
     }
 
-    public function isThereExistingValidPeriod(Parameters $params)
+    /**
+     * Returns true if there is an existing valid period we can use, or false if there isn't and the invalidation should go through.
+     *
+     * Note: this method should only be used in the context of invalidation.
+     *
+     * @params Parameters $params The parameters for the archive we want to invalidate.
+     */
+    public function canWeSkipInvalidatingBecauseThereIsAUsablePeriod(Parameters $params, $doNotIncludeTtlInExistingArchiveCheck = false)
     {
         $today = Date::factoryInTimezone('today', Site::getTimezoneFor($params->getSite()->getId()));
 
         $isYesterday = $params->getPeriod()->getLabel() == 'day' && $params->getPeriod()->getDateStart()->toString() == Date::factory('yesterday')->toString();
 
         $isPeriodIncludesToday = $params->getPeriod()->isDateInPeriod($today);
-        $minArchiveProcessedTime = $isPeriodIncludesToday ? Date::now()->subSeconds(Rules::getPeriodArchiveTimeToLiveDefault($params->getPeriod()->getLabel())) : null;
+
+        $minArchiveProcessedTime = $doNotIncludeTtlInExistingArchiveCheck ? null :
+            Date::now()->subSeconds(Rules::getPeriodArchiveTimeToLiveDefault($params->getPeriod()->getLabel()));
 
         // empty plugins param since we only check for an 'all' archive
         list($idArchive, $visits, $visitsConverted, $ignore, $tsArchived) = ArchiveSelector::getArchiveIdAndVisits($params, $minArchiveProcessedTime, $includeInvalidated = $isPeriodIncludesToday);