If archive_invalidations is in inconsistent state, fix as getting next archive to process. (#16886)

* If archive_invalidations is in inconsistent state, fix as getting next archive to process.

* remove in progress invalidation block

* move repair logic to after archives launched

* Add debug log for reparing invalidation table

* apply review feedback

* fix test

3 files changed, 76 insertions, 18 deletions

diff --git a/core/CronArchive.php b/core/CronArchive.php
index 3fb20a58fd..cc7d228c63 100644
--- a/core/CronArchive.php
+++ b/core/CronArchive.php
@@ -485,9 +485,10 @@ class CronArchive
 
             $this->logArchiveJobFinished($url, $timers[$index], $visitsForPeriod, $archivesBeingQueried[$index]['plugin'], $archivesBeingQueried[$index]['report']);
 
-            // TODO: do in ArchiveWriter
             $this->deleteInvalidatedArchives($archivesBeingQueried[$index]);
 
+            $this->repairInvalidationsIfNeeded($archivesBeingQueried[$index]);
+
             ++$successCount;
         }
 
@@ -764,11 +765,6 @@ class CronArchive
 
     public function invalidateArchivedReportsForSitesThatNeedToBeArchivedAgain($idSiteToInvalidate)
     {
-        if ($this->model->isInvalidationsScheduledForSite($idSiteToInvalidate)) {
-            $this->logger->debug("Invalidations currently exist for idSite $idSiteToInvalidate, skipping invalidating for now...");
-            return;
-        }
-
         if (empty($this->segmentArchiving)) {
             // might not be initialised if init is not called
             $this->segmentArchiving = new SegmentArchiving($this->processNewSegmentsFrom, $this->dateLastForced);
@@ -938,6 +934,77 @@ class CronArchive
         return !empty($idArchive);
     }
 
+    // public for tests
+    public function repairInvalidationsIfNeeded($archiveToProcess)
+    {
+        $table = Common::prefixTable('archive_invalidations');
+
+        $bind = [
+            $archiveToProcess['idsite'],
+            $archiveToProcess['name'],
+            $archiveToProcess['period'],
+            $archiveToProcess['date1'],
+            $archiveToProcess['date2'],
+        ];
+
+        $reportClause = '';
+        if (!empty($archiveToProcess['report'])) {
+            $reportClause = " AND report = ?";
+            $bind[] = $archiveToProcess['report'];
+        }
+
+        $sql = "SELECT DISTINCT period FROM `$table`
+                 WHERE idsite = ? AND name = ? AND period > ? AND ? >= date1 AND date2 >= ? AND status = " . ArchiveInvalidator::INVALIDATION_STATUS_QUEUED . " $reportClause";
+
+        $higherPeriods = Db::fetchAll($sql, $bind);
+        $higherPeriods = array_column($higherPeriods, 'period');
+
+        $invalidationsToInsert = [];
+        foreach (Piwik::$idPeriods as $label => $id) {
+            // lower period than the one we're processing or range, don't care
+            if ($id <= $archiveToProcess['period'] || $label == 'range') {
+                continue;
+            }
+
+            if (in_array($id, $higherPeriods)) { // period exists in table
+                continue;
+            }
+
+            // archive is for week that is over two months, we don't need to care about the month
+            if ($label == 'month'
+                && Date::factory($archiveToProcess['date1'])->toString('m') != Date::factory($archiveToProcess['date2'])->toString('m')
+            ) {
+                continue;
+            }
+
+            $period = Period\Factory::build($label, $archiveToProcess['date1']);
+
+            $invalidationToInsert = [
+                'idarchive' => null,
+                'name' => $archiveToProcess['name'],
+                'report' => $archiveToProcess['report'],
+                'idsite' => $archiveToProcess['idsite'],
+                'date1' => $period->getDateStart()->getDatetime(),
+                'date2' => $period->getDateEnd()->getDatetime(),
+                'period' => $id,
+                'ts_invalidated' => $archiveToProcess['ts_invalidated'],
+            ];
+
+            $this->logger->debug("Found dangling invalidation, inserting {invalidationToInsert}", [
+                'invalidationToInsert' => json_encode($invalidationToInsert),
+            ]);
+
+            $invalidationsToInsert[] = $invalidationToInsert;
+        }
+
+        if (empty($invalidationsToInsert)) {
+            return;
+        }
+
+        $fields = ['idarchive', 'name', 'report', 'idsite', 'date1', 'date2', 'period', 'ts_invalidated'];
+        Db\BatchInsert::tableInsertBatch(Common::prefixTable('archive_invalidations'), $fields, $invalidationsToInsert);
+    }
+
     private function setInvalidationTime()
     {
         $cache = Cache::getTransientCache();
diff --git a/core/CronArchive/QueueConsumer.php b/core/CronArchive/QueueConsumer.php
index 734a4ce962..f7a98b9498 100644
--- a/core/CronArchive/QueueConsumer.php
+++ b/core/CronArchive/QueueConsumer.php
@@ -14,10 +14,12 @@ use Piwik\ArchiveProcessor\Loader;
 use Piwik\ArchiveProcessor\Parameters;
 use Piwik\ArchiveProcessor\Rules;
 use Piwik\CliMulti\RequestParser;
+use Piwik\Common;
 use Piwik\CronArchive;
 use Piwik\DataAccess\ArchiveSelector;
 use Piwik\DataAccess\Model;
 use Piwik\Date;
+use Piwik\Db;
 use Piwik\Exception\UnexpectedWebsiteFoundException;
 use Piwik\Period;
 use Piwik\Period\Factory as PeriodFactory;
diff --git a/core/DataAccess/Model.php b/core/DataAccess/Model.php
index 6ba5c165cf..8f75740dd5 100644
--- a/core/DataAccess/Model.php
+++ b/core/DataAccess/Model.php
@@ -713,7 +713,7 @@ class Model
     public function getNextInvalidatedArchive($idSite, $archivingStartTime, $idInvalidationsToExclude = null, $useLimit = true)
     {
         $table = Common::prefixTable('archive_invalidations');
-        $sql = "SELECT idinvalidation, idarchive, idsite, date1, date2, period, `name`, report
+        $sql = "SELECT idinvalidation, idarchive, idsite, date1, date2, period, `name`, report, ts_invalidated
                   FROM `$table`
                  WHERE idsite = ? AND status != ? AND ts_invalidated <= ?";
         $bind = [
@@ -876,15 +876,4 @@ class Model
         $query = Db::query($sql, $bind);
         return $query->rowCount();
     }
-
-    public function isInvalidationsScheduledForSite($idSite)
-    {
-        $table = Common::prefixTable('archive_invalidations');
-
-        $bind = [(int) $idSite];
-
-        $sql = "SELECT idsite FROM `$table` WHERE idsite = ? LIMIT 1";
-        $value = Db::fetchOne($sql, $bind);
-        return !empty($value);
-    }
 }