Better detect failed archiving jobs. (#16701)4.0.0-rc3

6 files changed, 104 insertions, 5 deletions

diff --git a/core/CronArchive.php b/core/CronArchive.php
index 358e6bfa78..4f36b4e485 100644
--- a/core/CronArchive.php
+++ b/core/CronArchive.php
@@ -345,6 +345,13 @@ class CronArchive
             return;
         }
 
+        $failedJobs = $this->model->resetFailedArchivingJobs();
+        if ($failedJobs) {
+            $this->logger->info("Found {failed} failed jobs (ts_invalidated older than 1 day), resetings status to try them again.", [
+                'failed' => $failedJobs,
+            ]);
+        }
+
         $countOfProcesses = $this->getMaxConcurrentApiRequests();
 
         $queueConsumer = new QueueConsumer($this->logger, $this->websiteIdArchiveList, $countOfProcesses, $pid,
diff --git a/core/DataAccess/Model.php b/core/DataAccess/Model.php
index ed09659fae..6470305281 100644
--- a/core/DataAccess/Model.php
+++ b/core/DataAccess/Model.php
@@ -632,7 +632,7 @@ class Model
         $table = Common::prefixTable('archive_invalidations');
 
         // set archive value to in progress if not set already
-        $statement = Db::query("UPDATE `$table` SET `status` = ? WHERE idinvalidation = ? AND status = ?", [
+        $statement = Db::query("UPDATE `$table` SET `status` = ? AND ts_started = NOW() WHERE idinvalidation = ? AND status = ?", [
             ArchiveInvalidator::INVALIDATION_STATUS_IN_PROGRESS,
             $invalidation['idinvalidation'],
             ArchiveInvalidator::INVALIDATION_STATUS_QUEUED,
@@ -867,4 +867,19 @@ class Model
         $sql = "UPDATE $table SET status = " . ArchiveInvalidator::INVALIDATION_STATUS_QUEUED . " WHERE idinvalidation = ?";
         Db::query($sql, [$idinvalidation]);
     }
+
+    public function resetFailedArchivingJobs()
+    {
+        $table = Common::prefixTable('archive_invalidations');
+        $sql = "UPDATE $table SET status = ? WHERE status = ? AND ts_started IS NOT NULL AND ts_started < ?";
+
+        $bind = [
+            ArchiveInvalidator::INVALIDATION_STATUS_QUEUED,
+            ArchiveInvalidator::INVALIDATION_STATUS_IN_PROGRESS,
+            Date::now()->subDay(1)->getDatetime(),
+        ];
+
+        $query = Db::query($sql, $bind);
+        return $query->rowCount();
+    }
 }
diff --git a/core/Db/Schema/Mysql.php b/core/Db/Schema/Mysql.php
index dfa14e4373..61b133d319 100644
--- a/core/Db/Schema/Mysql.php
+++ b/core/Db/Schema/Mysql.php
@@ -318,6 +318,7 @@ class Mysql implements SchemaInterface
                                             date2 DATE NOT NULL,
                                             period TINYINT UNSIGNED NOT NULL,
                                             ts_invalidated DATETIME NULL,
+                                            ts_started DATETIME NULL,
                                             status TINYINT(1) UNSIGNED DEFAULT 0,
                                             `report` VARCHAR(255) NULL,
                                             PRIMARY KEY(idinvalidation),
diff --git a/core/Updates/4.0.0-rc3.php b/core/Updates/4.0.0-rc3.php
new file mode 100644
index 0000000000..86c9ca0189
--- /dev/null
+++ b/core/Updates/4.0.0-rc3.php
@@ -0,0 +1,44 @@
+<?php
+/**
+ * Matomo - free/libre analytics platform
+ *
+ * @link https://matomo.org
+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
+ *
+ */
+
+namespace Piwik\Updates;
+
+use Piwik\Config;
+use Piwik\Updater;
+use Piwik\Updates as PiwikUpdates;
+use Piwik\Updater\Migration\Factory as MigrationFactory;
+
+/**
+ * Update for version 4.0.0-b3.
+ */
+class Updates_4_0_0_rc3 extends PiwikUpdates
+{
+    /**
+     * @var MigrationFactory
+     */
+    private $migration;
+
+    public function __construct(MigrationFactory $factory)
+    {
+        $this->migration = $factory;
+    }
+
+    public function getMigrations(Updater $updater)
+    {
+        $migrations = [];
+        $migrations[] = $this->migration->db->addColumn('archive_invalidations', 'ts_started', 'DATETIME NULL');
+        return $migrations;
+    }
+
+    public function doUpdate(Updater $updater)
+    {
+        $updater->executeMigrations(__FILE__, $this->getMigrations($updater));
+    }
+
+}
diff --git a/core/Version.php b/core/Version.php
index 9e4e5cbe2c..9bcba1327e 100644
--- a/core/Version.php
+++ b/core/Version.php
@@ -20,7 +20,7 @@ final class Version
      * The current Matomo version.
      * @var string
      */
-    const VERSION = '4.0.0-rc2';
+    const VERSION = '4.0.0-rc3';
     const MAJOR_VERSION = 4;
 
     public function isStableVersion($version)
diff --git a/tests/PHPUnit/Integration/DataAccess/ModelTest.php b/tests/PHPUnit/Integration/DataAccess/ModelTest.php
index 532b0e48f7..15305796ee 100644
--- a/tests/PHPUnit/Integration/DataAccess/ModelTest.php
+++ b/tests/PHPUnit/Integration/DataAccess/ModelTest.php
@@ -38,6 +38,33 @@ class ModelTest extends IntegrationTestCase
         $this->model->createArchiveTable($this->tableName, 'archive_numeric');
     }
 
+    public function test_resetFailedArchivingJobs_updatesCorrectStatuses()
+    {
+        Date::$now = strtotime('2020-03-03 04:00:00');
+
+        $this->insertInvalidations([
+            ['idsite' => 1, 'date1' => '2020-02-03', 'date2' => '2020-02-03', 'period' => 1, 'name' => 'done', 'value' => 1, 'status' => 1, 'ts_invalidated' => '2020-03-01 00:00:00', 'ts_started' => '2020-03-02 03:00:00'],
+            ['idsite' => 2, 'date1' => '2020-02-03', 'date2' => '2020-02-03', 'period' => 1, 'name' => 'done.Plugin', 'value' => 2, 'status' => 0, 'ts_invalidated' => '2020-03-01 00:00:00', 'ts_started' => '2020-03-02 03:00:00'],
+            ['idsite' => 1, 'date1' => '2020-02-03', 'date2' => '2020-02-03', 'period' => 1, 'name' => 'doneblablah', 'value' => 3, 'status' => 0, 'ts_invalidated' => '2020-03-01 00:00:00', 'ts_started' => '2020-03-03 00:00:00'],
+            ['idsite' => 3, 'date1' => '2020-02-03', 'date2' => '2020-02-03', 'period' => 1, 'name' => 'donebluhbluh', 'value' => 4, 'status' => 1, 'ts_invalidated' => '2020-03-01 00:00:00', 'ts_started' => '2020-03-02 12:00:00'],
+            ['idsite' => 1, 'date1' => '2020-02-03', 'date2' => '2020-02-03', 'period' => 1, 'name' => 'donedone', 'value' => 5, 'status' => 1, 'ts_invalidated' => '2020-03-01 00:00:00', 'ts_started' => '2020-03-01 03:00:00'],
+        ]);
+
+        $this->model->resetFailedArchivingJobs();
+
+        $idinvalidationStatus = Db::fetchAll('SELECT idinvalidation, status FROM ' . Common::prefixTable('archive_invalidations'));
+
+        $expected = [
+            ['idinvalidation' => 1, 'status' => 0],
+            ['idinvalidation' => 2, 'status' => 0],
+            ['idinvalidation' => 3, 'status' => 0],
+            ['idinvalidation' => 4, 'status' => 1],
+            ['idinvalidation' => 5, 'status' => 0],
+        ];
+
+        $this->assertEquals($expected, $idinvalidationStatus);
+    }
+
     public function test_insertNewArchiveId()
     {
         $this->assertAllocatedArchiveId(1);
@@ -509,7 +536,9 @@ class ModelTest extends IntegrationTestCase
         foreach ($archivesToInsert as $archive) {
             $table = ArchiveTableCreator::getNumericTable(Date::factory($archive['date1']));
             $sql = "INSERT INTO `$table` (idarchive, idsite, date1, date2, period, `name`, `value`) VALUES (?, ?, ?, ?, ?, ?, ?)";
-            Db::query($sql, [$idarchive, 1, $archive['date1'], $archive['date2'], $archive['period'], $archive['name'], $archive['value']]);
+            Db::query($sql, [
+                $idarchive, 1, $archive['date1'], $archive['date2'], $archive['period'], $archive['name'], $archive['value'],
+            ]);
 
             ++$idarchive;
         }
@@ -519,8 +548,11 @@ class ModelTest extends IntegrationTestCase
     {
         $table = Common::prefixTable('archive_invalidations');
         foreach ($invalidations as $invalidation) {
-            $sql = "INSERT INTO `$table` (idsite, date1, date2, period, `name`) VALUES (?, ?, ?, ?, ?)";
-            Db::query($sql, [$invalidation['idsite'] ?? 1, $invalidation['date1'], $invalidation['date2'], $invalidation['period'], $invalidation['name']]);
+            $sql = "INSERT INTO `$table` (idsite, date1, date2, period, `name`, status, ts_invalidated, ts_started) VALUES (?, ?, ?, ?, ?, ?, ?, ?)";
+            Db::query($sql, [
+                $invalidation['idsite'] ?? 1, $invalidation['date1'], $invalidation['date2'], $invalidation['period'], $invalidation['name'],
+                $invalidation['status'] ?? 0, $invalidation['ts_invalidated'] ?? null, $invalidation['ts_started'] ?? null,
+            ]);
         }
     }
 }