diff options
author | diosmosis <diosmosis@users.noreply.github.com> | 2020-11-12 06:20:07 +0300 |
---|---|---|
committer | GitHub <noreply@github.com> | 2020-11-12 06:20:07 +0300 |
commit | 0e7530e93f0ada22024e7226badc1d94c8309f3c (patch) | |
tree | 412d841c666ebb90da3dbfdcf887f48b8f2b4028 | |
parent | 4039709af923620063f14adf1817015efc8da375 (diff) |
Better detect failed archiving jobs. (#16701)4.0.0-rc3
-rw-r--r-- | core/CronArchive.php | 7 | ||||
-rw-r--r-- | core/DataAccess/Model.php | 17 | ||||
-rw-r--r-- | core/Db/Schema/Mysql.php | 1 | ||||
-rw-r--r-- | core/Updates/4.0.0-rc3.php | 44 | ||||
-rw-r--r-- | core/Version.php | 2 | ||||
-rw-r--r-- | tests/PHPUnit/Integration/DataAccess/ModelTest.php | 38 |
6 files changed, 104 insertions, 5 deletions
diff --git a/core/CronArchive.php b/core/CronArchive.php index 358e6bfa78..4f36b4e485 100644 --- a/core/CronArchive.php +++ b/core/CronArchive.php @@ -345,6 +345,13 @@ class CronArchive return; } + $failedJobs = $this->model->resetFailedArchivingJobs(); + if ($failedJobs) { + $this->logger->info("Found {failed} failed jobs (ts_invalidated older than 1 day), resetings status to try them again.", [ + 'failed' => $failedJobs, + ]); + } + $countOfProcesses = $this->getMaxConcurrentApiRequests(); $queueConsumer = new QueueConsumer($this->logger, $this->websiteIdArchiveList, $countOfProcesses, $pid, diff --git a/core/DataAccess/Model.php b/core/DataAccess/Model.php index ed09659fae..6470305281 100644 --- a/core/DataAccess/Model.php +++ b/core/DataAccess/Model.php @@ -632,7 +632,7 @@ class Model $table = Common::prefixTable('archive_invalidations'); // set archive value to in progress if not set already - $statement = Db::query("UPDATE `$table` SET `status` = ? WHERE idinvalidation = ? AND status = ?", [ + $statement = Db::query("UPDATE `$table` SET `status` = ? AND ts_started = NOW() WHERE idinvalidation = ? AND status = ?", [ ArchiveInvalidator::INVALIDATION_STATUS_IN_PROGRESS, $invalidation['idinvalidation'], ArchiveInvalidator::INVALIDATION_STATUS_QUEUED, @@ -867,4 +867,19 @@ class Model $sql = "UPDATE $table SET status = " . ArchiveInvalidator::INVALIDATION_STATUS_QUEUED . " WHERE idinvalidation = ?"; Db::query($sql, [$idinvalidation]); } + + public function resetFailedArchivingJobs() + { + $table = Common::prefixTable('archive_invalidations'); + $sql = "UPDATE $table SET status = ? WHERE status = ? AND ts_started IS NOT NULL AND ts_started < ?"; + + $bind = [ + ArchiveInvalidator::INVALIDATION_STATUS_QUEUED, + ArchiveInvalidator::INVALIDATION_STATUS_IN_PROGRESS, + Date::now()->subDay(1)->getDatetime(), + ]; + + $query = Db::query($sql, $bind); + return $query->rowCount(); + } } diff --git a/core/Db/Schema/Mysql.php b/core/Db/Schema/Mysql.php index dfa14e4373..61b133d319 100644 --- a/core/Db/Schema/Mysql.php +++ b/core/Db/Schema/Mysql.php @@ -318,6 +318,7 @@ class Mysql implements SchemaInterface date2 DATE NOT NULL, period TINYINT UNSIGNED NOT NULL, ts_invalidated DATETIME NULL, + ts_started DATETIME NULL, status TINYINT(1) UNSIGNED DEFAULT 0, `report` VARCHAR(255) NULL, PRIMARY KEY(idinvalidation), diff --git a/core/Updates/4.0.0-rc3.php b/core/Updates/4.0.0-rc3.php new file mode 100644 index 0000000000..86c9ca0189 --- /dev/null +++ b/core/Updates/4.0.0-rc3.php @@ -0,0 +1,44 @@ +<?php +/** + * Matomo - free/libre analytics platform + * + * @link https://matomo.org + * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later + * + */ + +namespace Piwik\Updates; + +use Piwik\Config; +use Piwik\Updater; +use Piwik\Updates as PiwikUpdates; +use Piwik\Updater\Migration\Factory as MigrationFactory; + +/** + * Update for version 4.0.0-b3. + */ +class Updates_4_0_0_rc3 extends PiwikUpdates +{ + /** + * @var MigrationFactory + */ + private $migration; + + public function __construct(MigrationFactory $factory) + { + $this->migration = $factory; + } + + public function getMigrations(Updater $updater) + { + $migrations = []; + $migrations[] = $this->migration->db->addColumn('archive_invalidations', 'ts_started', 'DATETIME NULL'); + return $migrations; + } + + public function doUpdate(Updater $updater) + { + $updater->executeMigrations(__FILE__, $this->getMigrations($updater)); + } + +} diff --git a/core/Version.php b/core/Version.php index 9e4e5cbe2c..9bcba1327e 100644 --- a/core/Version.php +++ b/core/Version.php @@ -20,7 +20,7 @@ final class Version * The current Matomo version. * @var string */ - const VERSION = '4.0.0-rc2'; + const VERSION = '4.0.0-rc3'; const MAJOR_VERSION = 4; public function isStableVersion($version) diff --git a/tests/PHPUnit/Integration/DataAccess/ModelTest.php b/tests/PHPUnit/Integration/DataAccess/ModelTest.php index 532b0e48f7..15305796ee 100644 --- a/tests/PHPUnit/Integration/DataAccess/ModelTest.php +++ b/tests/PHPUnit/Integration/DataAccess/ModelTest.php @@ -38,6 +38,33 @@ class ModelTest extends IntegrationTestCase $this->model->createArchiveTable($this->tableName, 'archive_numeric'); } + public function test_resetFailedArchivingJobs_updatesCorrectStatuses() + { + Date::$now = strtotime('2020-03-03 04:00:00'); + + $this->insertInvalidations([ + ['idsite' => 1, 'date1' => '2020-02-03', 'date2' => '2020-02-03', 'period' => 1, 'name' => 'done', 'value' => 1, 'status' => 1, 'ts_invalidated' => '2020-03-01 00:00:00', 'ts_started' => '2020-03-02 03:00:00'], + ['idsite' => 2, 'date1' => '2020-02-03', 'date2' => '2020-02-03', 'period' => 1, 'name' => 'done.Plugin', 'value' => 2, 'status' => 0, 'ts_invalidated' => '2020-03-01 00:00:00', 'ts_started' => '2020-03-02 03:00:00'], + ['idsite' => 1, 'date1' => '2020-02-03', 'date2' => '2020-02-03', 'period' => 1, 'name' => 'doneblablah', 'value' => 3, 'status' => 0, 'ts_invalidated' => '2020-03-01 00:00:00', 'ts_started' => '2020-03-03 00:00:00'], + ['idsite' => 3, 'date1' => '2020-02-03', 'date2' => '2020-02-03', 'period' => 1, 'name' => 'donebluhbluh', 'value' => 4, 'status' => 1, 'ts_invalidated' => '2020-03-01 00:00:00', 'ts_started' => '2020-03-02 12:00:00'], + ['idsite' => 1, 'date1' => '2020-02-03', 'date2' => '2020-02-03', 'period' => 1, 'name' => 'donedone', 'value' => 5, 'status' => 1, 'ts_invalidated' => '2020-03-01 00:00:00', 'ts_started' => '2020-03-01 03:00:00'], + ]); + + $this->model->resetFailedArchivingJobs(); + + $idinvalidationStatus = Db::fetchAll('SELECT idinvalidation, status FROM ' . Common::prefixTable('archive_invalidations')); + + $expected = [ + ['idinvalidation' => 1, 'status' => 0], + ['idinvalidation' => 2, 'status' => 0], + ['idinvalidation' => 3, 'status' => 0], + ['idinvalidation' => 4, 'status' => 1], + ['idinvalidation' => 5, 'status' => 0], + ]; + + $this->assertEquals($expected, $idinvalidationStatus); + } + public function test_insertNewArchiveId() { $this->assertAllocatedArchiveId(1); @@ -509,7 +536,9 @@ class ModelTest extends IntegrationTestCase foreach ($archivesToInsert as $archive) { $table = ArchiveTableCreator::getNumericTable(Date::factory($archive['date1'])); $sql = "INSERT INTO `$table` (idarchive, idsite, date1, date2, period, `name`, `value`) VALUES (?, ?, ?, ?, ?, ?, ?)"; - Db::query($sql, [$idarchive, 1, $archive['date1'], $archive['date2'], $archive['period'], $archive['name'], $archive['value']]); + Db::query($sql, [ + $idarchive, 1, $archive['date1'], $archive['date2'], $archive['period'], $archive['name'], $archive['value'], + ]); ++$idarchive; } @@ -519,8 +548,11 @@ class ModelTest extends IntegrationTestCase { $table = Common::prefixTable('archive_invalidations'); foreach ($invalidations as $invalidation) { - $sql = "INSERT INTO `$table` (idsite, date1, date2, period, `name`) VALUES (?, ?, ?, ?, ?)"; - Db::query($sql, [$invalidation['idsite'] ?? 1, $invalidation['date1'], $invalidation['date2'], $invalidation['period'], $invalidation['name']]); + $sql = "INSERT INTO `$table` (idsite, date1, date2, period, `name`, status, ts_invalidated, ts_started) VALUES (?, ?, ?, ?, ?, ?, ?, ?)"; + Db::query($sql, [ + $invalidation['idsite'] ?? 1, $invalidation['date1'], $invalidation['date2'], $invalidation['period'], $invalidation['name'], + $invalidation['status'] ?? 0, $invalidation['ts_invalidated'] ?? null, $invalidation['ts_started'] ?? null, + ]); } } } |