Delayed rearchiving (#16702)

* Schedule rearchiving so potentially long running inserts into archive_invalidations do not slow UI

* make sure to apply rearchiving

* add file

* apply review feedback, add tests + get to pass

* apply review feedback and fix tests

2 files changed, 44 insertions, 2 deletions

diff --git a/tests/PHPUnit/Integration/CronArchiveTest.php b/tests/PHPUnit/Integration/CronArchiveTest.php
index 37b5c07153..acf239d784 100644
--- a/tests/PHPUnit/Integration/CronArchiveTest.php
+++ b/tests/PHPUnit/Integration/CronArchiveTest.php
@@ -444,6 +444,7 @@ NOTES
 ---------------------------
 START
 Starting Matomo reports archiving...
+Applying queued rearchiving...
 Start processing archives for site 1.
 Checking for queued invalidations...
   Will invalidate archived reports for 2019-12-12 for following websites ids: 1
@@ -602,6 +603,7 @@ LOG;
 ---------------------------
 START
 Starting Matomo reports archiving...
+Applying queued rearchiving...
 Start processing archives for site 1.
 Checking for queued invalidations...
   Today archive can be skipped due to no visits for idSite = 1, skipping invalidation...
diff --git a/tests/PHPUnit/Integration/DataAccess/ArchiveInvalidatorTest.php b/tests/PHPUnit/Integration/DataAccess/ArchiveInvalidatorTest.php
index 1b01c8ce73..be8ceb4f77 100644
--- a/tests/PHPUnit/Integration/DataAccess/ArchiveInvalidatorTest.php
+++ b/tests/PHPUnit/Integration/DataAccess/ArchiveInvalidatorTest.php
@@ -13,6 +13,7 @@ use Piwik\ArchiveProcessor\Rules;
 use Piwik\Common;
 use Piwik\Config;
 use Piwik\Container\StaticContainer;
+use Piwik\CronArchive\ReArchiveList;
 use Piwik\DataAccess\ArchiveTableCreator;
 use Piwik\DataAccess\ArchiveWriter;
 use Piwik\DataAccess\Model;
@@ -27,6 +28,7 @@ use Piwik\Tests\Framework\Fixture;
 use Piwik\Tests\Framework\TestCase\IntegrationTestCase;
 use Piwik\Archive\ArchiveInvalidator;
 use Piwik\Segment;
+use Psr\Log\NullLogger;
 
 /**
  * @group Archiver
@@ -79,7 +81,44 @@ class ArchiveInvalidatorTest extends IntegrationTestCase
     {
         parent::setUp();
 
-        $this->invalidator = new ArchiveInvalidator(new Model(), StaticContainer::get(ArchivingStatus::class));
+        $this->invalidator = new ArchiveInvalidator(new Model(), StaticContainer::get(ArchivingStatus::class), new NullLogger());
+    }
+
+    public function test_removeInvalidationsFromDistributedList_removesEntriesFromList_WhenNoPluginSpecified()
+    {
+        $this->invalidator->scheduleReArchiving([1,2,3], 'ExamplePlugin');
+        $this->invalidator->scheduleReArchiving([1,4,5], 'MyOtherPlugin');
+
+        $list = new ReArchiveList();
+        $list->add('badjson');
+
+        $this->invalidator->removeInvalidationsFromDistributedList([2,3]);
+
+        $items = $list->getAll();
+
+        $expected = [
+            '{"idSites":[1],"pluginName":"ExamplePlugin","report":null,"startDate":null}',
+            '{"idSites":[1,4,5],"pluginName":"MyOtherPlugin","report":null,"startDate":null}',
+        ];
+
+        $this->assertEquals($expected, $items);
+    }
+
+    public function test_removeInvalidationsFromDistributedList_removesEntriesFromList_WhenPluginNameIsSpecified()
+    {
+        $this->invalidator->scheduleReArchiving([1,2,3], 'ExamplePlugin');
+        $this->invalidator->scheduleReArchiving([1,4,5], 'MyOtherPlugin');
+
+        $this->invalidator->removeInvalidationsFromDistributedList([1,2,3], 'ExamplePlugin');
+
+        $list = new ReArchiveList();
+        $items = $list->getAll();
+
+        $expected = [
+            '{"idSites":[1,4,5],"pluginName":"MyOtherPlugin","report":null,"startDate":null}',
+        ];
+
+        $this->assertEquals($expected, $items);
     }
 
     public function test_removeInvalidations_removesAll_ifAllSitesSpecified()
@@ -1118,7 +1157,8 @@ class ArchiveInvalidatorTest extends IntegrationTestCase
 
         Config::getInstance()->General['rearchive_reports_in_past_last_n_months'] = 'last1';
 
-        $this->invalidator->reArchiveReport('all', 'VisitsSummary');
+        $this->invalidator->scheduleReArchiving('all', 'VisitsSummary');
+        $this->invalidator->applyScheduledReArchiving();
 
         $countInvalidations = Db::fetchOne("SELECT COUNT(*) FROM " . Common::prefixTable('archive_invalidations'));