Add methods to remove invalidations. (#16400)

* Add methods to remove invalidations.

* Add check if site exists to beginning of queue consumers loop to find archive data

* Delete invalidations w/ no site in task and for site when deleting.

* fix test

* fix test

* another test fix

8 files changed, 192 insertions, 1 deletions

diff --git a/core/Archive/ArchiveInvalidator.php b/core/Archive/ArchiveInvalidator.php
index 709a79e60f..5576d78fac 100644
--- a/core/Archive/ArchiveInvalidator.php
+++ b/core/Archive/ArchiveInvalidator.php
@@ -506,6 +506,24 @@ class ArchiveInvalidator
     }
 
     /**
+     * Remove invalidations for a specific report or all invalidations for a specific plugin. If your plugin supports
+     * archiving data in the past, you may want to call this method to remove any pending invalidations if, for example,
+     * your plugin is deactivated or a report deleted.
+     *
+     * @param int $idSite
+     * @param string $string
+     * @param string|null $report
+     */
+    public function removeInvalidations($idSite, $plugin, $report = null)
+    {
+        if (empty($report)) {
+            $this->model->removeInvalidationsLike($idSite, $plugin);
+        } else {
+            $this->model->removeInvalidations($idSite, $plugin, $report);
+        }
+    }
+
+    /**
      * @param int[] $idSites
      * @param string[][][] $dates
      * @throws \Exception
diff --git a/core/CronArchive/QueueConsumer.php b/core/CronArchive/QueueConsumer.php
index 2d64701ef1..4af3afbb7c 100644
--- a/core/CronArchive/QueueConsumer.php
+++ b/core/CronArchive/QueueConsumer.php
@@ -22,6 +22,7 @@ use Piwik\Period;
 use Piwik\Period\Factory as PeriodFactory;
 use Piwik\Piwik;
 use Piwik\Plugin\Manager;
+use Piwik\Plugins\SitesManager\API;
 use Piwik\Segment;
 use Piwik\Site;
 use Piwik\Timer;
@@ -122,6 +123,12 @@ class QueueConsumer
 
     public function getNextArchivesToProcess()
     {
+        // in case a site is deleted while archiving is running
+        if (!empty($this->idSite) && !$this->isSiteExists($this->idSite)) {
+            $this->logger->debug("Site ID = {$this->idSite} was deleted during archiving process, moving on.");
+            $this->idSite = null;
+        }
+
         if (empty($this->idSite)) {
             $this->idSite = $this->getNextIdSiteToArchive();
             if (empty($this->idSite)) { // no sites left to archive, stop
@@ -541,4 +548,10 @@ class QueueConsumer
         $idArchive = $archiveIdAndVisits[0];
         return !empty($idArchive);
     }
+
+    private function isSiteExists($idSite)
+    {
+        $site = API::getInstance()->getSiteFromId($idSite);
+        return !empty($site);
+    }
 }
\ No newline at end of file
diff --git a/core/DataAccess/Model.php b/core/DataAccess/Model.php
index a06e488696..b623d7fc24 100644
--- a/core/DataAccess/Model.php
+++ b/core/DataAccess/Model.php
@@ -699,6 +699,22 @@ class Model
         Db::query($sql);
     }
 
+    public function removeInvalidationsLike($idSite, $start)
+    {
+        $table = Common::prefixTable('archive_invalidations');
+        $sql = "DELETE FROM `$table` WHERE idsite = ? AND `name` LIKE ?";
+
+        Db::query($sql, [$idSite, 'done.' . str_replace('_', "\\_", $start) . '%']);
+    }
+
+    public function removeInvalidations($idSite, $plugin, $report)
+    {
+        $table = Common::prefixTable('archive_invalidations');
+        $sql = "DELETE FROM `$table` WHERE idsite = ? AND `name` = ? AND report = ?";
+
+        Db::query($sql, [$idSite, 'done.' . $plugin, $report]);
+    }
+
     /**
      * Returns true if there is an archive that exists that can be used when aggregating an archive for $period.
      *
@@ -728,4 +744,22 @@ class Model
         }
         return false;
     }
+
+    public function deleteInvalidationsForSites(array $idSites)
+    {
+        $idSites = array_map('intval', $idSites);
+
+        $table = Common::prefixTable('archive_invalidations');
+        $sql = "DELETE FROM `$table` WHERE idsite IN (" . implode(',', $idSites) . ")";
+
+        Db::query($sql);
+    }
+
+    public function deleteInvalidationsForDeletedSites()
+    {
+        $siteTable = Common::prefixTable('site');
+        $table = Common::prefixTable('archive_invalidations');
+        $sql = "DELETE a FROM `$table` a LEFT JOIN `$siteTable` s ON a.idsite = s.idsite WHERE s.idsite IS NULL";
+        Db::query($sql);
+    }
 }
diff --git a/plugins/CoreAdminHome/Tasks.php b/plugins/CoreAdminHome/Tasks.php
index 21008b78a3..9cccf23aba 100644
--- a/plugins/CoreAdminHome/Tasks.php
+++ b/plugins/CoreAdminHome/Tasks.php
@@ -18,6 +18,7 @@ use Piwik\Config;
 use Piwik\Container\StaticContainer;
 use Piwik\CronArchive;
 use Piwik\DataAccess\ArchiveTableCreator;
+use Piwik\DataAccess\Model as CoreModel;
 use Piwik\Date;
 use Piwik\Db;
 use Piwik\Http;
@@ -75,6 +76,7 @@ class Tasks extends \Piwik\Plugin\Tasks
 
         // general data purge on invalidated archive records, executed daily
         $this->daily('purgeInvalidatedArchives', null, self::LOW_PRIORITY);
+        $this->daily('purgeInvalidationsForDeletedSites', null, self::LOW_PRIORITY);
 
         $this->weekly('purgeOrphanedArchives', null, self::NORMAL_PRIORITY);
 
@@ -92,6 +94,12 @@ class Tasks extends \Piwik\Plugin\Tasks
         $this->scheduleTrackingCodeReminderChecks();
     }
 
+    public function purgeInvalidationsForDeletedSites()
+    {
+        $coreModel = new CoreModel();
+        $coreModel->deleteInvalidationsForDeletedSites();
+    }
+
     public function deleteOldFingerprintSalts()
     {
         StaticContainer::get(FingerprintSalt::class)->deleteOldSalts();
diff --git a/plugins/CoreAdminHome/tests/Integration/TasksTest.php b/plugins/CoreAdminHome/tests/Integration/TasksTest.php
index e98791a131..9edcbc5fcc 100644
--- a/plugins/CoreAdminHome/tests/Integration/TasksTest.php
+++ b/plugins/CoreAdminHome/tests/Integration/TasksTest.php
@@ -136,6 +136,7 @@ class TasksTest extends IntegrationTestCase
             'deleteOldFingerprintSalts.',
             'purgeOutdatedArchives.',
             'purgeInvalidatedArchives.',
+            'purgeInvalidationsForDeletedSites.',
             'purgeOrphanedArchives.',
             'optimizeArchiveTable.',
             'cleanupTrackingFailures.',
diff --git a/plugins/SitesManager/API.php b/plugins/SitesManager/API.php
index ef6f85c8ca..d52003a71f 100644
--- a/plugins/SitesManager/API.php
+++ b/plugins/SitesManager/API.php
@@ -34,6 +34,7 @@ use Piwik\Tracker\TrackerCodeGenerator;
 use Piwik\Measurable\Type;
 use Piwik\Url;
 use Piwik\UrlHelper;
+use Piwik\DataAccess\Model as CoreModel;
 
 /**
  * The SitesManager API gives you full control on Websites in Matomo (create, update and delete), and many methods to retrieve websites based on various attributes.
@@ -782,6 +783,9 @@ class API extends \Piwik\Plugin\API
 
         $this->getModel()->deleteSite($idSite);
 
+        $coreModel = new CoreModel();
+        $coreModel->deleteInvalidationsForSites([$idSite]);
+
         /**
          * Triggered after a site has been deleted.
          *
diff --git a/tests/PHPUnit/Integration/DataAccess/ArchiveInvalidatorTest.php b/tests/PHPUnit/Integration/DataAccess/ArchiveInvalidatorTest.php
index e49a2c9699..506311287e 100644
--- a/tests/PHPUnit/Integration/DataAccess/ArchiveInvalidatorTest.php
+++ b/tests/PHPUnit/Integration/DataAccess/ArchiveInvalidatorTest.php
@@ -82,6 +82,82 @@ class ArchiveInvalidatorTest extends IntegrationTestCase
         $this->invalidator = new ArchiveInvalidator(new Model(), StaticContainer::get(ArchivingStatus::class));
     }
 
+    public function test_removeInvalidations_removesAllForPlugin()
+    {
+        $this->insertInvalidations([
+            ['name' => 'done.MyPlugin', 'idsite' => 1, 'date1' => '2012-03-04', 'date2' => '2015-03-04', 'period' => 1, 'report' => 'myReport'],
+            ['name' => 'done.MyPlugin', 'idsite' => 1, 'date1' => '2012-03-04', 'date2' => '2015-03-04', 'period' => 1, 'report' => null],
+            ['name' => 'done.MyOtherPlugin', 'idsite' => 1, 'date1' => '2012-03-04', 'date2' => '2015-05-05', 'period' => 1, 'report' => ''],
+            ['name' => 'done', 'idsite' => 1, 'date1' => '2012-03-04', 'date2' => '2015-05-05', 'period' => 1, 'report' => ''],
+            ['name' => 'done.MyPlugin', 'idsite' => 1, 'date1' => '2012-03-04', 'date2' => '2015-03-04', 'period' => 1, 'report' => 'myOtherReport'],
+        ]);
+
+        $this->invalidator->removeInvalidations($idSite = 1, 'MyPlugin');
+
+        $invalidations = $this->getInvalidatedArchiveTableEntries();
+        $expectedInvalidations = [
+            ['idarchive' => null, 'idsite' => 1, 'date1' => '2012-03-04', 'date2' => '2015-05-05', 'period' => 1, 'name' => 'done.MyOtherPlugin', 'report' => null],
+            ['idarchive' => null, 'idsite' => 1, 'date1' => '2012-03-04', 'date2' => '2015-05-05', 'period' => 1, 'name' => 'done', 'report' => null],
+        ];
+
+        $this->assertEquals($expectedInvalidations, $invalidations);
+    }
+
+    public function test_removeInvalidations_removesAllForSingleReport()
+    {
+        $this->insertInvalidations([
+            ['name' => 'done.MyPlugin', 'idsite' => 1, 'date1' => '2012-03-04', 'date2' => '2015-03-04', 'period' => 1, 'report' => 'myReport'],
+            ['name' => 'done.MyPlugin', 'idsite' => 1, 'date1' => '2012-03-04', 'date2' => '2015-03-04', 'period' => 1, 'report' => null],
+            ['name' => 'done.MyOtherPlugin', 'idsite' => 1, 'date1' => '2012-03-04', 'date2' => '2015-05-05', 'period' => 1, 'report' => ''],
+            ['name' => 'done', 'idsite' => 1, 'date1' => '2012-03-04', 'date2' => '2015-05-05', 'period' => 1, 'report' => ''],
+            ['name' => 'done.MyPlugin', 'idsite' => 1, 'date1' => '2012-03-04', 'date2' => '2015-03-04', 'period' => 1, 'report' => 'myOtherReport'],
+        ]);
+
+        $this->invalidator->removeInvalidations($idSite = 1, 'MyPlugin', 'myReport');
+
+        $invalidations = $this->getInvalidatedArchiveTableEntries();
+        $expectedInvalidations = [
+            [
+                'idarchive' => NULL,
+                'idsite' => '1',
+                'date1' => '2012-03-04',
+                'date2' => '2015-03-04',
+                'period' => '1',
+                'name' => 'done.MyPlugin',
+                'report' => NULL,
+            ],
+            [
+                'idarchive' => NULL,
+                'idsite' => '1',
+                'date1' => '2012-03-04',
+                'date2' => '2015-05-05',
+                'period' => '1',
+                'name' => 'done.MyOtherPlugin',
+                'report' => '',
+            ],
+            [
+                'idarchive' => NULL,
+                'idsite' => '1',
+                'date1' => '2012-03-04',
+                'date2' => '2015-05-05',
+                'period' => '1',
+                'name' => 'done',
+                'report' => '',
+            ],
+            [
+                'idarchive' => NULL,
+                'idsite' => '1',
+                'date1' => '2012-03-04',
+                'date2' => '2015-03-04',
+                'period' => '1',
+                'name' => 'done.MyPlugin',
+                'report' => 'myOtherReport',
+            ],
+        ];
+
+        $this->assertEquals($expectedInvalidations, $invalidations);
+    }
+
     public function test_rememberToInvalidateArchivedReportsLater_shouldCreateAnEntryInCaseThereIsNoneYet()
     {
         //Updated for change to allow for multiple transactions to invalidate the same report without deadlock.
@@ -1334,4 +1410,20 @@ class ArchiveInvalidatorTest extends IntegrationTestCase
         parent::configureFixture($fixture);
         $fixture->createSuperUser = true;
     }
+
+    private function insertInvalidations(array $invalidations)
+    {
+        $table = Common::prefixTable('archive_invalidations');
+        foreach ($invalidations as $invalidation) {
+            $sql = "INSERT INTO $table (name, idsite, date1, date2, period, ts_invalidated, report) VALUES (?, ?, ?, ?, ?, NOW(), ?)";
+            Db::query($sql, [
+                $invalidation['name'],
+                $invalidation['idsite'],
+                $invalidation['date1'],
+                $invalidation['date2'],
+                $invalidation['period'],
+                $invalidation['report'],
+            ]);
+        }
+    }
 }
diff --git a/tests/PHPUnit/Integration/DataAccess/ModelTest.php b/tests/PHPUnit/Integration/DataAccess/ModelTest.php
index 42f67fea92..532b0e48f7 100644
--- a/tests/PHPUnit/Integration/DataAccess/ModelTest.php
+++ b/tests/PHPUnit/Integration/DataAccess/ModelTest.php
@@ -14,6 +14,7 @@ use Piwik\DataAccess\ArchiveWriter;
 use Piwik\Date;
 use Piwik\Db;
 use Piwik\Period\Factory;
+use Piwik\Tests\Framework\Fixture;
 use Piwik\Tests\Framework\TestCase\IntegrationTestCase;
 use Piwik\DataAccess\Model;
 
@@ -482,6 +483,26 @@ class ModelTest extends IntegrationTestCase
         $this->assertEquals($expected, $actual);
     }
 
+    public function test_deleteInvalidationsForDeletedSites()
+    {
+        Fixture::createWebsite('2014-01-01 00:00:00');
+
+        $this->insertInvalidations([
+            ['idsite' => 1, 'date1' => '2014-02-03', 'date2' => '2014-02-03', 'period' => 1, 'name' => 'done'],
+            ['idsite' => 2, 'date1' => '2014-02-01', 'date2' => '2014-02-28', 'period' => 2, 'name' => 'done'],
+            ['idsite' => 2, 'date1' => '2014-02-01', 'date2' => '2014-02-01', 'period' => 1, 'name' => 'done'],
+            ['idsite' => 3, 'date1' => '2014-02-01', 'date2' => '2014-02-01', 'period' => 1, 'name' => 'done'],
+        ]);
+
+        $this->model->deleteInvalidationsForDeletedSites();
+
+        $invalidations = Db::fetchAll("SELECT idsite, idinvalidation FROM " . Common::prefixTable('archive_invalidations') .
+            " ORDER BY idinvalidation ASC");
+        $this->assertEquals([
+            ['idsite' => 1, 'idinvalidation' => 1],
+        ], $invalidations);
+    }
+
     private function insertArchiveData($archivesToInsert)
     {
         $idarchive = 1;
@@ -499,7 +520,7 @@ class ModelTest extends IntegrationTestCase
         $table = Common::prefixTable('archive_invalidations');
         foreach ($invalidations as $invalidation) {
             $sql = "INSERT INTO `$table` (idsite, date1, date2, period, `name`) VALUES (?, ?, ?, ?, ?)";
-            Db::query($sql, [1, $invalidation['date1'], $invalidation['date2'], $invalidation['period'], $invalidation['name']]);
+            Db::query($sql, [$invalidation['idsite'] ?? 1, $invalidation['date1'], $invalidation['date2'], $invalidation['period'], $invalidation['name']]);
         }
     }
 }