diff options
author | diosmosis <diosmosis@users.noreply.github.com> | 2020-09-11 04:19:41 +0300 |
---|---|---|
committer | GitHub <noreply@github.com> | 2020-09-11 04:19:41 +0300 |
commit | 6b70e8cc7a12e596bfa08e35db52531ce989c2f2 (patch) | |
tree | 151cda62467fb0c5266adb6e901c3bd50499fb2b | |
parent | d01085cec58a437e30b0dd4b149c52fc3e19abaa (diff) |
Add methods to remove invalidations. (#16400)
* Add methods to remove invalidations.
* Add check if site exists to beginning of queue consumers loop to find archive data
* Delete invalidations w/ no site in task and for site when deleting.
* fix test
* fix test
* another test fix
-rw-r--r-- | core/Archive/ArchiveInvalidator.php | 18 | ||||
-rw-r--r-- | core/CronArchive/QueueConsumer.php | 13 | ||||
-rw-r--r-- | core/DataAccess/Model.php | 34 | ||||
-rw-r--r-- | plugins/CoreAdminHome/Tasks.php | 8 | ||||
-rw-r--r-- | plugins/CoreAdminHome/tests/Integration/TasksTest.php | 1 | ||||
-rw-r--r-- | plugins/SitesManager/API.php | 4 | ||||
-rw-r--r-- | tests/PHPUnit/Integration/DataAccess/ArchiveInvalidatorTest.php | 92 | ||||
-rw-r--r-- | tests/PHPUnit/Integration/DataAccess/ModelTest.php | 23 |
8 files changed, 192 insertions, 1 deletions
diff --git a/core/Archive/ArchiveInvalidator.php b/core/Archive/ArchiveInvalidator.php index 709a79e60f..5576d78fac 100644 --- a/core/Archive/ArchiveInvalidator.php +++ b/core/Archive/ArchiveInvalidator.php @@ -506,6 +506,24 @@ class ArchiveInvalidator } /** + * Remove invalidations for a specific report or all invalidations for a specific plugin. If your plugin supports + * archiving data in the past, you may want to call this method to remove any pending invalidations if, for example, + * your plugin is deactivated or a report deleted. + * + * @param int $idSite + * @param string $string + * @param string|null $report + */ + public function removeInvalidations($idSite, $plugin, $report = null) + { + if (empty($report)) { + $this->model->removeInvalidationsLike($idSite, $plugin); + } else { + $this->model->removeInvalidations($idSite, $plugin, $report); + } + } + + /** * @param int[] $idSites * @param string[][][] $dates * @throws \Exception diff --git a/core/CronArchive/QueueConsumer.php b/core/CronArchive/QueueConsumer.php index 2d64701ef1..4af3afbb7c 100644 --- a/core/CronArchive/QueueConsumer.php +++ b/core/CronArchive/QueueConsumer.php @@ -22,6 +22,7 @@ use Piwik\Period; use Piwik\Period\Factory as PeriodFactory; use Piwik\Piwik; use Piwik\Plugin\Manager; +use Piwik\Plugins\SitesManager\API; use Piwik\Segment; use Piwik\Site; use Piwik\Timer; @@ -122,6 +123,12 @@ class QueueConsumer public function getNextArchivesToProcess() { + // in case a site is deleted while archiving is running + if (!empty($this->idSite) && !$this->isSiteExists($this->idSite)) { + $this->logger->debug("Site ID = {$this->idSite} was deleted during archiving process, moving on."); + $this->idSite = null; + } + if (empty($this->idSite)) { $this->idSite = $this->getNextIdSiteToArchive(); if (empty($this->idSite)) { // no sites left to archive, stop @@ -541,4 +548,10 @@ class QueueConsumer $idArchive = $archiveIdAndVisits[0]; return !empty($idArchive); } + + private function isSiteExists($idSite) + { + $site = API::getInstance()->getSiteFromId($idSite); + return !empty($site); + } }
\ No newline at end of file diff --git a/core/DataAccess/Model.php b/core/DataAccess/Model.php index a06e488696..b623d7fc24 100644 --- a/core/DataAccess/Model.php +++ b/core/DataAccess/Model.php @@ -699,6 +699,22 @@ class Model Db::query($sql); } + public function removeInvalidationsLike($idSite, $start) + { + $table = Common::prefixTable('archive_invalidations'); + $sql = "DELETE FROM `$table` WHERE idsite = ? AND `name` LIKE ?"; + + Db::query($sql, [$idSite, 'done.' . str_replace('_', "\\_", $start) . '%']); + } + + public function removeInvalidations($idSite, $plugin, $report) + { + $table = Common::prefixTable('archive_invalidations'); + $sql = "DELETE FROM `$table` WHERE idsite = ? AND `name` = ? AND report = ?"; + + Db::query($sql, [$idSite, 'done.' . $plugin, $report]); + } + /** * Returns true if there is an archive that exists that can be used when aggregating an archive for $period. * @@ -728,4 +744,22 @@ class Model } return false; } + + public function deleteInvalidationsForSites(array $idSites) + { + $idSites = array_map('intval', $idSites); + + $table = Common::prefixTable('archive_invalidations'); + $sql = "DELETE FROM `$table` WHERE idsite IN (" . implode(',', $idSites) . ")"; + + Db::query($sql); + } + + public function deleteInvalidationsForDeletedSites() + { + $siteTable = Common::prefixTable('site'); + $table = Common::prefixTable('archive_invalidations'); + $sql = "DELETE a FROM `$table` a LEFT JOIN `$siteTable` s ON a.idsite = s.idsite WHERE s.idsite IS NULL"; + Db::query($sql); + } } diff --git a/plugins/CoreAdminHome/Tasks.php b/plugins/CoreAdminHome/Tasks.php index 21008b78a3..9cccf23aba 100644 --- a/plugins/CoreAdminHome/Tasks.php +++ b/plugins/CoreAdminHome/Tasks.php @@ -18,6 +18,7 @@ use Piwik\Config; use Piwik\Container\StaticContainer; use Piwik\CronArchive; use Piwik\DataAccess\ArchiveTableCreator; +use Piwik\DataAccess\Model as CoreModel; use Piwik\Date; use Piwik\Db; use Piwik\Http; @@ -75,6 +76,7 @@ class Tasks extends \Piwik\Plugin\Tasks // general data purge on invalidated archive records, executed daily $this->daily('purgeInvalidatedArchives', null, self::LOW_PRIORITY); + $this->daily('purgeInvalidationsForDeletedSites', null, self::LOW_PRIORITY); $this->weekly('purgeOrphanedArchives', null, self::NORMAL_PRIORITY); @@ -92,6 +94,12 @@ class Tasks extends \Piwik\Plugin\Tasks $this->scheduleTrackingCodeReminderChecks(); } + public function purgeInvalidationsForDeletedSites() + { + $coreModel = new CoreModel(); + $coreModel->deleteInvalidationsForDeletedSites(); + } + public function deleteOldFingerprintSalts() { StaticContainer::get(FingerprintSalt::class)->deleteOldSalts(); diff --git a/plugins/CoreAdminHome/tests/Integration/TasksTest.php b/plugins/CoreAdminHome/tests/Integration/TasksTest.php index e98791a131..9edcbc5fcc 100644 --- a/plugins/CoreAdminHome/tests/Integration/TasksTest.php +++ b/plugins/CoreAdminHome/tests/Integration/TasksTest.php @@ -136,6 +136,7 @@ class TasksTest extends IntegrationTestCase 'deleteOldFingerprintSalts.', 'purgeOutdatedArchives.', 'purgeInvalidatedArchives.', + 'purgeInvalidationsForDeletedSites.', 'purgeOrphanedArchives.', 'optimizeArchiveTable.', 'cleanupTrackingFailures.', diff --git a/plugins/SitesManager/API.php b/plugins/SitesManager/API.php index ef6f85c8ca..d52003a71f 100644 --- a/plugins/SitesManager/API.php +++ b/plugins/SitesManager/API.php @@ -34,6 +34,7 @@ use Piwik\Tracker\TrackerCodeGenerator; use Piwik\Measurable\Type; use Piwik\Url; use Piwik\UrlHelper; +use Piwik\DataAccess\Model as CoreModel; /** * The SitesManager API gives you full control on Websites in Matomo (create, update and delete), and many methods to retrieve websites based on various attributes. @@ -782,6 +783,9 @@ class API extends \Piwik\Plugin\API $this->getModel()->deleteSite($idSite); + $coreModel = new CoreModel(); + $coreModel->deleteInvalidationsForSites([$idSite]); + /** * Triggered after a site has been deleted. * diff --git a/tests/PHPUnit/Integration/DataAccess/ArchiveInvalidatorTest.php b/tests/PHPUnit/Integration/DataAccess/ArchiveInvalidatorTest.php index e49a2c9699..506311287e 100644 --- a/tests/PHPUnit/Integration/DataAccess/ArchiveInvalidatorTest.php +++ b/tests/PHPUnit/Integration/DataAccess/ArchiveInvalidatorTest.php @@ -82,6 +82,82 @@ class ArchiveInvalidatorTest extends IntegrationTestCase $this->invalidator = new ArchiveInvalidator(new Model(), StaticContainer::get(ArchivingStatus::class)); } + public function test_removeInvalidations_removesAllForPlugin() + { + $this->insertInvalidations([ + ['name' => 'done.MyPlugin', 'idsite' => 1, 'date1' => '2012-03-04', 'date2' => '2015-03-04', 'period' => 1, 'report' => 'myReport'], + ['name' => 'done.MyPlugin', 'idsite' => 1, 'date1' => '2012-03-04', 'date2' => '2015-03-04', 'period' => 1, 'report' => null], + ['name' => 'done.MyOtherPlugin', 'idsite' => 1, 'date1' => '2012-03-04', 'date2' => '2015-05-05', 'period' => 1, 'report' => ''], + ['name' => 'done', 'idsite' => 1, 'date1' => '2012-03-04', 'date2' => '2015-05-05', 'period' => 1, 'report' => ''], + ['name' => 'done.MyPlugin', 'idsite' => 1, 'date1' => '2012-03-04', 'date2' => '2015-03-04', 'period' => 1, 'report' => 'myOtherReport'], + ]); + + $this->invalidator->removeInvalidations($idSite = 1, 'MyPlugin'); + + $invalidations = $this->getInvalidatedArchiveTableEntries(); + $expectedInvalidations = [ + ['idarchive' => null, 'idsite' => 1, 'date1' => '2012-03-04', 'date2' => '2015-05-05', 'period' => 1, 'name' => 'done.MyOtherPlugin', 'report' => null], + ['idarchive' => null, 'idsite' => 1, 'date1' => '2012-03-04', 'date2' => '2015-05-05', 'period' => 1, 'name' => 'done', 'report' => null], + ]; + + $this->assertEquals($expectedInvalidations, $invalidations); + } + + public function test_removeInvalidations_removesAllForSingleReport() + { + $this->insertInvalidations([ + ['name' => 'done.MyPlugin', 'idsite' => 1, 'date1' => '2012-03-04', 'date2' => '2015-03-04', 'period' => 1, 'report' => 'myReport'], + ['name' => 'done.MyPlugin', 'idsite' => 1, 'date1' => '2012-03-04', 'date2' => '2015-03-04', 'period' => 1, 'report' => null], + ['name' => 'done.MyOtherPlugin', 'idsite' => 1, 'date1' => '2012-03-04', 'date2' => '2015-05-05', 'period' => 1, 'report' => ''], + ['name' => 'done', 'idsite' => 1, 'date1' => '2012-03-04', 'date2' => '2015-05-05', 'period' => 1, 'report' => ''], + ['name' => 'done.MyPlugin', 'idsite' => 1, 'date1' => '2012-03-04', 'date2' => '2015-03-04', 'period' => 1, 'report' => 'myOtherReport'], + ]); + + $this->invalidator->removeInvalidations($idSite = 1, 'MyPlugin', 'myReport'); + + $invalidations = $this->getInvalidatedArchiveTableEntries(); + $expectedInvalidations = [ + [ + 'idarchive' => NULL, + 'idsite' => '1', + 'date1' => '2012-03-04', + 'date2' => '2015-03-04', + 'period' => '1', + 'name' => 'done.MyPlugin', + 'report' => NULL, + ], + [ + 'idarchive' => NULL, + 'idsite' => '1', + 'date1' => '2012-03-04', + 'date2' => '2015-05-05', + 'period' => '1', + 'name' => 'done.MyOtherPlugin', + 'report' => '', + ], + [ + 'idarchive' => NULL, + 'idsite' => '1', + 'date1' => '2012-03-04', + 'date2' => '2015-05-05', + 'period' => '1', + 'name' => 'done', + 'report' => '', + ], + [ + 'idarchive' => NULL, + 'idsite' => '1', + 'date1' => '2012-03-04', + 'date2' => '2015-03-04', + 'period' => '1', + 'name' => 'done.MyPlugin', + 'report' => 'myOtherReport', + ], + ]; + + $this->assertEquals($expectedInvalidations, $invalidations); + } + public function test_rememberToInvalidateArchivedReportsLater_shouldCreateAnEntryInCaseThereIsNoneYet() { //Updated for change to allow for multiple transactions to invalidate the same report without deadlock. @@ -1334,4 +1410,20 @@ class ArchiveInvalidatorTest extends IntegrationTestCase parent::configureFixture($fixture); $fixture->createSuperUser = true; } + + private function insertInvalidations(array $invalidations) + { + $table = Common::prefixTable('archive_invalidations'); + foreach ($invalidations as $invalidation) { + $sql = "INSERT INTO $table (name, idsite, date1, date2, period, ts_invalidated, report) VALUES (?, ?, ?, ?, ?, NOW(), ?)"; + Db::query($sql, [ + $invalidation['name'], + $invalidation['idsite'], + $invalidation['date1'], + $invalidation['date2'], + $invalidation['period'], + $invalidation['report'], + ]); + } + } } diff --git a/tests/PHPUnit/Integration/DataAccess/ModelTest.php b/tests/PHPUnit/Integration/DataAccess/ModelTest.php index 42f67fea92..532b0e48f7 100644 --- a/tests/PHPUnit/Integration/DataAccess/ModelTest.php +++ b/tests/PHPUnit/Integration/DataAccess/ModelTest.php @@ -14,6 +14,7 @@ use Piwik\DataAccess\ArchiveWriter; use Piwik\Date; use Piwik\Db; use Piwik\Period\Factory; +use Piwik\Tests\Framework\Fixture; use Piwik\Tests\Framework\TestCase\IntegrationTestCase; use Piwik\DataAccess\Model; @@ -482,6 +483,26 @@ class ModelTest extends IntegrationTestCase $this->assertEquals($expected, $actual); } + public function test_deleteInvalidationsForDeletedSites() + { + Fixture::createWebsite('2014-01-01 00:00:00'); + + $this->insertInvalidations([ + ['idsite' => 1, 'date1' => '2014-02-03', 'date2' => '2014-02-03', 'period' => 1, 'name' => 'done'], + ['idsite' => 2, 'date1' => '2014-02-01', 'date2' => '2014-02-28', 'period' => 2, 'name' => 'done'], + ['idsite' => 2, 'date1' => '2014-02-01', 'date2' => '2014-02-01', 'period' => 1, 'name' => 'done'], + ['idsite' => 3, 'date1' => '2014-02-01', 'date2' => '2014-02-01', 'period' => 1, 'name' => 'done'], + ]); + + $this->model->deleteInvalidationsForDeletedSites(); + + $invalidations = Db::fetchAll("SELECT idsite, idinvalidation FROM " . Common::prefixTable('archive_invalidations') . + " ORDER BY idinvalidation ASC"); + $this->assertEquals([ + ['idsite' => 1, 'idinvalidation' => 1], + ], $invalidations); + } + private function insertArchiveData($archivesToInsert) { $idarchive = 1; @@ -499,7 +520,7 @@ class ModelTest extends IntegrationTestCase $table = Common::prefixTable('archive_invalidations'); foreach ($invalidations as $invalidation) { $sql = "INSERT INTO `$table` (idsite, date1, date2, period, `name`) VALUES (?, ?, ?, ?, ?)"; - Db::query($sql, [1, $invalidation['date1'], $invalidation['date2'], $invalidation['period'], $invalidation['name']]); + Db::query($sql, [$invalidation['idsite'] ?? 1, $invalidation['date1'], $invalidation['date2'], $invalidation['period'], $invalidation['name']]); } } } |