Invalidate archives when visits removed via GDPR tool (#14399)

* Invalidate archives when visits removed via GDPR tool

* Remove message prompting user to manually invalidate the archives

* PR changes

* Ensure fixture datetimes are always reset after each test

* Update UI tests

12 files changed, 255 insertions, 20 deletions

diff --git a/plugins/PrivacyManager/Model/DataSubjects.php b/plugins/PrivacyManager/Model/DataSubjects.php
index 494cb335f2..6c5d7583c3 100644
--- a/plugins/PrivacyManager/Model/DataSubjects.php
+++ b/plugins/PrivacyManager/Model/DataSubjects.php
@@ -11,11 +11,14 @@ namespace Piwik\Plugins\PrivacyManager\Model;
 use Piwik\Columns\Dimension;
 use Piwik\Columns\Join\ActionNameJoin;
 use Piwik\Common;
+use Piwik\Container\StaticContainer;
+use Piwik\Date;
 use Piwik\Db;
 use Piwik\DbHelper;
 use Piwik\Metrics\Formatter;
 use Piwik\Piwik;
 use Piwik\Plugin\LogTablesProvider;
+use Piwik\Site;
 use Piwik\Tracker\LogTable;
 use Piwik\Tracker\PageUrl;
 
@@ -104,6 +107,8 @@ class DataSubjects
          */
         Piwik::postEvent('PrivacyManager.deleteDataSubjects', array(&$results, $visits));
 
+        $this->invalidateArchives($visits);
+
         $logTables = $this->getLogTablesToDeleteFrom();
         $deleteCounts = $this->deleteLogDataFrom($logTables, function ($tableToSelectFrom) use ($visits) {
             return $this->visitsToWhereAndBind($tableToSelectFrom, $visits);
@@ -114,6 +119,50 @@ class DataSubjects
         return $results;
     }
 
+    private function invalidateArchives($visits)
+    {
+        $datesToInvalidateByIdSite = $this->getDatesToInvalidate($visits);
+
+        $invalidator = StaticContainer::get('Piwik\Archive\ArchiveInvalidator');
+
+        foreach ($datesToInvalidateByIdSite as $idSite => $visitDates) {
+            foreach ($visitDates as $dateStr) {
+                $visitDate = Date::factory($dateStr);
+                $invalidator->rememberToInvalidateArchivedReportsLater($idSite, $visitDate);
+            }
+        }
+    }
+
+    private function getDatesToInvalidate($visits)
+    {
+        $idVisitsByIdSites = array();
+        foreach ($visits as $visit) {
+            $idSite = (int)$visit['idsite'];
+            if (!isset($idVisitsByIdSites[$idSite])) {
+                $idVisitsByIdSites[$idSite] = array();
+            }
+            $idVisitsByIdSites[$idSite][] = (int)$visit['idvisit'];
+        }
+
+        $datesToInvalidate = array();
+        foreach ($idVisitsByIdSites as $idSite => $idVisits) {
+            $timezone = Site::getTimezoneFor($idSite);
+
+            $sql = 'SELECT visit_last_action_time FROM '
+                . Common::prefixTable('log_visit') . ' WHERE idsite = ' . $idSite
+                . ' AND idvisit IN (' . implode(',', $idVisits) . ')';
+
+            $resultSet = Db::fetchAll($sql);
+            $dates = array();
+            foreach ($resultSet as $row) {
+                $date = Date::factory($row['visit_last_action_time'], $timezone);
+                $dates[$date->toString('Y-m-d')] = 1;
+            }
+            $datesToInvalidate[$idSite] = array_keys($dates);
+        }
+        return $datesToInvalidate;
+    }
+
     private function getLogTablesToDeleteFrom()
     {
         $logTables = $this->logTablesProvider->getAllLogTables();
diff --git a/plugins/PrivacyManager/angularjs/manage-gdpr/managegdpr.directive.html b/plugins/PrivacyManager/angularjs/manage-gdpr/managegdpr.directive.html
index 860573a8e6..e9fa7ea347 100644
--- a/plugins/PrivacyManager/angularjs/manage-gdpr/managegdpr.directive.html
+++ b/plugins/PrivacyManager/angularjs/manage-gdpr/managegdpr.directive.html
@@ -61,10 +61,6 @@
             In case you are exporting the data to exercise the right of access, please make sure the selected visits are actually performed by the data subject you want to export the data for.
             <br />
             <br />
-            When deleting data to exercise the right of erasure keep in mind that any data that will be deleted for this data subject might still be included in previously generated reports.
-            As many reports are aggregated this is in most cases not a problem unless you have for example URLs, Page Titles, Custom Variables or Custom Dimensions that include personal data. In this case
-            you may want to consider to <a href="https://matomo.org/faq/how-to/faq_155/" target="_blank" rel="noreferrer noopener">invalidate reports</a> after deleting these visits.
-            <br /><br />
             Please also note that any data will be only deleted from the Matomo database but not from your webserver logs. Also note that if you re-import any historical data, for example from logs, that any previously deleted data may be imported again.
 
             <br /><br />
diff --git a/plugins/PrivacyManager/tests/Fixtures/MultipleSitesMultipleVisitsFixture.php b/plugins/PrivacyManager/tests/Fixtures/MultipleSitesMultipleVisitsFixture.php
index e858b0cefe..2fb49aac45 100644
--- a/plugins/PrivacyManager/tests/Fixtures/MultipleSitesMultipleVisitsFixture.php
+++ b/plugins/PrivacyManager/tests/Fixtures/MultipleSitesMultipleVisitsFixture.php
@@ -8,9 +8,15 @@
 namespace Piwik\Plugins\PrivacyManager\tests\Fixtures;
 
 use Piwik\Common;
+use Piwik\DataAccess\ArchiveTableCreator;
+use Piwik\DataAccess\ArchiveWriter;
 use Piwik\Date;
 use Piwik\Db;
 use Piwik\DbHelper;
+use Piwik\Period\Day;
+use Piwik\Period\Month;
+use Piwik\Period\Week;
+use Piwik\Period\Year;
 use Piwik\Piwik;
 use Piwik\Plugins\UserCountry\LocationProvider;
 use Piwik\Tests\Framework\Fixture;
@@ -152,6 +158,7 @@ class MultipleSitesMultipleVisitsFixture extends Fixture
     public $dateTime = '2017-01-02 03:04:05';
     public $trackingTime = '2017-01-02 03:04:05';
     public $idSite = 1;
+    public $numVisitsPerIteration = 32;
     /**
      * @var \PiwikTracker
      */
@@ -318,6 +325,48 @@ class MultipleSitesMultipleVisitsFixture extends Fixture
         $logFooBar->insertEntry(52 + $toAddToId, 36 + $toAddToId);
     }
 
+    public function insertArchiveRows($idSite, $numVisits)
+    {
+        for ($day = 0; $day < $numVisits; $day++) {
+            $archiveDate = Date::factory($this->dateTime);
+            if ($day > 0) {
+                $archiveDate = $archiveDate->addDay($day * 3);
+            }
+            $doneRow = array(
+                'idarchive' => ($idSite * 100) + ($day * 10) + 1,
+                'idsite' => $idSite,
+                'name' => 'done',
+                'value' => ArchiveWriter::DONE_OK,
+                'date1' => $archiveDate->toString('Y-m-d'),
+                'date2' => $archiveDate->toString('Y-m-d'),
+                'period' => 1,
+                'ts_archived' => $archiveDate->getDatetime()
+            );
+            $this->insertArchiveRow($archiveDate, $doneRow);
+        }
+    }
+
+    private function insertArchiveRow($date, $row)
+    {
+        $table = ArchiveTableCreator::getNumericTable($date);
+        $sql = "INSERT INTO %s (idarchive, idsite, name, value, date1, date2, period, ts_archived) VALUES ('%s')";
+
+        $row['period'] = Day::PERIOD_ID;
+        Db::exec(sprintf($sql, $table, implode("','", $row)));
+
+        $row['period'] = Week::PERIOD_ID;
+        $row['idarchive']++;
+        Db::exec(sprintf($sql, $table, implode("','", $row)));
+
+        $row['period'] = Month::PERIOD_ID;
+        $row['idarchive']++;
+        Db::exec(sprintf($sql, $table, implode("','", $row)));
+
+        $row['period'] = Year::PERIOD_ID;
+        $row['idarchive']++;
+        Db::exec(sprintf($sql, $table, implode("','", $row)));
+    }
+
     public function trackVisits($idSite, $numIterations)
     {
         for ($day = 0; $day < $numIterations; $day++) {
diff --git a/plugins/PrivacyManager/tests/Integration/Model/DataSubjectsTest.php b/plugins/PrivacyManager/tests/Integration/Model/DataSubjectsTest.php
index aa6357b17e..9adc22c7c2 100644
--- a/plugins/PrivacyManager/tests/Integration/Model/DataSubjectsTest.php
+++ b/plugins/PrivacyManager/tests/Integration/Model/DataSubjectsTest.php
@@ -10,7 +10,10 @@ namespace Piwik\Plugins\PrivacyManager\tests\Integration\Model;
 
 use Piwik\Common;
 use Piwik\Container\StaticContainer;
+use Piwik\DataAccess\ArchiveTableCreator;
+use Piwik\Date;
 use Piwik\Db;
+use Piwik\Option;
 use Piwik\Plugins\PrivacyManager\Model\DataSubjects;
 use Piwik\Plugins\PrivacyManager\tests\Fixtures\MultipleSitesMultipleVisitsFixture;
 use Piwik\Plugins\PrivacyManager\tests\Fixtures\TestLogFoo;
@@ -36,6 +39,10 @@ class DataSubjectsTest extends IntegrationTestCase
      */
     private $theFixture;
 
+    private $originalTrackingTime;
+
+    private $originalTimezone;
+
     public function setUp()
     {
         parent::setUp();
@@ -46,12 +53,17 @@ class DataSubjectsTest extends IntegrationTestCase
 
         $logTablesProvider = StaticContainer::get('Piwik\Plugin\LogTablesProvider');
         $this->dataSubjects = new DataSubjects($logTablesProvider);
+        $this->originalTimezone = ini_get('date.timezone');
+        $this->originalTrackingTime = $this->theFixture->trackingTime;
     }
 
     public function tearDown()
     {
         $this->theFixture->uninstallLogTables();
         $this->theFixture->tearDownLocation();
+        $this->removeArchiveInvalidationOptions();
+        ini_set('date.timezone', $this->originalTimezone);
+        $this->theFixture->trackingTime = $this->originalTrackingTime;
     }
 
     public function test_deleteExport_deleteOneVisit()
@@ -285,6 +297,123 @@ class DataSubjectsTest extends IntegrationTestCase
         ], $this->getAllLogFooBarBaz());
     }
 
+    public function test_deleteOneVisit_doesInvalidateArchive()
+    {
+        $this->theFixture->setUpWebsites();
+        $this->theFixture->trackVisits($idSite = 1, 2);
+        $this->theFixture->insertArchiveRows($idSite, 2);
+        $this->removeArchiveInvalidationOptions();
+
+        $visitDate = Date::factory($this->theFixture->dateTime);
+        $secondVisitDate = $visitDate->addDay(3);
+
+        $this->assertArchivesHaveNotBeenInvalidated($visitDate, $idSite);
+
+        $this->dataSubjects->deleteDataSubjects(array(array('idsite' => '1', 'idvisit' => 1)));
+
+        $this->assertArchivesHaveBeenInvalidated($visitDate, $idSite);
+        $this->assertArchivesHaveNotBeenInvalidated($secondVisitDate, $idSite);
+    }
+
+    public function test_deleteTwoVisits_doesInvalidateArchive()
+    {
+        $this->theFixture->setUpWebsites();
+        $this->theFixture->trackVisits($idSite = 1, 2);
+        $this->theFixture->insertArchiveRows($idSite, 2);
+        $this->removeArchiveInvalidationOptions();
+
+        $this->dataSubjects->deleteDataSubjects(array(
+            array('idsite' => '1', 'idvisit' => 1),
+            array('idsite' => '1', 'idvisit' => 1 + $this->theFixture->numVisitsPerIteration),
+        ));
+
+        $visitDate = Date::factory($this->theFixture->dateTime);
+        $secondVisitDate = $visitDate->addDay(3);
+
+        $this->assertArchivesHaveBeenInvalidated($visitDate, $idSite);
+        $this->assertArchivesHaveBeenInvalidated($secondVisitDate, $idSite);
+    }
+
+    public function test_deleteVisitsForMultipleSites_doesInvalidateArchive()
+    {
+        $this->theFixture->setUpWebsites();
+        for ($idSite = 1; $idSite <= 2; $idSite++) {
+            $this->theFixture->trackingTime = $this->originalTrackingTime;
+            $this->theFixture->trackVisits($idSite, 2);
+            $this->theFixture->insertArchiveRows($idSite, 2);
+        }
+        $this->removeArchiveInvalidationOptions();
+
+        $this->dataSubjects->deleteDataSubjects(array(
+            array('idsite' => '1', 'idvisit' => 1 + $this->theFixture->numVisitsPerIteration),
+            array('idsite' => '2', 'idvisit' => 1 + (2 * $this->theFixture->numVisitsPerIteration)),
+        ));
+
+        $visitDate = Date::factory($this->theFixture->dateTime);
+        $secondVisitDate = $visitDate->addDay(3);
+
+        $this->assertArchivesHaveNotBeenInvalidated($visitDate, 1);
+        $this->assertArchivesHaveBeenInvalidated($visitDate, 2);
+        $this->assertArchivesHaveBeenInvalidated($secondVisitDate, 1);
+        $this->assertArchivesHaveNotBeenInvalidated($secondVisitDate, 2);
+    }
+
+    public function test_deleteOneVisit_alreadyMarkedForInvalidation()
+    {
+        $this->theFixture->setUpWebsites();
+        $this->theFixture->trackVisits($idSite = 1, 2);
+        $this->theFixture->insertArchiveRows($idSite, 2);
+        $this->removeArchiveInvalidationOptions();
+
+        $visitDate = Date::factory($this->theFixture->dateTime);
+        $key = 'report_to_invalidate_' . $idSite . '_' . $visitDate->toString('Y-m-d') . '_12345';
+        Option::set($key, '1');
+
+        $this->assertArchivesHaveBeenInvalidated($visitDate, $idSite);
+
+        $this->dataSubjects->deleteDataSubjects(array(array('idsite' => '1', 'idvisit' => 1)));
+
+        $this->assertArchivesHaveBeenInvalidated($visitDate, $idSite);
+    }
+
+    public function test_deleteOneVisit_siteInDifferentTimezone()
+    {
+        ini_set('date.timezone', 'UTC');
+        $websiteTimezone = 'UTC+5';
+
+        // It's 2 January in UTC but 3 January in UTC+5
+        $testTime = '2017-01-02 23:00:00';
+        $this->theFixture->trackingTime = Date::factory($testTime)->getDatetime();
+        $this->theFixture->setUpWebsites();
+        $this->setWebsiteTimezone($idSite = 1, $websiteTimezone);
+        $this->theFixture->trackVisits($idSite, 1);
+        $this->theFixture->insertArchiveRows($idSite, 1);
+        $this->removeArchiveInvalidationOptions();
+
+        $visitDate = Date::factory($testTime, $websiteTimezone);
+
+        $this->assertArchivesHaveNotBeenInvalidated($visitDate, $idSite);
+
+        $this->dataSubjects->deleteDataSubjects(array(array('idsite' => '1', 'idvisit' => 1)));
+
+        $this->assertArchivesHaveBeenInvalidated($visitDate, $idSite);
+    }
+
+    private function assertArchivesHaveNotBeenInvalidated(Date $visitDate, $idSite)
+    {
+        $key = 'report_to_invalidate_' . $idSite . '_' . $visitDate->toString('Y-m-d');
+        $value = Option::getLike($key . '%');
+        $this->assertEmpty($value);
+    }
+
+    private function assertArchivesHaveBeenInvalidated(Date $visitDate, $idSite)
+    {
+        $key = 'report_to_invalidate_' . $idSite . '_' . $visitDate->toString('Y-m-d');
+        $value = Option::getLike($key . '%');
+        $this->assertNotEmpty($value);
+        $this->assertEquals('1', array_values($value)[0]);
+    }
+
     private function getFromTable($table)
     {
         $rows = Db::fetchAll('SELECT * from ' . Common::prefixTable($table));
@@ -376,4 +505,16 @@ class DataSubjectsTest extends IntegrationTestCase
     {
         return Db::fetchOne("SELECT COUNT(*) FROM `" . Common::prefixTable($table) . "` WHERE idsite = ?", [$idSite]);
     }
+
+    private function removeArchiveInvalidationOptions()
+    {
+        Option::deleteLike('report_to_invalidate_%');
+    }
+
+    private function setWebsiteTimezone($idSite, $timezone)
+    {
+        $sql = 'UPDATE ' . Common::prefixTable('site') . ' SET timezone = ? WHERE idsite = ?';
+        $bind = array($timezone, $idSite);
+        Db::query($sql, $bind);
+    }
 }
diff --git a/plugins/PrivacyManager/tests/UI/expected-screenshots/PrivacyManager_gdpr_tools_delete_visit_cancelled.png b/plugins/PrivacyManager/tests/UI/expected-screenshots/PrivacyManager_gdpr_tools_delete_visit_cancelled.png
index 9e5a072759..bf07948ecc 100644
--- a/plugins/PrivacyManager/tests/UI/expected-screenshots/PrivacyManager_gdpr_tools_delete_visit_cancelled.png
+++ b/plugins/PrivacyManager/tests/UI/expected-screenshots/PrivacyManager_gdpr_tools_delete_visit_cancelled.png
@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:4c3199d54a6d07e7ed3c48dd5c10ceed2fffc3b0fcfde25e6aa678066ae69843
-size 609788
+oid sha256:4ea11b2bd1d1b3f1eca0dfd110639ed7a92130ed28d3aa0276a1af140cd25009
+size 583171
diff --git a/plugins/PrivacyManager/tests/UI/expected-screenshots/PrivacyManager_gdpr_tools_delete_visit_cancelled_verified_no_data_deleted.png b/plugins/PrivacyManager/tests/UI/expected-screenshots/PrivacyManager_gdpr_tools_delete_visit_cancelled_verified_no_data_deleted.png
index 6ab9a8a0b7..eb0a1a7784 100644
--- a/plugins/PrivacyManager/tests/UI/expected-screenshots/PrivacyManager_gdpr_tools_delete_visit_cancelled_verified_no_data_deleted.png
+++ b/plugins/PrivacyManager/tests/UI/expected-screenshots/PrivacyManager_gdpr_tools_delete_visit_cancelled_verified_no_data_deleted.png
@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:7d611d47840117c1e2a29ebfd0d8f651027a4842a857b40cef6f12a049d73529
-size 603444
+oid sha256:f3a9504aa54808809e34fa4b72e475253a97b6e38285c0413f03cc7ad8f30ed9
+size 576791
diff --git a/plugins/PrivacyManager/tests/UI/expected-screenshots/PrivacyManager_gdpr_tools_delete_visit_confirmed.png b/plugins/PrivacyManager/tests/UI/expected-screenshots/PrivacyManager_gdpr_tools_delete_visit_confirmed.png
index d967cb7c11..ac74dd79cb 100644
--- a/plugins/PrivacyManager/tests/UI/expected-screenshots/PrivacyManager_gdpr_tools_delete_visit_confirmed.png
+++ b/plugins/PrivacyManager/tests/UI/expected-screenshots/PrivacyManager_gdpr_tools_delete_visit_confirmed.png
@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:c0f62c9f4b4405d64c5460da42d722ffa19e5c9650c284024a090cd16f1511a3
-size 158652
+oid sha256:f9173c687489eb273552068e36260002aa9b864426736493840f313550fd8928
+size 132088
diff --git a/plugins/PrivacyManager/tests/UI/expected-screenshots/PrivacyManager_gdpr_tools_delete_visit_unconfirmed.png b/plugins/PrivacyManager/tests/UI/expected-screenshots/PrivacyManager_gdpr_tools_delete_visit_unconfirmed.png
index 98e8d4f566..3ffd940c7e 100644
--- a/plugins/PrivacyManager/tests/UI/expected-screenshots/PrivacyManager_gdpr_tools_delete_visit_unconfirmed.png
+++ b/plugins/PrivacyManager/tests/UI/expected-screenshots/PrivacyManager_gdpr_tools_delete_visit_unconfirmed.png
@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:0f7c838cf1f548ebd5efc72329281c0702c6ea8e858ceda5058ce42edd1911e9
-size 577366
+oid sha256:a73f94c9f69458fbb294ecf4dd79efc106fb0b18811fcca23c42c35793d70169
+size 552099
diff --git a/plugins/PrivacyManager/tests/UI/expected-screenshots/PrivacyManager_gdpr_tools_enrich_segment_by_ip.png b/plugins/PrivacyManager/tests/UI/expected-screenshots/PrivacyManager_gdpr_tools_enrich_segment_by_ip.png
index 4bd4a9d706..37c5f88d81 100644
--- a/plugins/PrivacyManager/tests/UI/expected-screenshots/PrivacyManager_gdpr_tools_enrich_segment_by_ip.png
+++ b/plugins/PrivacyManager/tests/UI/expected-screenshots/PrivacyManager_gdpr_tools_enrich_segment_by_ip.png
@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ffdd4dc602758bd24b8075764a05df7336af8be9a681259a0f32ae5255852970
-size 610145
+oid sha256:c0d9688cd9cd41655b51f6eba4fd8a1db3d04b1312db6f2f02c96014647fcb54
+size 583527
diff --git a/plugins/PrivacyManager/tests/UI/expected-screenshots/PrivacyManager_gdpr_tools_uncheck_one_visit.png b/plugins/PrivacyManager/tests/UI/expected-screenshots/PrivacyManager_gdpr_tools_uncheck_one_visit.png
index 3dd8afad62..30ea3497dc 100644
--- a/plugins/PrivacyManager/tests/UI/expected-screenshots/PrivacyManager_gdpr_tools_uncheck_one_visit.png
+++ b/plugins/PrivacyManager/tests/UI/expected-screenshots/PrivacyManager_gdpr_tools_uncheck_one_visit.png
@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:94a85ed72b2e8c3d082c229834102fdb1085e12c546142f0d6b76eecae211418
-size 609737
+oid sha256:53b8546474391ef2da5552731cad634aba5e6543dd7e73ac6bf10143c3e7caa8
+size 583122
diff --git a/plugins/PrivacyManager/tests/UI/expected-screenshots/PrivacyManager_gdpr_tools_visits_found.png b/plugins/PrivacyManager/tests/UI/expected-screenshots/PrivacyManager_gdpr_tools_visits_found.png
index 6955c2f856..02c6bf6672 100644
--- a/plugins/PrivacyManager/tests/UI/expected-screenshots/PrivacyManager_gdpr_tools_visits_found.png
+++ b/plugins/PrivacyManager/tests/UI/expected-screenshots/PrivacyManager_gdpr_tools_visits_found.png
@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:de208902296e5a5eb263269560cf86ab342cb3825ed4f6d81e09f0daeda2aa98
-size 604746
+oid sha256:97ecdf9bdb087863d3bf767d591c2d8f82fade12ca811b84aff6cc2d49d42ee0
+size 578233
diff --git a/plugins/PrivacyManager/tests/UI/expected-screenshots/PrivacyManager_gdpr_tools_visits_showprofile.png b/plugins/PrivacyManager/tests/UI/expected-screenshots/PrivacyManager_gdpr_tools_visits_showprofile.png
index c9e86bcdde..d8c2d71064 100644
--- a/plugins/PrivacyManager/tests/UI/expected-screenshots/PrivacyManager_gdpr_tools_visits_showprofile.png
+++ b/plugins/PrivacyManager/tests/UI/expected-screenshots/PrivacyManager_gdpr_tools_visits_showprofile.png
@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:4de0dc8be8d8b7e83f624fa827de3c5209c444dbd10d4c1e43e98e1f6751bb01
-size 621894
+oid sha256:407af20487bbb811d1e895d8bb4ccadf0c8aec1698be3298a7f2a3270dcceb2b
+size 604924