Task to purge archives for deleted websites and segments (#14317)

* Purge archives for deleted sites and segments

* Purge archives for deleted sites and segments

* Add new purgeOrphanedArchives task to expected list

* Fix build

* PR improvements

* Fix consistency of method names

* Fix typo

* Unit tests for getSegmentHashesByIdSite

* PR changes

* add note on how to test the command

* minor tweak to make sure no injections are possible

2 files changed, 149 insertions, 5 deletions

diff --git a/tests/PHPUnit/Fixtures/RawArchiveDataWithTempAndInvalidated.php b/tests/PHPUnit/Fixtures/RawArchiveDataWithTempAndInvalidated.php
index b93e6d42e7..6d094132b5 100644
--- a/tests/PHPUnit/Fixtures/RawArchiveDataWithTempAndInvalidated.php
+++ b/tests/PHPUnit/Fixtures/RawArchiveDataWithTempAndInvalidated.php
@@ -241,6 +241,110 @@ class RawArchiveDataWithTempAndInvalidated extends Fixture
         ),
     );
 
+
+    private static $segmentArchiveData = array(
+        array(
+            'idarchive' => 20,
+            'idsite' => 1,
+            'name' => 'doneabcd1234abcd5678',
+            'value' => ArchiveWriter::DONE_OK,
+            'date1' => '2015-02-03',
+            'date2' => '2015-02-03',
+            'period' => 1,
+            'ts_archived' => '2015-02-03 12:12:12'
+        ),
+        array(
+            'idarchive' => 21,
+            'idsite' => 1,
+            'name' => 'doneabcd1234abcd5678.MyPlugin',
+            'value' => ArchiveWriter::DONE_OK,
+            'date1' => '2015-02-03',
+            'date2' => '2015-02-03',
+            'period' => 1,
+            'ts_archived' => '2015-02-03 12:12:12'
+        ),
+        array(
+            'idarchive' => 22,
+            'idsite' => 2,
+            'name' => 'doneabcd1234abcd5678',
+            'value' => ArchiveWriter::DONE_OK,
+            'date1' => '2015-02-03',
+            'date2' => '2015-02-03',
+            'period' => 1,
+            'ts_archived' => '2015-02-03 12:12:12'
+        ),
+        array(
+            'idarchive' => 23,
+            'idsite' => 2,
+            'name' => 'doneabcd1234abcd5678.MyPlugin',
+            'value' => ArchiveWriter::DONE_OK,
+            'date1' => '2015-02-03',
+            'date2' => '2015-02-03',
+            'period' => 1,
+            'ts_archived' => '2015-02-03 12:12:12'
+        ),
+        array(
+            'idarchive' => 24,
+            'idsite' => 1,
+            'name' => 'done9876fedc5432abcd',
+            'value' => ArchiveWriter::DONE_OK,
+            'date1' => '2015-02-03',
+            'date2' => '2015-02-03',
+            'period' => 1,
+            'ts_archived' => '2015-02-03 12:12:12'
+        ),
+        array(
+            'idarchive' => 25,
+            'idsite' => 2,
+            'name' => 'donehash1',
+            'value' => ArchiveWriter::DONE_OK,
+            'date1' => '2015-02-03',
+            'date2' => '2015-02-03',
+            'period' => 1,
+            'ts_archived' => '2015-02-03 12:12:12'
+        ),
+        array(
+            'idarchive' => 26,
+            'idsite' => 2,
+            'name' => 'donehash2',
+            'value' => ArchiveWriter::DONE_OK,
+            'date1' => '2015-02-03',
+            'date2' => '2015-02-03',
+            'period' => 1,
+            'ts_archived' => '2015-02-03 12:12:12'
+        ),
+        array(
+            'idarchive' => 27,
+            'idsite' => 2,
+            'name' => 'donehash2.MyPlugin',
+            'value' => ArchiveWriter::DONE_OK,
+            'date1' => '2015-02-03',
+            'date2' => '2015-02-03',
+            'period' => 1,
+            'ts_archived' => '2015-02-03 12:12:12'
+        ),
+        array(
+            'idarchive' => 28,
+            'idsite' => 2,
+            'name' => 'donehash3',
+            'value' => ArchiveWriter::DONE_OK,
+            'date1' => '2015-02-03',
+            'date2' => '2015-02-03',
+            'period' => 1,
+            'ts_archived' => '2015-02-03 12:12:12'
+        ),
+        array(
+            'idarchive' => 29   ,
+            'idsite' => 2,
+            'name' => 'donehash3',
+            'value' => ArchiveWriter::DONE_OK,
+            'date1' => '2015-02-03',
+            'date2' => '2015-02-03',
+            'period' => 1,
+            'ts_archived' => '2015-02-27 12:12:12'
+        ),
+    );
+
     /**
      * @var Date
      */
@@ -262,10 +366,21 @@ class RawArchiveDataWithTempAndInvalidated extends Fixture
         $this->insertOutdatedArchives($this->february);
     }
 
+    public function insertSegmentArchives(Date $archiveDate)
+    {
+        $dummyArchiveData = $this->setDatesOnArchiveData($archiveDate, self::$segmentArchiveData);
+        $this->insertArchiveRows($archiveDate, $dummyArchiveData);
+    }
+
     private function insertOutdatedArchives(Date $archiveDate)
     {
-        $dummyArchiveData = $this->getDummyArchiveDataForDate($archiveDate);
+        $dummyArchiveData = $this->setDatesOnArchiveData($archiveDate, self::$dummyArchiveData);
+        $this->insertArchiveRows($archiveDate, $dummyArchiveData);
 
+    }
+
+    private function insertArchiveRows(Date $archiveDate, array $dummyArchiveData)
+    {
         $numericTable = ArchiveTableCreator::getNumericTable($archiveDate);
         foreach ($dummyArchiveData as $row) {
             // done row
@@ -301,14 +416,13 @@ class RawArchiveDataWithTempAndInvalidated extends Fixture
         Db::exec(sprintf($insertSqlTemplate, $table, implode("','", $row)));
     }
 
-    private function getDummyArchiveDataForDate($archiveDate)
+    private function setDatesOnArchiveData($archiveDate, $data)
     {
-        $rows = self::$dummyArchiveData;
-        foreach ($rows as &$row) {
+        foreach ($data as &$row) {
             $row['date1'] = $this->setDateMonthAndYear($row['date1'], $archiveDate);
             $row['date2'] = $this->setDateMonthAndYear($row['date1'], $archiveDate);
         }
-        return$rows;
+        return $data;
     }
 
     private function setDateMonthAndYear($dateString, Date $archiveDate)
diff --git a/tests/PHPUnit/Integration/Archive/ArchivePurgerTest.php b/tests/PHPUnit/Integration/Archive/ArchivePurgerTest.php
index d520e72fb9..f7051937e9 100644
--- a/tests/PHPUnit/Integration/Archive/ArchivePurgerTest.php
+++ b/tests/PHPUnit/Integration/Archive/ArchivePurgerTest.php
@@ -12,6 +12,7 @@ use Piwik\Config;
 use Piwik\Date;
 use Piwik\Db;
 use Piwik\Tests\Fixtures\RawArchiveDataWithTempAndInvalidated;
+use Piwik\Tests\Framework\Fixture;
 use Piwik\Tests\Framework\TestCase\IntegrationTestCase;
 
 /**
@@ -107,6 +108,35 @@ class ArchivePurgerTest extends IntegrationTestCase
         $this->assertEquals(3 * RawArchiveDataWithTempAndInvalidated::ROWS_PER_ARCHIVE, $deletedRowCount);
     }
 
+    public function test_purgeNoSiteArchives_PurgesAllNoSiteArchives()
+    {
+        //Create two websites (IDs #1 and #2). Existing rows for website #3 will be invalid.
+        Fixture::createWebsite($this->january);
+        Fixture::createWebsite($this->january);
+
+        //There are 5 rows for website #3. We leave the other two because they're before our purge threshold.
+        $deletedRowCount = $this->archivePurger->purgeDeletedSiteArchives($this->january);
+        $this->assertEquals(3 * RawArchiveDataWithTempAndInvalidated::ROWS_PER_ARCHIVE, $deletedRowCount);
+        self::$fixture->assertArchivesDoNotExist(array(3, 7, 10), $this->january);
+    }
+
+    public function test_purgeNoSegmentArchives_PurgesSegmentForAppropriateSitesOnly()
+    {
+        //Extra data set with segment and plugin archives
+        self::$fixture->insertSegmentArchives($this->january);
+
+        $validSegmentIds = array(
+            0 => ['DUMMYHASHSTR'],          //valid for all sites
+            1 => ['abcd1234abcd5678'],      //valid for site 1. should be ignored for site 2
+            2 => ['hashthatdontexist', 'hash1', 'hash2']
+        );
+
+        //Archive #29 also has a deleted segment but it's before the purge threshold so it stays for now.
+        $deletedRowCount = $this->archivePurger->purgeDeletedSegmentArchives($this->january, $validSegmentIds);
+        $this->assertEquals(4 * RawArchiveDataWithTempAndInvalidated::ROWS_PER_ARCHIVE, $deletedRowCount);
+        self::$fixture->assertArchivesDoNotExist(array(22, 23, 24, 28), $this->january);
+    }
+
     private function configureCustomRangePurging()
     {
         Config::getInstance()->General['purge_date_range_archives_after_X_days'] = 3;