diff options
| -rw-r--r-- | core/Archive/ArchivePurger.php | 17 | ||||
| -rw-r--r-- | core/DataAccess/Model.php | 66 | ||||
| -rw-r--r-- | plugins/CoreAdminHome/Tasks.php | 33 | ||||
| -rw-r--r-- | plugins/CoreAdminHome/tests/Integration/TasksTest.php | 127 | ||||
| -rw-r--r-- | plugins/SegmentEditor/Model.php | 81 | ||||
| -rw-r--r-- | plugins/SegmentEditor/tests/Integration/ModelTest.php | 186 | ||||
| -rw-r--r-- | tests/PHPUnit/Fixtures/RawArchiveDataWithTempAndInvalidated.php | 22 | ||||
| -rw-r--r-- | tests/PHPUnit/Integration/Archive/ArchivePurgerTest.php | 38 |
8 files changed, 362 insertions, 208 deletions
diff --git a/core/Archive/ArchivePurger.php b/core/Archive/ArchivePurger.php index c69bf3ecb9..cd330ea37c 100644 --- a/core/Archive/ArchivePurger.php +++ b/core/Archive/ArchivePurger.php @@ -166,14 +166,15 @@ class ArchivePurger /** * @param Date $dateStart - * @param array $segmentHashesByIdSite List of valid segment hashes, indexed by site ID + * @param array $deletedSegments List of segments whose archives should be purged * @return int */ - public function purgeDeletedSegmentArchives(Date $dateStart, array $segmentHashesByIdSite) + public function purgeDeletedSegmentArchives(Date $dateStart, array $deletedSegments) { - $idArchivesToDelete = $this->getDeletedSegmentArchiveIds($dateStart, $segmentHashesByIdSite); - - return $this->purge($idArchivesToDelete, $dateStart, 'deleted segments'); + if (count($deletedSegments)) { + $idArchivesToDelete = $this->getDeletedSegmentArchiveIds($dateStart, $deletedSegments); + return $this->purge($idArchivesToDelete, $dateStart, 'deleted segments'); + } } /** @@ -211,11 +212,11 @@ class ArchivePurger return $deletedRowCount; } - protected function getDeletedSegmentArchiveIds(Date $date, array $segmentHashesByIdSite) + protected function getDeletedSegmentArchiveIds(Date $date, array $deletedSegments) { $archiveTable = ArchiveTableCreator::getNumericTable($date); - return $this->model->getArchiveIdsForDeletedSegments( - $archiveTable, $segmentHashesByIdSite, $this->getOldestTemporaryArchiveToKeepThreshold() + return $this->model->getArchiveIdsForSegments( + $archiveTable, $deletedSegments, $this->getOldestTemporaryArchiveToKeepThreshold() ); } diff --git a/core/DataAccess/Model.php b/core/DataAccess/Model.php index 85294b8bcc..3f80753835 100644 --- a/core/DataAccess/Model.php +++ b/core/DataAccess/Model.php @@ -354,50 +354,54 @@ class Model * Get a list of IDs of archives with segments that no longer exist in the DB. Excludes temporary archives that * may still be in use, as specified by the $oldestToKeep passed in. * @param string $archiveTableName - * @param array $segmentHashesById Whitelist of existing segments, indexed by site ID + * @param array $segments List of segments to match against * @param string $oldestToKeep Datetime string * @return array With keys idarchive, name, idsite */ - public function getArchiveIdsForDeletedSegments($archiveTableName, array $segmentHashesById, $oldestToKeep) + public function getArchiveIdsForSegments($archiveTableName, array $segments, $oldestToKeep) { - $validSegmentClauses = []; - - foreach ($segmentHashesById as $idSite => $segments) { - // segments are md5 hashes and such not a problem re sql injection. for performance etc we don't want to use - // bound parameters for the query - foreach ($segments as $segment) { - if (!preg_match('/^[a-z0-9A-Z]+$/', $segment)) { - throw new Exception($segment . ' expected to be an md5 hash'); - } - } - - // Special case as idsite=0 means the segments are not site-specific - if ($idSite === 0) { - foreach ($segments as $segmentHash) { - $validSegmentClauses[] = '(name LIKE "done' . $segmentHash . '%")'; - } - continue; + $segmentClauses = []; + foreach ($segments as $segment) { + if (!empty($segment['definition'])) { + $segmentClauses[] = $this->getDeletedSegmentWhereClause($segment); } + } - $idSite = (int)$idSite; - - // Vanilla case - segments that are valid for a single site only - $sql = '(idsite = ' . $idSite . ' AND ('; - $sql .= 'name LIKE "done' . implode('%" OR name LIKE "done', $segments) . '%"'; - $sql .= '))'; - $validSegmentClauses[] = $sql; + if (empty($segmentClauses)) { + return array(); } - $isValidSegmentSql = implode(' OR ', $validSegmentClauses); + $segmentClauses = implode(' OR ', $segmentClauses); $sql = 'SELECT idarchive FROM ' . $archiveTableName - . ' WHERE name LIKE "done%" AND name != "done"' - . ' AND ts_archived < ?' - . ' AND NOT (' . $isValidSegmentSql . ')'; + . ' WHERE ts_archived < ?' + . ' AND (' . $segmentClauses . ')'; $rows = Db::fetchAll($sql, array($oldestToKeep)); - return array_map(function($row) { return $row['idarchive']; }, $rows); + return array_column($rows, 'idarchive'); + } + + private function getDeletedSegmentWhereClause(array $segment) + { + $idSite = (int)$segment['enable_only_idsite']; + $segmentHash = Segment::getSegmentHash($segment['definition']); + // Valid segment hashes are md5 strings - just confirm that it is so it's safe for SQL injection + if (!ctype_xdigit($segmentHash)) { + throw new Exception($segment . ' expected to be an md5 hash'); + } + + $nameClause = 'name LIKE "done' . $segmentHash . '%"'; + $idSiteClause = ''; + if ($idSite > 0) { + $idSiteClause = ' AND idsite = ' . $idSite; + } elseif (! empty($segment['idsites_to_preserve'])) { + // A segment for all sites was deleted, but there are segments for a single site with the same definition + $idSitesToPreserve = array_map('intval', $segment['idsites_to_preserve']); + $idSiteClause = ' AND idsite NOT IN (' . implode(',', $idSitesToPreserve) . ')'; + } + + return "($nameClause $idSiteClause)"; } /** diff --git a/plugins/CoreAdminHome/Tasks.php b/plugins/CoreAdminHome/Tasks.php index ef9b0bd215..d94979c9c3 100644 --- a/plugins/CoreAdminHome/Tasks.php +++ b/plugins/CoreAdminHome/Tasks.php @@ -23,9 +23,9 @@ use Piwik\Piwik; use Piwik\Plugins\CoreAdminHome\Emails\JsTrackingCodeMissingEmail; use Piwik\Plugins\CoreAdminHome\Emails\TrackingFailuresEmail; use Piwik\Plugins\CoreAdminHome\Tasks\ArchivesToPurgeDistributedList; +use Piwik\Plugins\SegmentEditor\Model; use Piwik\Plugins\SitesManager\SitesManager; use Piwik\Scheduler\Schedule\SpecificTime; -use Piwik\Segment; use Piwik\Settings\Storage\Backend\MeasurableSettingsTable; use Piwik\Tracker\Failures; use Piwik\Site; @@ -275,7 +275,10 @@ class Tasks extends \Piwik\Plugin\Tasks */ public function purgeOrphanedArchives() { - $segmentHashesByIdSite = $this->getSegmentHashesByIdSite(); + $eightDaysAgo = Date::factory('now')->subDay(8); + $model = new Model(); + $deletedSegments = $model->getSegmentsDeletedSince($eightDaysAgo); + $archiveTables = ArchiveTableCreator::getTablesArchivesInstalled('numeric'); $datesPurged = array(); @@ -286,35 +289,15 @@ class Tasks extends \Piwik\Plugin\Tasks $dateObj = Date::factory("$year-$month-15"); $this->archivePurger->purgeDeletedSiteArchives($dateObj); - $this->archivePurger->purgeDeletedSegmentArchives($dateObj, $segmentHashesByIdSite); + if (count($deletedSegments)) { + $this->archivePurger->purgeDeletedSegmentArchives($dateObj, $deletedSegments); + } $datesPurged[$date] = true; } } /** - * Get a list of all segment hashes that currently exist, indexed by idSite. - * @return array - */ - public function getSegmentHashesByIdSite() - { - //Get a list of hashes of all segments that exist now - $sql = "SELECT DISTINCT definition, enable_only_idsite FROM " . Common::prefixTable('segment') - . " WHERE deleted = 0"; - $rows = Db::fetchAll($sql); - $segmentHashes = array(); - foreach ($rows as $row) { - $idSite = (int)$row['enable_only_idsite']; - if (! isset($segmentHashes[$idSite])) { - $segmentHashes[$idSite] = array(); - } - $segmentHashes[$idSite][] = Segment::getSegmentHash($row['definition']); - } - - return $segmentHashes; - } - - /** * we should only purge outdated & custom range archives if we know cron archiving has just run, * or if browser triggered archiving is enabled. if cron archiving has run, then we know the latest * archives are in the database, and we can remove temporary ones. if browser triggered archiving is diff --git a/plugins/CoreAdminHome/tests/Integration/TasksTest.php b/plugins/CoreAdminHome/tests/Integration/TasksTest.php index 830da7ef97..a23d368de4 100644 --- a/plugins/CoreAdminHome/tests/Integration/TasksTest.php +++ b/plugins/CoreAdminHome/tests/Integration/TasksTest.php @@ -219,133 +219,6 @@ class TasksTest extends IntegrationTestCase $this->assertEquals(2, $mail->getNumFailures()); } - public function test_getSegmentHashesByIdSite_emptyWhenNoSegments() - { - $segmentsByIdSite = $this->tasks->getSegmentHashesByIdSite(); - $this->assertEquals(array(), $segmentsByIdSite); - } - - public function test_getSegmentHashesByIdSite_allWebsiteAndSiteSpecificSegments() - { - $model = new Model(); - $model->createSegment(array( - 'name' => 'Test Segment 1', - 'definition' => 'continentCode==eur', - 'enable_only_idsite' => 0, - 'deleted' => 0 - )); - $model->createSegment(array( - 'name' => 'Test Segment 2', - 'definition' => 'countryCode==nz', - 'enable_only_idsite' => 0, - 'deleted' => 0 - )); - $model->createSegment(array( - 'name' => 'Test Segment 3', - 'definition' => 'countryCode==au', - 'enable_only_idsite' => 2, - 'deleted' => 0 - )); - - $segmentsByIdSite = $this->tasks->getSegmentHashesByIdSite(); - $expected = array( - 0 => array('be90051048558489e1d62f4245a6dc65', 'b92fbb3009b32cf632965802de2fb760'), - 2 => array('cffd4336c22c6782211f853495076b1a') - ); - $this->assertEquals($expected, $segmentsByIdSite); - } - - public function test_getSegmentHashesByIdSite_deletedSegment() - { - $model = new Model(); - $model->createSegment(array( - 'name' => 'Test Segment 1', - 'definition' => 'continentCode==eur', - 'enable_only_idsite' => 0, - 'deleted' => 0 - )); - $model->createSegment(array( - 'name' => 'Test Segment 2', - 'definition' => 'countryCode==nz', - 'enable_only_idsite' => 0, - 'deleted' => 1 - )); - $model->createSegment(array( - 'name' => 'Test Segment 3', - 'definition' => 'countryCode==au', - 'enable_only_idsite' => 2, - 'deleted' => 0 - )); - - $segmentsByIdSite = $this->tasks->getSegmentHashesByIdSite(); - $expected = array( - 0 => array('be90051048558489e1d62f4245a6dc65'), - 2 => array('cffd4336c22c6782211f853495076b1a') - ); - $this->assertEquals($expected, $segmentsByIdSite); - } - - public function test_getSegmentHashesByIdSite_invalidSegment() - { - $model = new Model(); - $model->createSegment(array( - 'name' => 'Test Segment 4', - 'definition' => 'countryCode=nz', //The single "=" is invalid - we should generate a hash anyway - 'enable_only_idsite' => 0, - 'deleted' => 0 - )); - $model->createSegment(array( - 'name' => 'Test Segment 5', - 'definition' => 'countryCode==au', - 'enable_only_idsite' => 0, - 'deleted' => 0 - )); - - $expected = array( - 0 => array('5ffe7e116fae7576c047b1fb811584a5', 'cffd4336c22c6782211f853495076b1a'), - ); - - $segmentsByIdSite = $this->tasks->getSegmentHashesByIdSite(); - $this->assertEquals($expected, $segmentsByIdSite); - } - - public function test_getSegmentHashesByIdSite_siteSpecificCustomDimension() - { - // Insert a custom dimension for idsite = 1 - $configuration = new Configuration(); - $configuration->configureNewDimension( - 1, - 'mydimension', - CustomDimensions::SCOPE_VISIT, - 1, - 1, - array(), - true - ); - - $model = new Model(); - $model->createSegment(array( - 'name' => 'Test Segment 6', - 'definition' => 'mydimension==red', - 'enable_only_idsite' => 1, - 'deleted' => 0 - )); - $model->createSegment(array( - 'name' => 'Test Segment 7', - 'definition' => 'countryCode==au', - 'enable_only_idsite' => 2, - 'deleted' => 0 - )); - - $expected = array( - 1 => array('240d2a84a309debd26bdbaa8eb3d363c'), - 2 => array('cffd4336c22c6782211f853495076b1a') - ); - - $segmentsByIdSite = $this->tasks->getSegmentHashesByIdSite(); - $this->assertEquals($expected, $segmentsByIdSite); - } - /** * @param Date[] $dates */ diff --git a/plugins/SegmentEditor/Model.php b/plugins/SegmentEditor/Model.php index 0025c7c15c..c505f7a4f2 100644 --- a/plugins/SegmentEditor/Model.php +++ b/plugins/SegmentEditor/Model.php @@ -129,6 +129,87 @@ class Model return $segment; } + /** + * Gets a list of segments that have been deleted in the last week and therefore may have orphaned archives. + * @param Date $date Segments deleted on or after this date will be returned. + * @return array of segments. The segments are only populated with the fields needed for archive invalidation + * (e.g. definition, enable_only_idsite). + * @throws \Exception + */ + public function getSegmentsDeletedSince(Date $date) + { + $dateStr = $date->getDatetime(); + $sql = "SELECT DISTINCT definition, enable_only_idsite FROM " . Common::prefixTable('segment') + . " WHERE deleted = 1 AND ts_last_edit >= ?"; + $deletedSegments = Db::fetchAll($sql, array($dateStr)); + + if (empty($deletedSegments)) { + return array(); + } + + $existingSegments = $this->getExistingSegmentsLike($deletedSegments); + + foreach ($deletedSegments as $i => $deleted) { + $deletedSegments[$i]['idsites_to_preserve'] = array(); + foreach ($existingSegments as $existing) { + if ($existing['definition'] != $deleted['definition'] && + $existing['definition'] != urlencode($deleted['definition']) && + $existing['definition'] != urldecode($deleted['definition']) + ) { + continue; + } + + if ( + $existing['enable_only_idsite'] == $deleted['enable_only_idsite'] + || $existing['enable_only_idsite'] == 0 + ) { + // There is an identical segment (for either the specific site or for all sites) that is active + // The archives for this segment will therefore still be needed + unset($deletedSegments[$i]); + break; + } elseif ($deleted['enable_only_idsite'] == 0) { + // It is an all-sites segment that got deleted, but there is a single-site segment that is active + // Need to make sure we don't erase the segment's archives for that particular site + $deletedSegments[$i]['idsites_to_preserve'][] = $existing['enable_only_idsite']; + } + } + } + + return $deletedSegments; + } + + private function getExistingSegmentsLike(array $segments) + { + if (empty($segments)) { + return array(); + } + + $whereClauses = array(); + $bind = array(); + $definitionWhereClauseTemplate = '(definition = ? OR definition = ? OR definition = ?)'; + foreach ($segments as $segment) { + // Sometimes they are stored encoded and sometimes they aren't + $bind[] = $segment['definition']; + $bind[] = urlencode($segment['definition']); + $bind[] = urldecode($segment['definition']); + + if ($segment['enable_only_idsite'] == 0) { + // They deleted an all-sites segment, but there is a single-site segment with same definition? + // Need to handle this carefully so that the archives for the single-site segment are preserved + $whereClauses[] = "$definitionWhereClauseTemplate"; + } else { + $whereClauses[] = "($definitionWhereClauseTemplate AND (enable_only_idsite = ? OR enable_only_idsite = 0))"; + $bind[] = $segment['enable_only_idsite']; + } + } + $whereClauses = implode(' OR ', $whereClauses); + + // Check for any non-deleted segments with the same definition + $sql = "SELECT DISTINCT definition, enable_only_idsite FROM " . Common::prefixTable('segment') + . " WHERE deleted = 0 AND (" . $whereClauses . ")"; + return Db::fetchAll($sql, $bind); + } + public function deleteSegment($idSegment) { $fieldsToSet = array( diff --git a/plugins/SegmentEditor/tests/Integration/ModelTest.php b/plugins/SegmentEditor/tests/Integration/ModelTest.php index 92034f3efc..f309221c8f 100644 --- a/plugins/SegmentEditor/tests/Integration/ModelTest.php +++ b/plugins/SegmentEditor/tests/Integration/ModelTest.php @@ -25,6 +25,8 @@ class ModelTest extends IntegrationTestCase private $idSegment3; + private $idSegment4; + public function setUp() { parent::setUp(); @@ -55,7 +57,11 @@ class ModelTest extends IntegrationTestCase { parent::tearDown(); // Force a hard delete of segment - $idsToDelete = $this->idSegment1 . ', ' . $this->idSegment2 . ', ' . $this->idSegment3; + $idsToDelete = array($this->idSegment1, $this->idSegment2, $this->idSegment3); + if ($this->idSegment4) { + $idsToDelete[] = $this->idSegment4; + } + $idsToDelete = implode(',', $idsToDelete); Db::query( "DELETE FROM " . Common::prefixTable('segment') . " WHERE idsegment IN ($idsToDelete)" ); @@ -146,6 +152,184 @@ class ModelTest extends IntegrationTestCase $this->assertEmpty($segment); } + public function test_getSegmentsDeletedSince_noDeletedSegments() + { + $date = Date::factory('now'); + $segments = $this->model->getSegmentsDeletedSince($date); + $this->assertEmpty($segments); + } + + public function test_getSegmentsDeletedSince_oneDeletedSegment() + { + $this->model->deleteSegment($this->idSegment3); + + $date = Date::factory('now')->subDay(1); + $segments = $this->model->getSegmentsDeletedSince($date); + + $this->assertCount(1, $segments); + $this->assertEquals('country==Hobbiton', $segments[0]['definition']); + } + + public function test_getSegmentsDeletedSince_segmentDeletedTooLongAgo() + { + // Manually delete it to set timestamp 9 days in past + $deletedAt = Date::factory('now')->subDay(9)->toString('Y-m-d H:i:s'); + $this->model->updateSegment($this->idSegment1, array( + 'deleted' => 1, + 'ts_last_edit' => $deletedAt + )); + + // The segment deleted above should not be included as it was more than 8 days ago + $date = Date::factory('now')->subDay(8); + $segments = $this->model->getSegmentsDeletedSince($date); + + $this->assertEmpty($segments); + } + + public function test_getSegmentsDeletedSince_duplicateSegment() + { + // Turn segment1 into a duplicate of segment2, except it's also deleted + $this->model->updateSegment($this->idSegment1, array( + 'definition' => 'country==Genovia', + 'deleted' => 1, + 'ts_last_edit' => Date::factory('now')->toString('Y-m-d H:i:s') + )); + + $date = Date::factory('now')->subDay(8); + $segments = $this->model->getSegmentsDeletedSince($date); + + $this->assertEmpty($segments); + } + + public function test_getSegmentsDeletedSince_duplicateSegmentDifferentIdSite() + { + // Turn segment2 into a duplicate of segment3, except for a different idsite and also deleted + $this->model->updateSegment($this->idSegment2, array( + 'definition' => 'country==Hobbiton', + 'enable_only_idsite' => 2, + 'deleted' => 1, + 'ts_last_edit' => Date::factory('now')->toString('Y-m-d H:i:s') + )); + + $date = Date::factory('now')->subDay(8); + $segments = $this->model->getSegmentsDeletedSince($date); + + $this->assertCount(1, $segments); + $this->assertEquals('country==Hobbiton', $segments[0]['definition']); + $this->assertEquals(2, $segments[0]['enable_only_idsite']); + } + + public function test_getSegmentsDeletedSince_duplicateSegmentAllSitesAndSingleSite() + { + // Turn segment2 into a duplicate of segment3, except for all sites and also deleted + $this->model->updateSegment($this->idSegment2, array( + 'definition' => 'country==Hobbiton', + 'enable_only_idsite' => 0, + 'deleted' => 1, + 'ts_last_edit' => Date::factory('now')->toString('Y-m-d H:i:s') + )); + + $date = Date::factory('now')->subDay(8); + $segments = $this->model->getSegmentsDeletedSince($date); + + $this->assertCount(1, $segments); + $this->assertEquals('country==Hobbiton', $segments[0]['definition']); + $this->assertEquals(0, $segments[0]['enable_only_idsite']); + $this->assertEquals(array(1), $segments[0]['idsites_to_preserve']); + } + + public function test_getSegmentsDeletedSince_duplicateSegmentSingleSiteAndAllSites() + { + // Turn segment3 into a duplicate of segment1, except for a single site and deleted + $this->model->updateSegment($this->idSegment3, array( + 'definition' => 'country==Narnia', + 'enable_only_idsite' => 1, + 'deleted' => 1, + 'ts_last_edit' => Date::factory('now')->toString('Y-m-d H:i:s') + )); + + $date = Date::factory('now')->subDay(8); + $segments = $this->model->getSegmentsDeletedSince($date); + + // There is still a live segment for all sites, so the deleted site-specific one is ignored + $this->assertEmpty($segments); + } + + public function test_getSegmentsDeletedSince_ExistingSiteSpecificAndAllSitesMatch() + { + // A deleted all-sites segment, with both an all-sites and a site-specific segment still present + $this->model->updateSegment($this->idSegment1, array( + 'definition' => 'actions >= 1', + 'enable_only_idsite' => 0, + 'deleted' => 0, + 'ts_last_edit' => Date::factory('now')->toString('Y-m-d H:i:s') + )); + $this->model->updateSegment($this->idSegment2, array( + 'definition' => 'actions >= 1', + 'enable_only_idsite' => 0, + 'deleted' => 1, + 'ts_last_edit' => Date::factory('now')->toString('Y-m-d H:i:s') + )); + $this->model->updateSegment($this->idSegment3, array( + 'definition' => 'actions >= 1', + 'enable_only_idsite' => 3, + 'deleted' => 0, + 'ts_last_edit' => Date::factory('now')->toString('Y-m-d H:i:s') + )); + $this->idSegment4 = $this->model->createSegment(array( + 'definition' => 'actions >= 1', + 'enable_only_idsite' => 1, + 'deleted' => 1, + 'ts_last_edit' => Date::factory('now')->toString('Y-m-d H:i:s') + )); + + $date = Date::factory('now')->subDay(8); + $segments = $this->model->getSegmentsDeletedSince($date); + + $this->assertEmpty($segments); + } + + public function test_getSegmentsDeletedSince_urlDecodedVersionOfSegment() + { + // Turn segment2 into a duplicate of segment3, except a urlencoded version + $this->model->updateSegment($this->idSegment2, array( + 'definition' => 'country%3D%3DHobbiton', + 'enable_only_idsite' => 1, + 'deleted' => 1, + 'ts_last_edit' => Date::factory('now')->toString('Y-m-d H:i:s') + )); + + $date = Date::factory('now')->subDay(8); + $segments = $this->model->getSegmentsDeletedSince($date); + + // The two encoded and decoded version of the segments should be treated as duplicates + // This means there segment has a non-deleted version so it's not returned + $this->assertEmpty($segments); + } + + public function test_getSegmentsDeletedSince_urlEncodedVersionOfSegment() + { + // segment1 => url decoded version, deleted + $this->model->updateSegment($this->idSegment1, array( + 'definition' => 'country==Narnia', + 'deleted' => 1, + 'ts_last_edit' => Date::factory('now')->toString('Y-m-d H:i:s') + )); + // segment2 => url encoded version, not deleted + $this->model->updateSegment($this->idSegment2, array( + 'definition' => 'country%3D%3DNarnia', + 'deleted' => 0, + 'ts_last_edit' => Date::factory('now')->toString('Y-m-d H:i:s') + )); + + $date = Date::factory('now')->subDay(8); + $segments = $this->model->getSegmentsDeletedSince($date); + + // The two encoded and decoded version of the segments should be treated as duplicates + // This means there segment has a non-deleted version so it's not returned + $this->assertEmpty($segments); + } + private function assertReturnedIdsMatch(array $expectedIds, array $resultSet) { $this->assertEquals(count($expectedIds), count($resultSet)); diff --git a/tests/PHPUnit/Fixtures/RawArchiveDataWithTempAndInvalidated.php b/tests/PHPUnit/Fixtures/RawArchiveDataWithTempAndInvalidated.php index cc7da149c9..b4f597b97a 100644 --- a/tests/PHPUnit/Fixtures/RawArchiveDataWithTempAndInvalidated.php +++ b/tests/PHPUnit/Fixtures/RawArchiveDataWithTempAndInvalidated.php @@ -246,7 +246,7 @@ class RawArchiveDataWithTempAndInvalidated extends Fixture array( 'idarchive' => 20, 'idsite' => 1, - 'name' => 'doneabcd1234abcd5678', + 'name' => 'doneeb5d2797aedd15d819b1a20425982850', // Raw segment = abcd1234abcd5678 'value' => ArchiveWriter::DONE_OK, 'date1' => '2015-02-03', 'date2' => '2015-02-03', @@ -256,7 +256,7 @@ class RawArchiveDataWithTempAndInvalidated extends Fixture array( 'idarchive' => 21, 'idsite' => 1, - 'name' => 'doneabcd1234abcd5678.MyPlugin', + 'name' => 'doneeb5d2797aedd15d819b1a20425982850.MyPlugin', // Raw segment = abcd1234abcd5678 'value' => ArchiveWriter::DONE_OK, 'date1' => '2015-02-03', 'date2' => '2015-02-03', @@ -266,7 +266,7 @@ class RawArchiveDataWithTempAndInvalidated extends Fixture array( 'idarchive' => 22, 'idsite' => 2, - 'name' => 'doneabcd1234abcd5678', + 'name' => 'doneeb5d2797aedd15d819b1a20425982850', // Raw segment = abcd1234abcd5678 'value' => ArchiveWriter::DONE_OK, 'date1' => '2015-02-03', 'date2' => '2015-02-03', @@ -276,7 +276,7 @@ class RawArchiveDataWithTempAndInvalidated extends Fixture array( 'idarchive' => 23, 'idsite' => 2, - 'name' => 'doneabcd1234abcd5678.MyPlugin', + 'name' => 'doneeb5d2797aedd15d819b1a20425982850.MyPlugin', // Raw segment = abcd1234abcd5678 'value' => ArchiveWriter::DONE_OK, 'date1' => '2015-02-03', 'date2' => '2015-02-03', @@ -286,7 +286,7 @@ class RawArchiveDataWithTempAndInvalidated extends Fixture array( 'idarchive' => 24, 'idsite' => 1, - 'name' => 'done9876fedc5432abcd', + 'name' => 'done1e39a89fcc269acc36bd4d7c742763ed', // Raw segment = 9876fedc5432abcd 'value' => ArchiveWriter::DONE_OK, 'date1' => '2015-02-03', 'date2' => '2015-02-03', @@ -296,7 +296,7 @@ class RawArchiveDataWithTempAndInvalidated extends Fixture array( 'idarchive' => 25, 'idsite' => 2, - 'name' => 'donehash1', + 'name' => 'done00c6ee2e21a7548de6260cf72c4f4b5b', // Raw segment = hash1 'value' => ArchiveWriter::DONE_OK, 'date1' => '2015-02-03', 'date2' => '2015-02-03', @@ -306,7 +306,7 @@ class RawArchiveDataWithTempAndInvalidated extends Fixture array( 'idarchive' => 26, 'idsite' => 2, - 'name' => 'donehash2', + 'name' => 'done58833651db311ba4bc11cb26b1900b0f', // Raw segment = hash2 'value' => ArchiveWriter::DONE_OK, 'date1' => '2015-02-03', 'date2' => '2015-02-03', @@ -316,7 +316,7 @@ class RawArchiveDataWithTempAndInvalidated extends Fixture array( 'idarchive' => 27, 'idsite' => 2, - 'name' => 'donehash2.MyPlugin', + 'name' => 'done58833651db311ba4bc11cb26b1900b0f.MyPlugin', // Raw segment = hash2 'value' => ArchiveWriter::DONE_OK, 'date1' => '2015-02-03', 'date2' => '2015-02-03', @@ -326,7 +326,7 @@ class RawArchiveDataWithTempAndInvalidated extends Fixture array( 'idarchive' => 28, 'idsite' => 2, - 'name' => 'donehash3', + 'name' => 'done1a4ead8b39d17dfe89418452c9bba770', // Raw segment = hash3 'value' => ArchiveWriter::DONE_OK, 'date1' => '2015-02-03', 'date2' => '2015-02-03', @@ -334,9 +334,9 @@ class RawArchiveDataWithTempAndInvalidated extends Fixture 'ts_archived' => '2015-02-03 12:12:12' ), array( - 'idarchive' => 29 , + 'idarchive' => 29, 'idsite' => 2, - 'name' => 'donehash3', + 'name' => 'done1a4ead8b39d17dfe89418452c9bba770', // Raw segment = hash3 'value' => ArchiveWriter::DONE_OK, 'date1' => '2015-02-03', 'date2' => '2015-02-03', diff --git a/tests/PHPUnit/Integration/Archive/ArchivePurgerTest.php b/tests/PHPUnit/Integration/Archive/ArchivePurgerTest.php index 465988f4bd..8045376478 100644 --- a/tests/PHPUnit/Integration/Archive/ArchivePurgerTest.php +++ b/tests/PHPUnit/Integration/Archive/ArchivePurgerTest.php @@ -125,18 +125,46 @@ class ArchivePurgerTest extends IntegrationTestCase //Extra data set with segment and plugin archives self::$fixture->insertSegmentArchives($this->january); - $validSegmentIds = array( - 0 => ['DUMMYHASHSTR'], //valid for all sites - 1 => ['abcd1234abcd5678'], //valid for site 1. should be ignored for site 2 - 2 => ['hashthatdontexist', 'hash1', 'hash2'] + $segmentsToDelete = array( + array('definition' => '9876fedc5432abcd', 'enable_only_idsite' => 0), + array('definition' => 'hash3', 'enable_only_idsite' => 0), + // This segment also has archives for idsite = 1, which will be retained + array('definition' => 'abcd1234abcd5678', 'enable_only_idsite' => 2) ); //Archive #29 also has a deleted segment but it's before the purge threshold so it stays for now. - $deletedRowCount = $this->archivePurger->purgeDeletedSegmentArchives($this->january, $validSegmentIds); + $deletedRowCount = $this->archivePurger->purgeDeletedSegmentArchives($this->january, $segmentsToDelete); $this->assertEquals(4 * RawArchiveDataWithTempAndInvalidated::ROWS_PER_ARCHIVE, $deletedRowCount); self::$fixture->assertArchivesDoNotExist(array(22, 23, 24, 28), $this->january); } + public function test_purgeNoSegmentArchives_preservesSingleSiteSegmentArchivesForDeletedAllSiteSegment() + { + // Extra data set with segment and plugin archives + self::$fixture->insertSegmentArchives($this->january); + + $segmentsToDelete = array( + // This segment also has archives for idsite = 1, which will be retained + array('definition' => 'abcd1234abcd5678', 'enable_only_idsite' => 0, 'idsites_to_preserve' => array(2)) + ); + + // Archives for idsite=1 should be purged, but those for idsite=2 can stay + $deletedRowCount = $this->archivePurger->purgeDeletedSegmentArchives($this->january, $segmentsToDelete); + $this->assertEquals(2 * RawArchiveDataWithTempAndInvalidated::ROWS_PER_ARCHIVE, $deletedRowCount); + self::$fixture->assertArchivesDoNotExist(array(20, 21), $this->january); + } + + public function test_purgeNoSegmentArchives_blankSegmentName() + { + $segmentsToDelete = array( + array('definition' => '', 'enable_only_idsite' => 0) + ); + + // Should not purge all the "done%" archives! + $deletedRowCount = $this->archivePurger->purgeDeletedSegmentArchives($this->january, $segmentsToDelete); + $this->assertEquals(0, $deletedRowCount); + } + private function configureCustomRangePurging() { Config::getInstance()->General['purge_date_range_archives_after_X_days'] = 3; |