Tweak behaviour of orphaned segment archive purge (#14857)

2 files changed, 266 insertions, 1 deletions

diff --git a/plugins/SegmentEditor/Model.php b/plugins/SegmentEditor/Model.php
index 0025c7c15c..c505f7a4f2 100644
--- a/plugins/SegmentEditor/Model.php
+++ b/plugins/SegmentEditor/Model.php
@@ -129,6 +129,87 @@ class Model
         return $segment;
     }
 
+    /**
+     * Gets a list of segments that have been deleted in the last week and therefore may have orphaned archives.
+     * @param Date $date Segments deleted on or after this date will be returned.
+     * @return array of segments. The segments are only populated with the fields needed for archive invalidation
+     * (e.g. definition, enable_only_idsite).
+     * @throws \Exception
+     */
+    public function getSegmentsDeletedSince(Date $date)
+    {
+        $dateStr = $date->getDatetime();
+        $sql = "SELECT DISTINCT definition, enable_only_idsite FROM " . Common::prefixTable('segment')
+            . " WHERE deleted = 1 AND ts_last_edit >= ?";
+        $deletedSegments = Db::fetchAll($sql, array($dateStr));
+
+        if (empty($deletedSegments)) {
+            return array();
+        }
+
+        $existingSegments = $this->getExistingSegmentsLike($deletedSegments);
+
+        foreach ($deletedSegments as $i => $deleted) {
+            $deletedSegments[$i]['idsites_to_preserve'] = array();
+            foreach ($existingSegments as $existing) {
+                if ($existing['definition'] != $deleted['definition'] &&
+                    $existing['definition'] != urlencode($deleted['definition']) &&
+                    $existing['definition'] != urldecode($deleted['definition'])
+                ) {
+                    continue;
+                }
+
+                if (
+                    $existing['enable_only_idsite'] == $deleted['enable_only_idsite']
+                    || $existing['enable_only_idsite'] == 0
+                ) {
+                    // There is an identical segment (for either the specific site or for all sites) that is active
+                    // The archives for this segment will therefore still be needed
+                    unset($deletedSegments[$i]);
+                    break;
+                } elseif ($deleted['enable_only_idsite'] == 0) {
+                    // It is an all-sites segment that got deleted, but there is a single-site segment that is active
+                    // Need to make sure we don't erase the segment's archives for that particular site
+                    $deletedSegments[$i]['idsites_to_preserve'][] = $existing['enable_only_idsite'];
+                }
+            }
+        }
+
+        return $deletedSegments;
+    }
+
+    private function getExistingSegmentsLike(array $segments)
+    {
+        if (empty($segments)) {
+            return array();
+        }
+
+        $whereClauses = array();
+        $bind = array();
+        $definitionWhereClauseTemplate = '(definition = ? OR definition = ? OR definition = ?)';
+        foreach ($segments as $segment) {
+            // Sometimes they are stored encoded and sometimes they aren't
+            $bind[] = $segment['definition'];
+            $bind[] = urlencode($segment['definition']);
+            $bind[] = urldecode($segment['definition']);
+
+            if ($segment['enable_only_idsite'] == 0) {
+                // They deleted an all-sites segment, but there is a single-site segment with same definition?
+                // Need to handle this carefully so that the archives for the single-site segment are preserved
+                $whereClauses[] = "$definitionWhereClauseTemplate";
+            } else {
+                $whereClauses[] = "($definitionWhereClauseTemplate AND (enable_only_idsite = ? OR enable_only_idsite = 0))";
+                $bind[] = $segment['enable_only_idsite'];
+            }
+        }
+        $whereClauses = implode(' OR ', $whereClauses);
+
+        // Check for any non-deleted segments with the same definition
+        $sql = "SELECT DISTINCT definition, enable_only_idsite FROM " . Common::prefixTable('segment')
+            . " WHERE deleted = 0 AND (" . $whereClauses . ")";
+        return Db::fetchAll($sql, $bind);
+    }
+
     public function deleteSegment($idSegment)
     {
         $fieldsToSet = array(
diff --git a/plugins/SegmentEditor/tests/Integration/ModelTest.php b/plugins/SegmentEditor/tests/Integration/ModelTest.php
index 92034f3efc..f309221c8f 100644
--- a/plugins/SegmentEditor/tests/Integration/ModelTest.php
+++ b/plugins/SegmentEditor/tests/Integration/ModelTest.php
@@ -25,6 +25,8 @@ class ModelTest extends IntegrationTestCase
 
     private $idSegment3;
 
+    private $idSegment4;
+
     public function setUp()
     {
         parent::setUp();
@@ -55,7 +57,11 @@ class ModelTest extends IntegrationTestCase
     {
         parent::tearDown();
         // Force a hard delete of segment
-        $idsToDelete = $this->idSegment1 . ', ' . $this->idSegment2 . ', ' . $this->idSegment3;
+        $idsToDelete = array($this->idSegment1, $this->idSegment2, $this->idSegment3);
+        if ($this->idSegment4) {
+            $idsToDelete[] = $this->idSegment4;
+        }
+        $idsToDelete = implode(',', $idsToDelete);
         Db::query(
             "DELETE FROM " . Common::prefixTable('segment') . " WHERE idsegment IN ($idsToDelete)"
         );
@@ -146,6 +152,184 @@ class ModelTest extends IntegrationTestCase
         $this->assertEmpty($segment);
     }
 
+    public function test_getSegmentsDeletedSince_noDeletedSegments()
+    {
+        $date = Date::factory('now');
+        $segments = $this->model->getSegmentsDeletedSince($date);
+        $this->assertEmpty($segments);
+    }
+
+    public function test_getSegmentsDeletedSince_oneDeletedSegment()
+    {
+        $this->model->deleteSegment($this->idSegment3);
+
+        $date = Date::factory('now')->subDay(1);
+        $segments = $this->model->getSegmentsDeletedSince($date);
+
+        $this->assertCount(1, $segments);
+        $this->assertEquals('country==Hobbiton', $segments[0]['definition']);
+    }
+
+    public function test_getSegmentsDeletedSince_segmentDeletedTooLongAgo()
+    {
+        // Manually delete it to set timestamp 9 days in past
+        $deletedAt = Date::factory('now')->subDay(9)->toString('Y-m-d H:i:s');
+        $this->model->updateSegment($this->idSegment1, array(
+            'deleted' => 1,
+            'ts_last_edit' => $deletedAt
+        ));
+
+        // The segment deleted above should not be included as it was more than 8 days ago
+        $date = Date::factory('now')->subDay(8);
+        $segments = $this->model->getSegmentsDeletedSince($date);
+
+        $this->assertEmpty($segments);
+    }
+
+    public function test_getSegmentsDeletedSince_duplicateSegment()
+    {
+        // Turn segment1 into a duplicate of segment2, except it's also deleted
+        $this->model->updateSegment($this->idSegment1, array(
+            'definition' => 'country==Genovia',
+            'deleted' => 1,
+            'ts_last_edit' => Date::factory('now')->toString('Y-m-d H:i:s')
+        ));
+
+        $date = Date::factory('now')->subDay(8);
+        $segments = $this->model->getSegmentsDeletedSince($date);
+
+        $this->assertEmpty($segments);
+    }
+
+    public function test_getSegmentsDeletedSince_duplicateSegmentDifferentIdSite()
+    {
+        // Turn segment2 into a duplicate of segment3, except for a different idsite and also deleted
+        $this->model->updateSegment($this->idSegment2, array(
+            'definition' => 'country==Hobbiton',
+            'enable_only_idsite' => 2,
+            'deleted' => 1,
+            'ts_last_edit' => Date::factory('now')->toString('Y-m-d H:i:s')
+        ));
+
+        $date = Date::factory('now')->subDay(8);
+        $segments = $this->model->getSegmentsDeletedSince($date);
+
+        $this->assertCount(1, $segments);
+        $this->assertEquals('country==Hobbiton', $segments[0]['definition']);
+        $this->assertEquals(2, $segments[0]['enable_only_idsite']);
+    }
+
+    public function test_getSegmentsDeletedSince_duplicateSegmentAllSitesAndSingleSite()
+    {
+        // Turn segment2 into a duplicate of segment3, except for all sites and also deleted
+        $this->model->updateSegment($this->idSegment2, array(
+            'definition' => 'country==Hobbiton',
+            'enable_only_idsite' => 0,
+            'deleted' => 1,
+            'ts_last_edit' => Date::factory('now')->toString('Y-m-d H:i:s')
+        ));
+
+        $date = Date::factory('now')->subDay(8);
+        $segments = $this->model->getSegmentsDeletedSince($date);
+
+        $this->assertCount(1, $segments);
+        $this->assertEquals('country==Hobbiton', $segments[0]['definition']);
+        $this->assertEquals(0, $segments[0]['enable_only_idsite']);
+        $this->assertEquals(array(1), $segments[0]['idsites_to_preserve']);
+    }
+
+    public function test_getSegmentsDeletedSince_duplicateSegmentSingleSiteAndAllSites()
+    {
+        // Turn segment3 into a duplicate of segment1, except for a single site and deleted
+        $this->model->updateSegment($this->idSegment3, array(
+            'definition' => 'country==Narnia',
+            'enable_only_idsite' => 1,
+            'deleted' => 1,
+            'ts_last_edit' => Date::factory('now')->toString('Y-m-d H:i:s')
+        ));
+
+        $date = Date::factory('now')->subDay(8);
+        $segments = $this->model->getSegmentsDeletedSince($date);
+
+        // There is still a live segment for all sites, so the deleted site-specific one is ignored
+        $this->assertEmpty($segments);
+    }
+
+    public function test_getSegmentsDeletedSince_ExistingSiteSpecificAndAllSitesMatch()
+    {
+        // A deleted all-sites segment, with both an all-sites and a site-specific segment still present
+        $this->model->updateSegment($this->idSegment1, array(
+            'definition' => 'actions >= 1',
+            'enable_only_idsite' => 0,
+            'deleted' => 0,
+            'ts_last_edit' => Date::factory('now')->toString('Y-m-d H:i:s')
+        ));
+        $this->model->updateSegment($this->idSegment2, array(
+            'definition' => 'actions >= 1',
+            'enable_only_idsite' => 0,
+            'deleted' => 1,
+            'ts_last_edit' => Date::factory('now')->toString('Y-m-d H:i:s')
+        ));
+        $this->model->updateSegment($this->idSegment3, array(
+            'definition' => 'actions >= 1',
+            'enable_only_idsite' => 3,
+            'deleted' => 0,
+            'ts_last_edit' => Date::factory('now')->toString('Y-m-d H:i:s')
+        ));
+        $this->idSegment4 = $this->model->createSegment(array(
+            'definition' => 'actions >= 1',
+            'enable_only_idsite' => 1,
+            'deleted' => 1,
+            'ts_last_edit' => Date::factory('now')->toString('Y-m-d H:i:s')
+        ));
+
+        $date = Date::factory('now')->subDay(8);
+        $segments = $this->model->getSegmentsDeletedSince($date);
+
+        $this->assertEmpty($segments);
+    }
+
+    public function test_getSegmentsDeletedSince_urlDecodedVersionOfSegment()
+    {
+        // Turn segment2 into a duplicate of segment3, except a urlencoded version
+        $this->model->updateSegment($this->idSegment2, array(
+            'definition' => 'country%3D%3DHobbiton',
+            'enable_only_idsite' => 1,
+            'deleted' => 1,
+            'ts_last_edit' => Date::factory('now')->toString('Y-m-d H:i:s')
+        ));
+
+        $date = Date::factory('now')->subDay(8);
+        $segments = $this->model->getSegmentsDeletedSince($date);
+
+        // The two encoded and decoded version of the segments should be treated as duplicates
+        // This means there segment has a non-deleted version so it's not returned
+        $this->assertEmpty($segments);
+    }
+
+    public function test_getSegmentsDeletedSince_urlEncodedVersionOfSegment()
+    {
+        // segment1 => url decoded version, deleted
+        $this->model->updateSegment($this->idSegment1, array(
+            'definition' => 'country==Narnia',
+            'deleted' => 1,
+            'ts_last_edit' => Date::factory('now')->toString('Y-m-d H:i:s')
+        ));
+        // segment2 => url encoded version, not deleted
+        $this->model->updateSegment($this->idSegment2, array(
+            'definition' => 'country%3D%3DNarnia',
+            'deleted' => 0,
+            'ts_last_edit' => Date::factory('now')->toString('Y-m-d H:i:s')
+        ));
+
+        $date = Date::factory('now')->subDay(8);
+        $segments = $this->model->getSegmentsDeletedSince($date);
+
+        // The two encoded and decoded version of the segments should be treated as duplicates
+        // This means there segment has a non-deleted version so it's not returned
+        $this->assertEmpty($segments);
+    }
+
     private function assertReturnedIdsMatch(array $expectedIds, array $resultSet)
     {
         $this->assertEquals(count($expectedIds), count($resultSet));