segment invalidations must be paired w/ normal invalidations (#16845)

1 files changed, 180 insertions, 127 deletions

diff --git a/tests/PHPUnit/Integration/CronArchiveTest.php b/tests/PHPUnit/Integration/CronArchiveTest.php
index 4a1719f346..26830b4820 100644
--- a/tests/PHPUnit/Integration/CronArchiveTest.php
+++ b/tests/PHPUnit/Integration/CronArchiveTest.php
@@ -8,6 +8,7 @@
 
 namespace Piwik\Tests\Integration;
 
+use Monolog\Logger;
 use Piwik\ArchiveProcessor\Parameters;
 use Piwik\ArchiveProcessor\Rules;
 use Piwik\Container\StaticContainer;
@@ -34,6 +35,130 @@ use Psr\Log\NullLogger;
  */
 class CronArchiveTest extends IntegrationTestCase
 {
+    /**
+     * @dataProvider getTestDataForInvalidateRecentDate
+     */
+    public function test_invalidateRecentDate_invalidatesCorrectPeriodsAndSegments($dateStr, $segments,
+                                                                                   $expectedInvalidationCalls)
+    {
+        $idSite = Fixture::createWebsite('2019-04-04 03:45:45', 0, false, false, 1, null, null, 'Australia/Sydney');
+
+        Rules::setBrowserTriggerArchiving(false);
+        foreach ($segments as $idx => $segment) {
+            SegmentAPI::getInstance()->add('segment #' . $idx, $segment, $idx % 2 === 0 ? $idSite : false, true, true);
+        }
+        Rules::setBrowserTriggerArchiving(true);
+
+        $t = Fixture::getTracker($idSite, Date::yesterday()->addHour(2)->getDatetime());
+        $t->setUrl('http://someurl.com/abc');
+        Fixture::checkResponse($t->doTrackPageView('some page'));
+
+        $t = Fixture::getTracker($idSite, Date::today()->addHour(2)->getDatetime());
+        $t->setUrl('http://someurl.com/def');
+        Fixture::checkResponse($t->doTrackPageView('some page 2'));
+
+        $mockInvalidateApi = $this->getMockInvalidateApi();
+
+        $archiver = new CronArchive();
+        $archiver->init();
+        $archiver->setApiToInvalidateArchivedReport($mockInvalidateApi);
+
+        $archiver->invalidateRecentDate($dateStr, $idSite);
+
+        $actualInvalidationCalls = $mockInvalidateApi->getInvalidations();
+
+        $this->assertEquals($expectedInvalidationCalls, $actualInvalidationCalls);
+    }
+
+    public function getTestDataForInvalidateRecentDate()
+    {
+        $segments = [
+            'browserCode==IE',
+            'visitCount>5',
+        ];
+
+        return [
+            [
+                'today',
+                $segments,
+                [
+                    array (
+                        1,
+                        '2020-02-03',
+                        'day',
+                        false,
+                        false,
+                        false,
+                    ),
+                    array (
+                        1,
+                        '2020-02-03',
+                        'day',
+                        'browserCode==IE',
+                        false,
+                        false,
+                    ),
+                    array (
+                        1,
+                        '2020-02-03',
+                        'day',
+                        'visitCount>5',
+                        false,
+                        false,
+                    ),
+                ],
+            ],
+            [
+                'yesterday',
+                $segments,
+                [
+                    array (
+                        1,
+                        '2020-02-02',
+                        'day',
+                        false,
+                        false,
+                        false,
+                    ),
+                    array (
+                        1,
+                        '2020-02-02',
+                        'day',
+                        'browserCode==IE',
+                        false,
+                        false,
+                    ),
+                    array (
+                        1,
+                        '2020-02-02',
+                        'day',
+                        'visitCount>5',
+                        false,
+                        false,
+                    ),
+                ],
+            ],
+        ];
+    }
+
+    private function getMockInvalidateApi()
+    {
+        $mock = new class {
+            private $calls = [];
+
+            public function invalidateArchivedReports()
+            {
+                $this->calls[] = func_get_args();
+            }
+
+            public function getInvalidations()
+            {
+                return $this->calls;
+            }
+        };
+        return $mock;
+    }
+
     public function test_isThereExistingValidPeriod_returnsTrueIfPeriodHasToday_AndExistingArchiveIsNewEnough()
     {
         Fixture::createWebsite('2019-04-04 03:45:45');
@@ -51,7 +176,7 @@ class CronArchiveTest extends IntegrationTestCase
             1, 1,2, '2020-03-30', '2020-04-05', 'done', ArchiveWriter::DONE_OK, $tsArchived
         ]);
 
-        $actual =$archiver->isThereExistingValidPeriod($params, $isYesterday = false);
+        $actual =$archiver->isThereExistingValidPeriod($params);
         $this->assertTrue($actual);
     }
 
@@ -166,20 +291,18 @@ class CronArchiveTest extends IntegrationTestCase
          */
         $invalidatedReports = $api->getInvalidatedReports();
         $this->assertCount(3, $invalidatedReports);
-        sort($invalidatedReports[0][0]);
-        sort($invalidatedReports[1][0]);
-        sort($invalidatedReports[2][0]);
+
         usort($invalidatedReports, function ($a, $b) {
             return strcmp($a[1], $b[1]);
         });
 
-        $this->assertSame(array(1), $invalidatedReports[0][0]);
+        $this->assertSame(1, $invalidatedReports[0][0]);
         $this->assertSame('2014-04-05', $invalidatedReports[0][1]);
 
-        $this->assertSame(array(2), $invalidatedReports[1][0]);
+        $this->assertSame(2, $invalidatedReports[1][0]);
         $this->assertSame('2014-04-05', $invalidatedReports[1][1]);
 
-        $this->assertSame(array(2), $invalidatedReports[2][0]);
+        $this->assertSame(2, $invalidatedReports[2][0]);
         $this->assertSame('2014-04-06', $invalidatedReports[2][1]);
     }
 
@@ -307,148 +430,78 @@ Checking for queued invalidations...
   Segment "actions>=2" was created or changed recently and will therefore archive today (for site ID = 1)
   Segment "actions>=4" was created or changed recently and will therefore archive today (for site ID = 1)
 Done invalidating
-Skipping invalidated archive 73, segment 'actions>=4' is not in --force-idsegments: [idinvalidation = 73, idsite = 1, period = day(2020-02-03 - 2020-02-03), name = done49a9440bd6dba4b8850035e09d043c67]
-No usable archive exists (ts_archived of existing = , now = 2020-02-03 04:05:06).
-Found invalidated archive we can skip (no visits): [idinvalidation = 43, idsite = 1, period = day(2020-02-03 - 2020-02-03), name = donee0512c03f7c20af6ef96a8d792c6bb9f]
-Skipping invalidated archive 74, segment 'actions>=4' is not in --force-idsegments: [idinvalidation = 74, idsite = 1, period = week(2020-02-03 - 2020-02-09), name = done49a9440bd6dba4b8850035e09d043c67]
-No usable archive exists (ts_archived of existing = , now = 2020-02-03 04:05:06).
-Found invalidated archive we can skip (no visits): [idinvalidation = 44, idsite = 1, period = week(2020-02-03 - 2020-02-09), name = donee0512c03f7c20af6ef96a8d792c6bb9f]
-Skipping invalidated archive 72, segment 'actions>=4' is not in --force-idsegments: [idinvalidation = 72, idsite = 1, period = day(2020-02-02 - 2020-02-02), name = done49a9440bd6dba4b8850035e09d043c67]
-No usable archive exists (ts_archived of existing = , now = 2020-02-03 04:05:06).
-Found invalidated archive we can skip (no visits): [idinvalidation = 42, idsite = 1, period = day(2020-02-02 - 2020-02-02), name = donee0512c03f7c20af6ef96a8d792c6bb9f]
-Skipping invalidated archive 70, segment 'actions>=4' is not in --force-idsegments: [idinvalidation = 70, idsite = 1, period = day(2020-02-01 - 2020-02-01), name = done49a9440bd6dba4b8850035e09d043c67]
-No usable archive exists (ts_archived of existing = , now = 2020-02-03 04:05:06).
-Found invalidated archive we can skip (no visits): [idinvalidation = 40, idsite = 1, period = day(2020-02-01 - 2020-02-01), name = donee0512c03f7c20af6ef96a8d792c6bb9f]
-Skipping invalidated archive 71, segment 'actions>=4' is not in --force-idsegments: [idinvalidation = 71, idsite = 1, period = month(2020-02-01 - 2020-02-29), name = done49a9440bd6dba4b8850035e09d043c67]
-No usable archive exists (ts_archived of existing = , now = 2020-02-03 04:05:06).
-Found invalidated archive we can skip (no visits): [idinvalidation = 41, idsite = 1, period = month(2020-02-01 - 2020-02-29), name = donee0512c03f7c20af6ef96a8d792c6bb9f]
-Skipping invalidated archive 66, segment 'actions>=4' is not in --force-idsegments: [idinvalidation = 66, idsite = 1, period = week(2020-01-27 - 2020-02-02), name = done49a9440bd6dba4b8850035e09d043c67]
-No usable archive exists (ts_archived of existing = , now = 2020-02-03 04:05:06).
-Found invalidated archive we can skip (no visits): [idinvalidation = 36, idsite = 1, period = week(2020-01-27 - 2020-02-02), name = donee0512c03f7c20af6ef96a8d792c6bb9f]
-Skipping invalidated archive 63, segment 'actions>=4' is not in --force-idsegments: [idinvalidation = 63, idsite = 1, period = day(2020-01-01 - 2020-01-01), name = done49a9440bd6dba4b8850035e09d043c67]
-No usable archive exists (ts_archived of existing = , now = 2020-02-03 04:05:06).
-Found invalidated archive we can skip (no visits): [idinvalidation = 33, idsite = 1, period = day(2020-01-01 - 2020-01-01), name = donee0512c03f7c20af6ef96a8d792c6bb9f]
-Skipping invalidated archive 64, segment 'actions>=4' is not in --force-idsegments: [idinvalidation = 64, idsite = 1, period = month(2020-01-01 - 2020-01-31), name = done49a9440bd6dba4b8850035e09d043c67]
-No usable archive exists (ts_archived of existing = , now = 2020-02-03 04:05:06).
-Found invalidated archive we can skip (no visits): [idinvalidation = 34, idsite = 1, period = month(2020-01-01 - 2020-01-31), name = donee0512c03f7c20af6ef96a8d792c6bb9f]
-Skipping invalidated archive 65, segment 'actions>=4' is not in --force-idsegments: [idinvalidation = 65, idsite = 1, period = year(2020-01-01 - 2020-12-31), name = done49a9440bd6dba4b8850035e09d043c67]
-No usable archive exists (ts_archived of existing = , now = 2020-02-03 04:05:06).
-Found invalidated archive we can skip (no visits): [idinvalidation = 35, idsite = 1, period = year(2020-01-01 - 2020-12-31), name = donee0512c03f7c20af6ef96a8d792c6bb9f]
-Skipping invalidated archive 58, segment 'actions>=4' is not in --force-idsegments: [idinvalidation = 58, idsite = 1, period = day(2019-12-31 - 2019-12-31), name = done49a9440bd6dba4b8850035e09d043c67]
-No usable archive exists (ts_archived of existing = , now = 2020-02-03 04:05:06).
-Found invalidated archive we can skip (no visits): [idinvalidation = 28, idsite = 1, period = day(2019-12-31 - 2019-12-31), name = donee0512c03f7c20af6ef96a8d792c6bb9f]
-Skipping invalidated archive 56, segment 'actions>=4' is not in --force-idsegments: [idinvalidation = 56, idsite = 1, period = day(2019-12-30 - 2019-12-30), name = done49a9440bd6dba4b8850035e09d043c67]
-No usable archive exists (ts_archived of existing = , now = 2020-02-03 04:05:06).
-Found invalidated archive we can skip (no visits): [idinvalidation = 26, idsite = 1, period = day(2019-12-30 - 2019-12-30), name = donee0512c03f7c20af6ef96a8d792c6bb9f]
-Skipping invalidated archive 57, segment 'actions>=4' is not in --force-idsegments: [idinvalidation = 57, idsite = 1, period = week(2019-12-30 - 2020-01-05), name = done49a9440bd6dba4b8850035e09d043c67]
-No usable archive exists (ts_archived of existing = , now = 2020-02-03 04:05:06).
-Found invalidated archive we can skip (no visits): [idinvalidation = 27, idsite = 1, period = week(2019-12-30 - 2020-01-05), name = donee0512c03f7c20af6ef96a8d792c6bb9f]
-Skipping invalidated archive 54, segment 'actions>=4' is not in --force-idsegments: [idinvalidation = 54, idsite = 1, period = day(2019-12-23 - 2019-12-23), name = done49a9440bd6dba4b8850035e09d043c67]
-No usable archive exists (ts_archived of existing = , now = 2020-02-03 04:05:06).
-Found invalidated archive we can skip (no visits): [idinvalidation = 24, idsite = 1, period = day(2019-12-23 - 2019-12-23), name = donee0512c03f7c20af6ef96a8d792c6bb9f]
-Skipping invalidated archive 55, segment 'actions>=4' is not in --force-idsegments: [idinvalidation = 55, idsite = 1, period = week(2019-12-23 - 2019-12-29), name = done49a9440bd6dba4b8850035e09d043c67]
-No usable archive exists (ts_archived of existing = , now = 2020-02-03 04:05:06).
-Found invalidated archive we can skip (no visits): [idinvalidation = 25, idsite = 1, period = week(2019-12-23 - 2019-12-29), name = donee0512c03f7c20af6ef96a8d792c6bb9f]
-Skipping invalidated archive 52, segment 'actions>=4' is not in --force-idsegments: [idinvalidation = 52, idsite = 1, period = day(2019-12-16 - 2019-12-16), name = done49a9440bd6dba4b8850035e09d043c67]
-No usable archive exists (ts_archived of existing = , now = 2020-02-03 04:05:06).
-Found invalidated archive we can skip (no visits): [idinvalidation = 22, idsite = 1, period = day(2019-12-16 - 2019-12-16), name = donee0512c03f7c20af6ef96a8d792c6bb9f]
-Skipping invalidated archive 53, segment 'actions>=4' is not in --force-idsegments: [idinvalidation = 53, idsite = 1, period = week(2019-12-16 - 2019-12-22), name = done49a9440bd6dba4b8850035e09d043c67]
-No usable archive exists (ts_archived of existing = , now = 2020-02-03 04:05:06).
-Found invalidated archive we can skip (no visits): [idinvalidation = 23, idsite = 1, period = week(2019-12-16 - 2019-12-22), name = donee0512c03f7c20af6ef96a8d792c6bb9f]
-Skipping invalidated archive 1, segment '' is not in --force-idsegments: [idinvalidation = 1, idsite = 1, period = day(2019-12-12 - 2019-12-12), name = done]
-Skipping invalidated archive 5, segment '' is not in --force-idsegments: [idinvalidation = 5, idsite = 1, period = day(2019-12-11 - 2019-12-11), name = done]
-Skipping invalidated archive 9, segment '' is not in --force-idsegments: [idinvalidation = 9, idsite = 1, period = day(2019-12-10 - 2019-12-10), name = done]
-Skipping invalidated archive 50, segment 'actions>=4' is not in --force-idsegments: [idinvalidation = 50, idsite = 1, period = day(2019-12-09 - 2019-12-09), name = done49a9440bd6dba4b8850035e09d043c67]
-No usable archive exists (ts_archived of existing = , now = 2020-02-03 04:05:06).
-Found invalidated archive we can skip (no visits): [idinvalidation = 20, idsite = 1, period = day(2019-12-09 - 2019-12-09), name = donee0512c03f7c20af6ef96a8d792c6bb9f]
-Skipping invalidated archive 10, segment '' is not in --force-idsegments: [idinvalidation = 10, idsite = 1, period = week(2019-12-09 - 2019-12-15), name = done]
-Skipping invalidated archive 6, segment '' is not in --force-idsegments: [idinvalidation = 6, idsite = 1, period = week(2019-12-09 - 2019-12-15), name = done]
-Skipping invalidated archive 2, segment '' is not in --force-idsegments: [idinvalidation = 2, idsite = 1, period = week(2019-12-09 - 2019-12-15), name = done]
-Skipping invalidated archive 51, segment 'actions>=4' is not in --force-idsegments: [idinvalidation = 51, idsite = 1, period = week(2019-12-09 - 2019-12-15), name = done49a9440bd6dba4b8850035e09d043c67]
-No usable archive exists (ts_archived of existing = , now = 2020-02-03 04:05:06).
-Processing invalidation: [idinvalidation = 21, idsite = 1, period = week(2019-12-09 - 2019-12-15), name = donee0512c03f7c20af6ef96a8d792c6bb9f].
-Skipping invalidated archive 13, segment '' is not in --force-idsegments: [idinvalidation = 13, idsite = 1, period = day(2019-12-02 - 2019-12-02), name = done]
-Skipping invalidated archive 47, segment 'actions>=4' is not in --force-idsegments: [idinvalidation = 47, idsite = 1, period = day(2019-12-02 - 2019-12-02), name = done49a9440bd6dba4b8850035e09d043c67]
-No usable archive exists (ts_archived of existing = , now = 2020-02-03 04:05:06).
-Processing invalidation: [idinvalidation = 17, idsite = 1, period = day(2019-12-02 - 2019-12-02), name = donee0512c03f7c20af6ef96a8d792c6bb9f].
-Found archive with intersecting period with others in concurrent batch, skipping until next batch: [idinvalidation = 14, idsite = 1, period = week(2019-12-02 - 2019-12-08), name = done]
-Found archive with intersecting period with others in concurrent batch, skipping until next batch: [idinvalidation = 48, idsite = 1, period = week(2019-12-02 - 2019-12-08), name = done49a9440bd6dba4b8850035e09d043c67]
-Found archive with intersecting period with others in concurrent batch, skipping until next batch: [idinvalidation = 18, idsite = 1, period = week(2019-12-02 - 2019-12-08), name = donee0512c03f7c20af6ef96a8d792c6bb9f]
-Found archive with intersecting period with others in concurrent batch, skipping until next batch: [idinvalidation = 15, idsite = 1, period = month(2019-12-01 - 2019-12-31), name = done]
-Found archive with intersecting period with others in concurrent batch, skipping until next batch: [idinvalidation = 11, idsite = 1, period = month(2019-12-01 - 2019-12-31), name = done]
-Found archive with intersecting period with others in concurrent batch, skipping until next batch: [idinvalidation = 7, idsite = 1, period = month(2019-12-01 - 2019-12-31), name = done]
-Found archive with intersecting period with others in concurrent batch, skipping until next batch: [idinvalidation = 3, idsite = 1, period = month(2019-12-01 - 2019-12-31), name = done]
-Found archive with intersecting period with others in concurrent batch, skipping until next batch: [idinvalidation = 49, idsite = 1, period = month(2019-12-01 - 2019-12-31), name = done49a9440bd6dba4b8850035e09d043c67]
-Found archive with intersecting period with others in concurrent batch, skipping until next batch: [idinvalidation = 19, idsite = 1, period = month(2019-12-01 - 2019-12-31), name = donee0512c03f7c20af6ef96a8d792c6bb9f]
-Found archive with intersecting period with others in concurrent batch, skipping until next batch: [idinvalidation = 16, idsite = 1, period = year(2019-01-01 - 2019-12-31), name = done]
-Found archive with intersecting period with others in concurrent batch, skipping until next batch: [idinvalidation = 12, idsite = 1, period = year(2019-01-01 - 2019-12-31), name = done]
-Found archive with intersecting period with others in concurrent batch, skipping until next batch: [idinvalidation = 8, idsite = 1, period = year(2019-01-01 - 2019-12-31), name = done]
-Found archive with intersecting period with others in concurrent batch, skipping until next batch: [idinvalidation = 4, idsite = 1, period = year(2019-01-01 - 2019-12-31), name = done]
-Found archive with intersecting period with others in concurrent batch, skipping until next batch: [idinvalidation = 62, idsite = 1, period = year(2019-01-01 - 2019-12-31), name = done49a9440bd6dba4b8850035e09d043c67]
-Found archive with intersecting period with others in concurrent batch, skipping until next batch: [idinvalidation = 32, idsite = 1, period = year(2019-01-01 - 2019-12-31), name = donee0512c03f7c20af6ef96a8d792c6bb9f]
+Found invalidated archive we can skip (no visits): [idinvalidation = 75, idsite = 1, period = day(2020-02-03 - 2020-02-03), name = donee0512c03f7c20af6ef96a8d792c6bb9f]
+Found invalidated archive we can skip (no visits): [idinvalidation = 76, idsite = 1, period = week(2020-02-03 - 2020-02-09), name = donee0512c03f7c20af6ef96a8d792c6bb9f]
+Found invalidated archive we can skip (no visits): [idinvalidation = 74, idsite = 1, period = day(2020-02-02 - 2020-02-02), name = donee0512c03f7c20af6ef96a8d792c6bb9f]
+Found invalidated archive we can skip (no visits): [idinvalidation = 72, idsite = 1, period = day(2020-02-01 - 2020-02-01), name = donee0512c03f7c20af6ef96a8d792c6bb9f]
+Found invalidated archive we can skip (no visits): [idinvalidation = 73, idsite = 1, period = month(2020-02-01 - 2020-02-29), name = donee0512c03f7c20af6ef96a8d792c6bb9f]
+Found invalidated archive we can skip (no visits): [idinvalidation = 68, idsite = 1, period = week(2020-01-27 - 2020-02-02), name = donee0512c03f7c20af6ef96a8d792c6bb9f]
+Found invalidated archive we can skip (no visits): [idinvalidation = 65, idsite = 1, period = day(2020-01-01 - 2020-01-01), name = donee0512c03f7c20af6ef96a8d792c6bb9f]
+Found invalidated archive we can skip (no visits): [idinvalidation = 66, idsite = 1, period = month(2020-01-01 - 2020-01-31), name = donee0512c03f7c20af6ef96a8d792c6bb9f]
+Found invalidated archive we can skip (no visits): [idinvalidation = 67, idsite = 1, period = year(2020-01-01 - 2020-12-31), name = donee0512c03f7c20af6ef96a8d792c6bb9f]
+Found invalidated archive we can skip (no visits): [idinvalidation = 60, idsite = 1, period = day(2019-12-31 - 2019-12-31), name = donee0512c03f7c20af6ef96a8d792c6bb9f]
+Found invalidated archive we can skip (no visits): [idinvalidation = 58, idsite = 1, period = day(2019-12-30 - 2019-12-30), name = donee0512c03f7c20af6ef96a8d792c6bb9f]
+Found invalidated archive we can skip (no visits): [idinvalidation = 59, idsite = 1, period = week(2019-12-30 - 2020-01-05), name = donee0512c03f7c20af6ef96a8d792c6bb9f]
+Found invalidated archive we can skip (no visits): [idinvalidation = 56, idsite = 1, period = day(2019-12-23 - 2019-12-23), name = donee0512c03f7c20af6ef96a8d792c6bb9f]
+Found invalidated archive we can skip (no visits): [idinvalidation = 57, idsite = 1, period = week(2019-12-23 - 2019-12-29), name = donee0512c03f7c20af6ef96a8d792c6bb9f]
+Found invalidated archive we can skip (no visits): [idinvalidation = 54, idsite = 1, period = day(2019-12-16 - 2019-12-16), name = donee0512c03f7c20af6ef96a8d792c6bb9f]
+Found invalidated archive we can skip (no visits): [idinvalidation = 55, idsite = 1, period = week(2019-12-16 - 2019-12-22), name = donee0512c03f7c20af6ef96a8d792c6bb9f]
+Processing invalidation: [idinvalidation = 5, idsite = 1, period = day(2019-12-12 - 2019-12-12), name = donee0512c03f7c20af6ef96a8d792c6bb9f].
+Processing invalidation: [idinvalidation = 17, idsite = 1, period = day(2019-12-11 - 2019-12-11), name = donee0512c03f7c20af6ef96a8d792c6bb9f].
+Processing invalidation: [idinvalidation = 29, idsite = 1, period = day(2019-12-10 - 2019-12-10), name = donee0512c03f7c20af6ef96a8d792c6bb9f].
+Starting archiving for ?module=API&method=CoreAdminHome.archiveReports&idSite=1&period=day&date=2019-12-12&format=json&segment=actions%3E%3D2&trigger=archivephp
+Starting archiving for ?module=API&method=CoreAdminHome.archiveReports&idSite=1&period=day&date=2019-12-11&format=json&segment=actions%3E%3D2&trigger=archivephp
+Starting archiving for ?module=API&method=CoreAdminHome.archiveReports&idSite=1&period=day&date=2019-12-10&format=json&segment=actions%3E%3D2&trigger=archivephp
+Archived website id 1, period = day, date = 2019-12-12, segment = 'actions%3E%3D2', 0 visits found. Time elapsed: %fs
+Archived website id 1, period = day, date = 2019-12-11, segment = 'actions%3E%3D2', 0 visits found. Time elapsed: %fs
+Archived website id 1, period = day, date = 2019-12-10, segment = 'actions%3E%3D2', 0 visits found. Time elapsed: %fs
+Found invalidated archive we can skip (no visits): [idinvalidation = 52, idsite = 1, period = day(2019-12-09 - 2019-12-09), name = donee0512c03f7c20af6ef96a8d792c6bb9f]
+Processing invalidation: [idinvalidation = 53, idsite = 1, period = week(2019-12-09 - 2019-12-15), name = donee0512c03f7c20af6ef96a8d792c6bb9f].
+Processing invalidation: [idinvalidation = 49, idsite = 1, period = day(2019-12-02 - 2019-12-02), name = donee0512c03f7c20af6ef96a8d792c6bb9f].
 No next invalidated archive.
 Starting archiving for ?module=API&method=CoreAdminHome.archiveReports&idSite=1&period=week&date=2019-12-09&format=json&segment=actions%3E%3D2&trigger=archivephp
 Starting archiving for ?module=API&method=CoreAdminHome.archiveReports&idSite=1&period=day&date=2019-12-02&format=json&segment=actions%3E%3D2&trigger=archivephp
 Archived website id 1, period = week, date = 2019-12-09, segment = 'actions%3E%3D2', 0 visits found. Time elapsed: %fs
 Archived website id 1, period = day, date = 2019-12-02, segment = 'actions%3E%3D2', 0 visits found. Time elapsed: %fs
-Skipping invalidated archive 14, segment '' is not in --force-idsegments: [idinvalidation = 14, idsite = 1, period = week(2019-12-02 - 2019-12-08), name = done]
-Skipping invalidated archive 48, segment 'actions>=4' is not in --force-idsegments: [idinvalidation = 48, idsite = 1, period = week(2019-12-02 - 2019-12-08), name = done49a9440bd6dba4b8850035e09d043c67]
-No usable archive exists (ts_archived of existing = , now = 2020-02-03 04:05:06).
-Processing invalidation: [idinvalidation = 18, idsite = 1, period = week(2019-12-02 - 2019-12-08), name = donee0512c03f7c20af6ef96a8d792c6bb9f].
-Found archive with intersecting period with others in concurrent batch, skipping until next batch: [idinvalidation = 15, idsite = 1, period = month(2019-12-01 - 2019-12-31), name = done]
-Found archive with intersecting period with others in concurrent batch, skipping until next batch: [idinvalidation = 11, idsite = 1, period = month(2019-12-01 - 2019-12-31), name = done]
-Found archive with intersecting period with others in concurrent batch, skipping until next batch: [idinvalidation = 7, idsite = 1, period = month(2019-12-01 - 2019-12-31), name = done]
-Found archive with intersecting period with others in concurrent batch, skipping until next batch: [idinvalidation = 3, idsite = 1, period = month(2019-12-01 - 2019-12-31), name = done]
-Found archive with intersecting period with others in concurrent batch, skipping until next batch: [idinvalidation = 49, idsite = 1, period = month(2019-12-01 - 2019-12-31), name = done49a9440bd6dba4b8850035e09d043c67]
-Found archive with intersecting period with others in concurrent batch, skipping until next batch: [idinvalidation = 19, idsite = 1, period = month(2019-12-01 - 2019-12-31), name = donee0512c03f7c20af6ef96a8d792c6bb9f]
-Found archive with intersecting period with others in concurrent batch, skipping until next batch: [idinvalidation = 16, idsite = 1, period = year(2019-01-01 - 2019-12-31), name = done]
-Found archive with intersecting period with others in concurrent batch, skipping until next batch: [idinvalidation = 12, idsite = 1, period = year(2019-01-01 - 2019-12-31), name = done]
-Found archive with intersecting period with others in concurrent batch, skipping until next batch: [idinvalidation = 8, idsite = 1, period = year(2019-01-01 - 2019-12-31), name = done]
-Found archive with intersecting period with others in concurrent batch, skipping until next batch: [idinvalidation = 4, idsite = 1, period = year(2019-01-01 - 2019-12-31), name = done]
-Found archive with intersecting period with others in concurrent batch, skipping until next batch: [idinvalidation = 62, idsite = 1, period = year(2019-01-01 - 2019-12-31), name = done49a9440bd6dba4b8850035e09d043c67]
-Found archive with intersecting period with others in concurrent batch, skipping until next batch: [idinvalidation = 32, idsite = 1, period = year(2019-01-01 - 2019-12-31), name = donee0512c03f7c20af6ef96a8d792c6bb9f]
+Processing invalidation: [idinvalidation = 50, idsite = 1, period = week(2019-12-02 - 2019-12-08), name = donee0512c03f7c20af6ef96a8d792c6bb9f].
 No next invalidated archive.
 Starting archiving for ?module=API&method=CoreAdminHome.archiveReports&idSite=1&period=week&date=2019-12-02&format=json&segment=actions%3E%3D2&trigger=archivephp
 Archived website id 1, period = week, date = 2019-12-02, segment = 'actions%3E%3D2', 0 visits found. Time elapsed: %fs
-Skipping invalidated archive 15, segment '' is not in --force-idsegments: [idinvalidation = 15, idsite = 1, period = month(2019-12-01 - 2019-12-31), name = done]
-Skipping invalidated archive 11, segment '' is not in --force-idsegments: [idinvalidation = 11, idsite = 1, period = month(2019-12-01 - 2019-12-31), name = done]
-Skipping invalidated archive 7, segment '' is not in --force-idsegments: [idinvalidation = 7, idsite = 1, period = month(2019-12-01 - 2019-12-31), name = done]
-Skipping invalidated archive 3, segment '' is not in --force-idsegments: [idinvalidation = 3, idsite = 1, period = month(2019-12-01 - 2019-12-31), name = done]
-Skipping invalidated archive 49, segment 'actions>=4' is not in --force-idsegments: [idinvalidation = 49, idsite = 1, period = month(2019-12-01 - 2019-12-31), name = done49a9440bd6dba4b8850035e09d043c67]
-No usable archive exists (ts_archived of existing = , now = 2020-02-03 04:05:06).
-Processing invalidation: [idinvalidation = 19, idsite = 1, period = month(2019-12-01 - 2019-12-31), name = donee0512c03f7c20af6ef96a8d792c6bb9f].
-Found archive with intersecting period with others in concurrent batch, skipping until next batch: [idinvalidation = 16, idsite = 1, period = year(2019-01-01 - 2019-12-31), name = done]
-Found archive with intersecting period with others in concurrent batch, skipping until next batch: [idinvalidation = 12, idsite = 1, period = year(2019-01-01 - 2019-12-31), name = done]
-Found archive with intersecting period with others in concurrent batch, skipping until next batch: [idinvalidation = 8, idsite = 1, period = year(2019-01-01 - 2019-12-31), name = done]
-Found archive with intersecting period with others in concurrent batch, skipping until next batch: [idinvalidation = 4, idsite = 1, period = year(2019-01-01 - 2019-12-31), name = done]
-Found archive with intersecting period with others in concurrent batch, skipping until next batch: [idinvalidation = 62, idsite = 1, period = year(2019-01-01 - 2019-12-31), name = done49a9440bd6dba4b8850035e09d043c67]
-Found archive with intersecting period with others in concurrent batch, skipping until next batch: [idinvalidation = 32, idsite = 1, period = year(2019-01-01 - 2019-12-31), name = donee0512c03f7c20af6ef96a8d792c6bb9f]
+Processing invalidation: [idinvalidation = 51, idsite = 1, period = month(2019-12-01 - 2019-12-31), name = donee0512c03f7c20af6ef96a8d792c6bb9f].
 No next invalidated archive.
 Starting archiving for ?module=API&method=CoreAdminHome.archiveReports&idSite=1&period=month&date=2019-12-01&format=json&segment=actions%3E%3D2&trigger=archivephp
 Archived website id 1, period = month, date = 2019-12-01, segment = 'actions%3E%3D2', 0 visits found. Time elapsed: %fs
-Skipping invalidated archive 16, segment '' is not in --force-idsegments: [idinvalidation = 16, idsite = 1, period = year(2019-01-01 - 2019-12-31), name = done]
-Skipping invalidated archive 12, segment '' is not in --force-idsegments: [idinvalidation = 12, idsite = 1, period = year(2019-01-01 - 2019-12-31), name = done]
-Skipping invalidated archive 8, segment '' is not in --force-idsegments: [idinvalidation = 8, idsite = 1, period = year(2019-01-01 - 2019-12-31), name = done]
-Skipping invalidated archive 4, segment '' is not in --force-idsegments: [idinvalidation = 4, idsite = 1, period = year(2019-01-01 - 2019-12-31), name = done]
-Skipping invalidated archive 62, segment 'actions>=4' is not in --force-idsegments: [idinvalidation = 62, idsite = 1, period = year(2019-01-01 - 2019-12-31), name = done49a9440bd6dba4b8850035e09d043c67]
-No usable archive exists (ts_archived of existing = , now = 2020-02-03 04:05:06).
-Processing invalidation: [idinvalidation = 32, idsite = 1, period = year(2019-01-01 - 2019-12-31), name = donee0512c03f7c20af6ef96a8d792c6bb9f].
+Processing invalidation: [idinvalidation = 64, idsite = 1, period = year(2019-01-01 - 2019-12-31), name = donee0512c03f7c20af6ef96a8d792c6bb9f].
 No next invalidated archive.
 Starting archiving for ?module=API&method=CoreAdminHome.archiveReports&idSite=1&period=year&date=2019-01-01&format=json&segment=actions%3E%3D2&trigger=archivephp
 Archived website id 1, period = year, date = 2019-01-01, segment = 'actions%3E%3D2', 0 visits found. Time elapsed: %fs
 No next invalidated archive.
-Finished archiving for site 1, 5 API requests, Time elapsed: %fs [1 / 1 done]
+Finished archiving for site 1, 8 API requests, Time elapsed: %fs [1 / 1 done]
 No more sites left to archive, stopping.
 Done archiving!
 ---------------------------
 SUMMARY
-Processed 5 archives.
-Total API requests: 5
-done: 5 req, %d ms, no error
+Processed 8 archives.
+Total API requests: 8
+done: 8 req, %d ms, no error
 Time elapsed: %fs
 LOG;
 
-        $this->assertStringMatchesFormat($expected, $logger->output);
+        // remove a bunch of debug lines since we can't have a sprintf format that long
+        $output = $this->cleanOutput($logger->output);
+
+        $this->assertStringMatchesFormat($expected, $output);
+    }
+
+    private function cleanOutput($output)
+    {
+        $output = explode("\n", $output);
+        $output = array_filter($output, function ($l) { return strpos($l, 'Skipping invalidated archive') === false; });
+        $output = array_filter($output, function ($l) { return strpos($l, 'Found archive with intersecting period') === false; });
+        $output = array_filter($output, function ($l) { return strpos($l, 'Found duplicate invalidated archive') === false; });
+        $output = array_filter($output, function ($l) { return strpos($l, 'No usable archive exists') === false; });
+        $output = implode("\n", $output);
+        return $output;
     }
 
     public function test_shouldNotStopProcessingWhenOneSiteIsInvalid()