diff options
author | diosmosis <diosmosis@users.noreply.github.com> | 2020-12-17 04:43:12 +0300 |
---|---|---|
committer | GitHub <noreply@github.com> | 2020-12-17 04:43:12 +0300 |
commit | 4a241a72c52ebb54df3f9fa92e6fb392d22c4cd8 (patch) | |
tree | 9cbf4c57989e304a514996fa91248e824680fd14 /tests | |
parent | b76eb501a373a08447d1ddb086833897d4c869f3 (diff) |
Fix intersecting period logic so proper archives will archive in parallel. (#16946)
Diffstat (limited to 'tests')
-rw-r--r-- | tests/PHPUnit/Integration/CronArchiveTest.php | 34 | ||||
-rw-r--r-- | tests/PHPUnit/Unit/CronArchive/QueueConsumerTest.php | 95 |
2 files changed, 104 insertions, 25 deletions
diff --git a/tests/PHPUnit/Integration/CronArchiveTest.php b/tests/PHPUnit/Integration/CronArchiveTest.php index 00a8b4867e..4693092589 100644 --- a/tests/PHPUnit/Integration/CronArchiveTest.php +++ b/tests/PHPUnit/Integration/CronArchiveTest.php @@ -697,48 +697,31 @@ Checking for queued invalidations... Segment "actions>=2" was created or changed recently and will therefore archive today (for site ID = 1) Segment "actions>=4" was created or changed recently and will therefore archive today (for site ID = 1) Done invalidating -Found invalidated archive we can skip (no visits): [idinvalidation = 53, idsite = 1, period = day(2020-02-03 - 2020-02-03), name = donee0512c03f7c20af6ef96a8d792c6bb9f] -Found invalidated archive we can skip (no visits): [idinvalidation = 54, idsite = 1, period = week(2020-02-03 - 2020-02-09), name = donee0512c03f7c20af6ef96a8d792c6bb9f] -Found invalidated archive we can skip (no visits): [idinvalidation = 52, idsite = 1, period = day(2020-02-02 - 2020-02-02), name = donee0512c03f7c20af6ef96a8d792c6bb9f] -Found invalidated archive we can skip (no visits): [idinvalidation = 50, idsite = 1, period = day(2020-02-01 - 2020-02-01), name = donee0512c03f7c20af6ef96a8d792c6bb9f] -Found invalidated archive we can skip (no visits): [idinvalidation = 51, idsite = 1, period = month(2020-02-01 - 2020-02-29), name = donee0512c03f7c20af6ef96a8d792c6bb9f] -Found invalidated archive we can skip (no visits): [idinvalidation = 46, idsite = 1, period = week(2020-01-27 - 2020-02-02), name = donee0512c03f7c20af6ef96a8d792c6bb9f] -Found invalidated archive we can skip (no visits): [idinvalidation = 43, idsite = 1, period = day(2020-01-01 - 2020-01-01), name = donee0512c03f7c20af6ef96a8d792c6bb9f] -Found invalidated archive we can skip (no visits): [idinvalidation = 44, idsite = 1, period = month(2020-01-01 - 2020-01-31), name = donee0512c03f7c20af6ef96a8d792c6bb9f] -Found invalidated archive we can skip (no visits): [idinvalidation = 45, idsite = 1, period = year(2020-01-01 - 2020-12-31), name = donee0512c03f7c20af6ef96a8d792c6bb9f] -Found invalidated archive we can skip (no visits): [idinvalidation = 35, idsite = 1, period = day(2019-12-31 - 2019-12-31), name = donee0512c03f7c20af6ef96a8d792c6bb9f] -Found invalidated archive we can skip (no visits): [idinvalidation = 33, idsite = 1, period = day(2019-12-30 - 2019-12-30), name = donee0512c03f7c20af6ef96a8d792c6bb9f] -Found invalidated archive we can skip (no visits): [idinvalidation = 34, idsite = 1, period = week(2019-12-30 - 2020-01-05), name = donee0512c03f7c20af6ef96a8d792c6bb9f] -Found invalidated archive we can skip (no visits): [idinvalidation = 31, idsite = 1, period = day(2019-12-23 - 2019-12-23), name = donee0512c03f7c20af6ef96a8d792c6bb9f] -Found invalidated archive we can skip (no visits): [idinvalidation = 32, idsite = 1, period = week(2019-12-23 - 2019-12-29), name = donee0512c03f7c20af6ef96a8d792c6bb9f] -Found invalidated archive we can skip (no visits): [idinvalidation = 29, idsite = 1, period = day(2019-12-16 - 2019-12-16), name = donee0512c03f7c20af6ef96a8d792c6bb9f] -Found invalidated archive we can skip (no visits): [idinvalidation = 30, idsite = 1, period = week(2019-12-16 - 2019-12-22), name = donee0512c03f7c20af6ef96a8d792c6bb9f] -Processing invalidation: [idinvalidation = 5, idsite = 1, period = day(2019-12-12 - 2019-12-12), name = donee0512c03f7c20af6ef96a8d792c6bb9f]. -Processing invalidation: [idinvalidation = 14, idsite = 1, period = day(2019-12-11 - 2019-12-11), name = donee0512c03f7c20af6ef96a8d792c6bb9f]. -Processing invalidation: [idinvalidation = 17, idsite = 1, period = day(2019-12-10 - 2019-12-10), name = donee0512c03f7c20af6ef96a8d792c6bb9f]. +Processing invalidation: [idinvalidation = 5, idsite = 1, period = day(2019-12-12 - 2019-12-12), name = donee0512c03f7c20af6ef96a8d792c6bb9f, segment = actions>=2]. +Processing invalidation: [idinvalidation = 14, idsite = 1, period = day(2019-12-11 - 2019-12-11), name = donee0512c03f7c20af6ef96a8d792c6bb9f, segment = actions>=2]. +Processing invalidation: [idinvalidation = 17, idsite = 1, period = day(2019-12-10 - 2019-12-10), name = donee0512c03f7c20af6ef96a8d792c6bb9f, segment = actions>=2]. Starting archiving for ?module=API&method=CoreAdminHome.archiveReports&idSite=1&period=day&date=2019-12-12&format=json&segment=actions%3E%3D2&trigger=archivephp Starting archiving for ?module=API&method=CoreAdminHome.archiveReports&idSite=1&period=day&date=2019-12-11&format=json&segment=actions%3E%3D2&trigger=archivephp Starting archiving for ?module=API&method=CoreAdminHome.archiveReports&idSite=1&period=day&date=2019-12-10&format=json&segment=actions%3E%3D2&trigger=archivephp Archived website id 1, period = day, date = 2019-12-12, segment = 'actions%3E%3D2', 0 visits found. Time elapsed: %fs Archived website id 1, period = day, date = 2019-12-11, segment = 'actions%3E%3D2', 0 visits found. Time elapsed: %fs Archived website id 1, period = day, date = 2019-12-10, segment = 'actions%3E%3D2', 0 visits found. Time elapsed: %fs -Found invalidated archive we can skip (no visits): [idinvalidation = 28, idsite = 1, period = day(2019-12-09 - 2019-12-09), name = donee0512c03f7c20af6ef96a8d792c6bb9f] -Processing invalidation: [idinvalidation = 6, idsite = 1, period = week(2019-12-09 - 2019-12-15), name = donee0512c03f7c20af6ef96a8d792c6bb9f]. -Processing invalidation: [idinvalidation = 22, idsite = 1, period = day(2019-12-02 - 2019-12-02), name = donee0512c03f7c20af6ef96a8d792c6bb9f]. +Processing invalidation: [idinvalidation = 6, idsite = 1, period = week(2019-12-09 - 2019-12-15), name = donee0512c03f7c20af6ef96a8d792c6bb9f, segment = actions>=2]. +Processing invalidation: [idinvalidation = 22, idsite = 1, period = day(2019-12-02 - 2019-12-02), name = donee0512c03f7c20af6ef96a8d792c6bb9f, segment = actions>=2]. No next invalidated archive. Starting archiving for ?module=API&method=CoreAdminHome.archiveReports&idSite=1&period=week&date=2019-12-09&format=json&segment=actions%3E%3D2&trigger=archivephp Starting archiving for ?module=API&method=CoreAdminHome.archiveReports&idSite=1&period=day&date=2019-12-02&format=json&segment=actions%3E%3D2&trigger=archivephp Archived website id 1, period = week, date = 2019-12-09, segment = 'actions%3E%3D2', 0 visits found. Time elapsed: %fs Archived website id 1, period = day, date = 2019-12-02, segment = 'actions%3E%3D2', 0 visits found. Time elapsed: %fs -Processing invalidation: [idinvalidation = 23, idsite = 1, period = week(2019-12-02 - 2019-12-08), name = donee0512c03f7c20af6ef96a8d792c6bb9f]. +Processing invalidation: [idinvalidation = 23, idsite = 1, period = week(2019-12-02 - 2019-12-08), name = donee0512c03f7c20af6ef96a8d792c6bb9f, segment = actions>=2]. No next invalidated archive. Starting archiving for ?module=API&method=CoreAdminHome.archiveReports&idSite=1&period=week&date=2019-12-02&format=json&segment=actions%3E%3D2&trigger=archivephp Archived website id 1, period = week, date = 2019-12-02, segment = 'actions%3E%3D2', 0 visits found. Time elapsed: %fs -Processing invalidation: [idinvalidation = 7, idsite = 1, period = month(2019-12-01 - 2019-12-31), name = donee0512c03f7c20af6ef96a8d792c6bb9f]. +Processing invalidation: [idinvalidation = 7, idsite = 1, period = month(2019-12-01 - 2019-12-31), name = donee0512c03f7c20af6ef96a8d792c6bb9f, segment = actions>=2]. No next invalidated archive. Starting archiving for ?module=API&method=CoreAdminHome.archiveReports&idSite=1&period=month&date=2019-12-01&format=json&segment=actions%3E%3D2&trigger=archivephp Archived website id 1, period = month, date = 2019-12-01, segment = 'actions%3E%3D2', 0 visits found. Time elapsed: %fs -Processing invalidation: [idinvalidation = 8, idsite = 1, period = year(2019-01-01 - 2019-12-31), name = donee0512c03f7c20af6ef96a8d792c6bb9f]. +Processing invalidation: [idinvalidation = 8, idsite = 1, period = year(2019-01-01 - 2019-12-31), name = donee0512c03f7c20af6ef96a8d792c6bb9f, segment = actions>=2]. No next invalidated archive. Starting archiving for ?module=API&method=CoreAdminHome.archiveReports&idSite=1&period=year&date=2019-01-01&format=json&segment=actions%3E%3D2&trigger=archivephp Archived website id 1, period = year, date = 2019-01-01, segment = 'actions%3E%3D2', 0 visits found. Time elapsed: %fs @@ -767,6 +750,7 @@ LOG; $output = array_filter($output, function ($l) { return strpos($l, 'Found archive with intersecting period') === false; }); $output = array_filter($output, function ($l) { return strpos($l, 'Found duplicate invalidated archive') === false; }); $output = array_filter($output, function ($l) { return strpos($l, 'No usable archive exists') === false; }); + $output = array_filter($output, function ($l) { return strpos($l, 'Found invalidated archive we can skip (no visits)') === false; }); $output = implode("\n", $output); return $output; } diff --git a/tests/PHPUnit/Unit/CronArchive/QueueConsumerTest.php b/tests/PHPUnit/Unit/CronArchive/QueueConsumerTest.php new file mode 100644 index 0000000000..f8353ae554 --- /dev/null +++ b/tests/PHPUnit/Unit/CronArchive/QueueConsumerTest.php @@ -0,0 +1,95 @@ +<?php +/** + * Matomo - free/libre analytics platform + * + * @link https://matomo.org + * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later + * + */ + +namespace PHPUnit\Unit\CronArchive; + + +use PHPUnit\Framework\TestCase; +use Piwik\CronArchive\QueueConsumer; +use Piwik\Period\Factory; +use Piwik\Piwik; + +class QueueConsumerTest extends TestCase +{ + /** + * @dataProvider getTestDataForHasIntersectingPeriod + */ + public function test_hasIntersectingPeriod($archivesToProcess, $invalidatedArchive, $expected) + { + $periods = array_flip(Piwik::$idPeriods); + foreach ($archivesToProcess as &$archive) { + $periodLabel = $periods[$archive['period']]; + $archive['periodObj'] = Factory::build($periodLabel, $archive['date1']); + } + + $periodLabel = $periods[$invalidatedArchive['period']]; + $invalidatedArchive['periodObj'] = Factory::build($periodLabel, $invalidatedArchive['date1']); + + $actual = QueueConsumer::hasIntersectingPeriod($archivesToProcess, $invalidatedArchive); + $this->assertEquals($expected, $actual); + } + + public function getTestDataForHasIntersectingPeriod() + { + return [ + // no intersecting periods + [ + [ + ['period' => 1, 'date1' => '2020-03-04', 'date2' => '2020-03-04'], + ['period' => 3, 'date1' => '2020-04-01', 'date2' => '2020-04-30'], + ['period' => 1, 'date1' => '2020-03-15', 'date2' => '2020-03-15', 'segment' => 'pageUrl==abc'], + ], + ['period' => 1, 'date1' => '2020-03-05', 'date2' => '2020-03-05'], + false, + ], + + // intersecting periods + [ + [ + ['period' => 1, 'date1' => '2020-03-04', 'date2' => '2020-03-04'], + ['period' => 3, 'date1' => '2020-04-01', 'date2' => '2020-04-30'], + ['period' => 1, 'date1' => '2020-03-15', 'date2' => '2020-03-15', 'segment' => 'pageUrl==abc'], + ], + ['period' => 2, 'date1' => '2020-03-02', 'date2' => '2020-03-08'], + true, + ], + + // all same period, different segments + [ + [ + ['period' => 1, 'date1' => '2020-03-15', 'date2' => '2020-03-15', 'segment' => 'pageUrl==def'], + ['period' => 1, 'date1' => '2020-03-15', 'date2' => '2020-03-15', 'segment' => 'pageUrl==ghi'], + ['period' => 1, 'date1' => '2020-03-15', 'date2' => '2020-03-15', 'segment' => 'pageUrl==abc'], + ], + ['period' => 1, 'date1' => '2020-03-15', 'date2' => '2020-03-15', 'segment' => 'pageUrl==lmn'], + false, + ], + + // all same period, all visits in one + [ + [ + ['period' => 1, 'date1' => '2020-03-15', 'date2' => '2020-03-15', 'segment' => ''], + ], + ['period' => 1, 'date1' => '2020-03-15', 'date2' => '2020-03-15', 'segment' => 'pageUrl==lmn'], + true, + ], + + // all same period, different segments, all visits in next + [ + [ + ['period' => 1, 'date1' => '2020-03-15', 'date2' => '2020-03-15', 'segment' => 'pageUrl==def'], + ['period' => 1, 'date1' => '2020-03-15', 'date2' => '2020-03-15', 'segment' => 'pageUrl==ghi'], + ['period' => 1, 'date1' => '2020-03-15', 'date2' => '2020-03-15', 'segment' => 'pageUrl==abc'], + ], + ['period' => 1, 'date1' => '2020-03-15', 'date2' => '2020-03-15'], + true, + ], + ]; + } +}
\ No newline at end of file |