From 490dc3a1a09e4263b4893cbbbf600ba505d9add4 Mon Sep 17 00:00:00 2001 From: dizzy Date: Thu, 4 Feb 2021 14:22:11 -0800 Subject: Refactor segment re-archiving in past behavior to be on demand (#17005) * Invalidate past archives on demand when adding/updating segments, rather than trying to check when running core:archive * start on rewriting test * rewrite SegmentArchivingTest and get to pass * get sites as superuser * add update to rearchive segments if they were created/update between last archive time and update time * remove unused parameter * fix build * fix tests * sanity check * fix bug, we should not forget archives to invalidate unless all related archives are being invalidated * fix tests and make fix more complete * fix test * update counts in test * fix test for last time hopefully * fix another test * remove debugging code --- .../CronArchive/SegmentArchivingTest.php | 308 ++++----------------- tests/PHPUnit/Integration/CronArchiveTest.php | 31 +-- .../DataAccess/ArchiveInvalidatorTest.php | 26 +- tests/PHPUnit/Integration/PluginTest.php | 6 +- tests/PHPUnit/System/ArchiveInvalidationTest.php | 5 +- ...ewDataShouldAppear__Actions.getPageUrls_day.xml | 173 ++++++++++++ ..._NewDataShouldAppear__VisitsSummary.get_day.xml | 20 ++ ...yWasNotInvalidated__Actions.getPageUrls_day.xml | 74 +++-- ...DayWasNotInvalidated__VisitsSummary.get_day.xml | 12 +- ...ataShouldNotAppear__Actions.getPageUrls_day.xml | 157 +++++++++++ ...wDataShouldNotAppear__VisitsSummary.get_day.xml | 20 ++ 11 files changed, 512 insertions(+), 320 deletions(-) create mode 100644 tests/PHPUnit/System/expected/test_Archive_InvalidationWebsite2_NewDataShouldAppear__Actions.getPageUrls_day.xml create mode 100644 tests/PHPUnit/System/expected/test_Archive_InvalidationWebsite2_NewDataShouldAppear__VisitsSummary.get_day.xml create mode 100644 tests/PHPUnit/System/expected/test_Archive_InvalidationWebsite2_NewDataShouldNotAppear__Actions.getPageUrls_day.xml create mode 100644 tests/PHPUnit/System/expected/test_Archive_InvalidationWebsite2_NewDataShouldNotAppear__VisitsSummary.get_day.xml (limited to 'tests') diff --git a/tests/PHPUnit/Integration/CronArchive/SegmentArchivingTest.php b/tests/PHPUnit/Integration/CronArchive/SegmentArchivingTest.php index 2eb1790980..919c063c37 100644 --- a/tests/PHPUnit/Integration/CronArchive/SegmentArchivingTest.php +++ b/tests/PHPUnit/Integration/CronArchive/SegmentArchivingTest.php @@ -22,294 +22,102 @@ use Piwik\Tests\Framework\TestCase\IntegrationTestCase; */ class SegmentArchivingTest extends IntegrationTestCase { - const TEST_NOW = '2015-03-01'; - - private $mockSegmentEntries; - - public function setUp(): void + protected static function beforeTableDataCached() { - parent::setUp(); - - Config::getInstance()->General['enabled_periods_API'] = 'day,week,month,year,range'; - - Site::setSites([ - 1 => [ - 'idsite' => 1, - 'ts_created' => '2013-03-03 00:00:00', - ], - ]); - - $this->mockSegmentEntries = array( - array( - 'ts_created' => '2014-01-01', - 'definition' => 'browserName==FF', - 'enable_only_idsite' => 1, - 'ts_last_edit' => '2014-05-05 00:22:33', - 'auto_archive' => 1, - ), - - array( - 'ts_created' => '2014-01-01', - 'definition' => 'countryCode==us', - 'enable_only_idsite' => 1, - 'ts_last_edit' => '2014-02-02 00:33:44', - 'auto_archive' => 1, - ), - - array( - 'ts_created' => '2012-01-01', - 'definition' => 'countryCode==us', - 'enable_only_idsite' => 1, - 'ts_last_edit' => '2014-02-03', - 'auto_archive' => 1, - ), - - array( - 'ts_created' => '2014-01-01', - 'definition' => 'countryCode==ca', - 'enable_only_idsite' => 2, - 'ts_last_edit' => '2013-01-01', - 'auto_archive' => 1, - ), - - array( - 'ts_created' => '2012-01-01', - 'definition' => 'countryCode==ca', - 'enable_only_idsite' => 2, - 'ts_last_edit' => '2011-01-01', - 'auto_archive' => 1, - ), - - array( - 'ts_created' => '2012-01-01', - 'definition' => 'countryCode==br', - 'enable_only_idsite' => 2, - 'ts_last_edit' => '2011-01-01', - 'auto_archive' => 1, - ), - - array( - 'ts_created' => '2011-01-01', - 'definition' => 'countryCode==ca', - 'enable_only_idsite' => 0, - 'ts_last_edit' => null, - 'auto_archive' => 1, - ), - - array( - 'ts_created' => '2015-03-01', - 'definition' => 'pageUrl==a', - 'enable_only_idsite' => 1, - 'ts_last_edit' => '2014-01-01', - 'auto_archive' => 1, - ), - - array( - 'ts_created' => '2015-02-01', - 'definition' => 'pageUrl==b', - 'enable_only_idsite' => 1, - 'ts_last_edit' => null, - 'auto_archive' => 1, - ), - ); - - Date::$now = strtotime('2020-01-30 00:00:00'); + parent::beforeTableDataCached(); + Fixture::createWebsite('2020-01-04 12:00:00'); } /** - * @dataProvider getTestDataForGetSegmentArchivesToInvalidateForNewSegments + * @dataProvider getTestDataForGetReArchiveSegmentStartDate */ - public function test_getSegmentArchivesToInvalidateForNewSegments_returnsAllSegments_IfInvalidationHasNotRun($processFrom, $idSite, $expected) + public function test_getReArchiveSegmentStartDate($processNewSegmentFrom, $segmentInfo, $expected) { - $archiving = $this->createUrlProviderToTest($processFrom); - $segments = $archiving->getSegmentArchivesToInvalidateForNewSegments($idSite); - $this->getStringDates($segments); - $this->assertEquals($expected, $segments); + Date::$now = strtotime('2020-10-12 13:45:00'); + + Config::getInstance()->General['process_new_segments_from'] = $processNewSegmentFrom; + + $segmentArchiving = new SegmentArchiving(); + $result = $segmentArchiving->getReArchiveSegmentStartDate($segmentInfo); + if (!empty($result)) { + $result = $result->toString(); + } + $this->assertEquals($expected, $result); } - public function getTestDataForGetSegmentArchivesToInvalidateForNewSegments() + public function getTestDataForGetReArchiveSegmentStartDate() { return [ + // no segment creation time [ - 'beginning_of_time', - 1, - [ - [ - 'date' => '2013-03-03 00:00:00', - 'segment' => 'browserName==FF', - ], - [ - 'date' => '2013-03-03 00:00:00', - 'segment' => 'countryCode==us', - ], - [ - 'date' => '2013-03-03 00:00:00', - 'segment' => 'countryCode==ca', - ], - [ - 'date' => '2013-03-03 00:00:00', - 'segment' => 'pageUrl==a', - ], - [ - 'date' => '2013-03-03 00:00:00', - 'segment' => 'pageUrl==b', - ], - ], + SegmentArchiving::CREATION_TIME, + [], + null, ], + // creation time [ - 'segment_creation_time', - 1, - [ - [ - 'date' => '2014-01-01 00:00:00', - 'segment' => 'browserName==FF', - ], - [ - 'date' => '2014-01-01 00:00:00', - 'segment' => 'countryCode==us', - ], - [ - 'date' => '2011-01-01 00:00:00', - 'segment' => 'countryCode==ca', - ], - [ - 'date' => '2015-03-01 00:00:00', - 'segment' => 'pageUrl==a', - ], - [ - 'date' => '2015-02-01 00:00:00', - 'segment' => 'pageUrl==b', - ], - ], + SegmentArchiving::CREATION_TIME, + ['ts_created' => '2020-04-12 03:34:55'], + '2020-04-12', ], + // last edit time [ - 'segment_last_edit_time', - 1, - [ - [ - 'date' => '2014-05-05 00:22:33', - 'segment' => 'browserName==FF', - ], - [ - 'date' => '2014-02-02 00:33:44', - 'segment' => 'countryCode==us', - ], - [ - 'date' => '2011-01-01 00:00:00', - 'segment' => 'countryCode==ca', - ], - [ - 'date' => '2015-03-01 00:00:00', - 'segment' => 'pageUrl==a', - ], - [ - 'date' => '2015-02-01 00:00:00', - 'segment' => 'pageUrl==b', - ], - ], + SegmentArchiving::LAST_EDIT_TIME, + ['ts_created' => '2020-02-02 03:00:00', 'ts_last_edit' => '2020-04-13 05:15:15'], + '2020-04-13', ], + // last edit time, no edit time in segment [ - 'segment_last_edit_time', - 2, - [ - [ - 'date' => '2014-01-01 00:00:00', - 'segment' => 'countryCode==ca', - ], - [ - 'date' => '2012-01-01 00:00:00', - 'segment' => 'countryCode==br', - ], - ], + SegmentArchiving::LAST_EDIT_TIME, + ['ts_created' => '2020-04-14 00:00:00'], + '2020-04-14', ], - ]; - } - - public function test_getSegmentArchivesToInvalidateForNewSegments_returnsSegmentsRecentlyCreated_IfInvalidationHasRun() - { - Option::set(CronArchive::CRON_INVALIDATION_TIME_OPTION_NAME, strtotime('2013-12-30 00:00:00')); - $archiving = $this->createUrlProviderToTest('beginning_of_time'); - $segments = $archiving->getSegmentArchivesToInvalidateForNewSegments(1); - $this->getStringDates($segments); - - $expected = [ - [ - 'segment' => 'browserName==FF', - 'date' => '2013-03-03 00:00:00', - ], + // lastN [ - 'segment' => 'countryCode==us', - 'date' => '2013-03-03 00:00:00', + 'last30', + ['ts_created' => '2020-06-12'], + '2020-05-13', ], + + // beginning of time [ - 'segment' => 'pageUrl==a', - 'date' => '2013-03-03 00:00:00', + SegmentArchiving::BEGINNING_OF_TIME, + ['ts_created' => '2020-06-12'], + '2013-01-01', ], + + // beginning of time (unreadable value) [ - 'segment' => 'pageUrl==b', - 'date' => '2013-03-03 00:00:00', + 'aslkdfjsdlkjf', + ['ts_created' => '2020-06-12'], + '2013-01-01', ], ]; - $this->assertEquals($expected, $segments); } - public function test_getSegmentArchivesToInvalidateForNewSegments_returnsNoSegments_IfInvalidationHasRunAndAllSegmentsCreatedBefore() + public function test_getReArchiveSegmentStartDate_whenSiteCreationDateIsLater() { - Option::set(CronArchive::CRON_INVALIDATION_TIME_OPTION_NAME, strtotime('2019-12-30 00:00:00')); - - $archiving = $this->createUrlProviderToTest('beginning_of_time'); - $segments = $archiving->getSegmentArchivesToInvalidateForNewSegments(1); - $this->getStringDates($segments); - - $expected = []; - $this->assertEquals($expected, $segments); - } - - public function test_getSegmentArchivesToInvalidateForNewSegments_usesLastArchiveFinishTimeIfInvalidationTimeMissing() - { - Option::set(CronArchive::OPTION_ARCHIVING_FINISHED_TS, strtotime('2013-12-30 00:00:00')); - - $archiving = $this->createUrlProviderToTest('beginning_of_time'); - $segments = $archiving->getSegmentArchivesToInvalidateForNewSegments(1); - $this->getStringDates($segments); - - $expected = [ - [ - 'segment' => 'browserName==FF', - 'date' => '2013-03-03 00:00:00', - ], - [ - 'segment' => 'countryCode==us', - 'date' => '2013-03-03 00:00:00', - ], - [ - 'segment' => 'pageUrl==a', - 'date' => '2013-03-03 00:00:00', - ], - [ - 'segment' => 'pageUrl==b', - 'date' => '2013-03-03 00:00:00', - ], - ]; - $this->assertEquals($expected, $segments); + $segmentInfo = ['ts_created' => '2019-05-03 00:00:00', 'enable_only_idsite' => 1]; + $this->test_getReArchiveSegmentStartDate(SegmentArchiving::BEGINNING_OF_TIME, $segmentInfo, '2020-01-03'); } - private function createUrlProviderToTest($processNewSegmentsFrom, $mockData = null) + public function test_getReArchiveSegmentStartDate_whenEarliestVisitTimeIsLater() { - $mockSegmentEditorModel = $this->createPartialMock('Piwik\Plugins\SegmentEditor\Model', array('getAllSegmentsAndIgnoreVisibility')); - $mockSegmentEditorModel->expects($this->any())->method('getAllSegmentsAndIgnoreVisibility')->will($this->returnValue($mockData ?: $this->mockSegmentEntries)); + $t = Fixture::getTracker(1, '2020-02-05 03:00:00'); + $t->setUrl('http://abc.com'); + Fixture::checkResponse($t->doTrackPageView('abc')); - return new SegmentArchiving($processNewSegmentsFrom, $beginningOfTimeLastN = 7, $mockSegmentEditorModel, null, Date::factory(self::TEST_NOW)); + $segmentInfo = ['ts_created' => '2019-05-03 00:00:00', 'enable_only_idsite' => 1]; + $this->test_getReArchiveSegmentStartDate(SegmentArchiving::BEGINNING_OF_TIME, $segmentInfo, '2020-02-05'); } - private function getStringDates(array &$entries) + protected static function configureFixture($fixture) { - foreach ($entries as &$entry) { - $entry['date'] = $entry['date']->getDatetime(); - } + parent::configureFixture($fixture); + $fixture->createSuperUser = true; } } \ No newline at end of file diff --git a/tests/PHPUnit/Integration/CronArchiveTest.php b/tests/PHPUnit/Integration/CronArchiveTest.php index 2fc991e7b4..06c93187d2 100644 --- a/tests/PHPUnit/Integration/CronArchiveTest.php +++ b/tests/PHPUnit/Integration/CronArchiveTest.php @@ -544,7 +544,7 @@ class CronArchiveTest extends IntegrationTestCase $api = API::getInstance(); - $cronarchive = new TestCronArchive(Fixture::getRootUrl() . 'tests/PHPUnit/proxy/index.php'); + $cronarchive = new TestCronArchive(); $cronarchive->init(); $cronarchive->setApiToInvalidateArchivedReport($api); $cronarchive->invalidateArchivedReportsForSitesThatNeedToBeArchivedAgain(1); @@ -588,7 +588,7 @@ class CronArchiveTest extends IntegrationTestCase $allSegments = $segments->getSegmentsToAutoArchive(1); - $cronarchive = new TestCronArchive(Fixture::getRootUrl() . 'tests/PHPUnit/proxy/index.php'); + $cronarchive = new TestCronArchive(); $this->assertTrue($cronarchive->wasSegmentChangedRecently('actions>=1', $allSegments)); // created 30 hours ago... @@ -615,7 +615,7 @@ class CronArchiveTest extends IntegrationTestCase $logger = new FakeLogger(); - $archiver = new CronArchive(null, $logger); + $archiver = new CronArchive($logger); $archiver->init(); $archiveFilter = new CronArchive\ArchiveFilter(); $archiveFilter->setSkipSegmentsForToday(true); @@ -626,7 +626,6 @@ class CronArchiveTest extends IntegrationTestCase $archiver->run(); self::assertStringContainsString('Will skip segments archiving for today unless they were created recently', $logger->output); - self::assertStringContainsString('Segment "actions>=1" was created or changed recently and will therefore archive today', $logger->output); self::assertStringNotContainsString('Segment "actions>=2" was created recently', $logger->output); } @@ -660,7 +659,7 @@ class CronArchiveTest extends IntegrationTestCase $logger = new FakeLogger(); - $archiver = new CronArchive(null, $logger); + $archiver = new CronArchive($logger); $archiveFilter = new CronArchive\ArchiveFilter(); $archiveFilter->setSegmentsToForce(['actions>=2;browserCode=FF', 'actions>=2']); @@ -695,34 +694,32 @@ Checking for queued invalidations... Will invalidate archived reports for 2019-12-02 for following websites ids: 1 Today archive can be skipped due to no visits for idSite = 1, skipping invalidation... Yesterday archive can be skipped due to no visits for idSite = 1, skipping invalidation... - Segment "actions>=2" was created or changed recently and will therefore archive today (for site ID = 1) - Segment "actions>=4" was created or changed recently and will therefore archive today (for site ID = 1) Done invalidating -Processing invalidation: [idinvalidation = 9, idsite = 1, period = day(2019-12-12 - 2019-12-12), name = donee0512c03f7c20af6ef96a8d792c6bb9f, segment = actions>=2]. -Processing invalidation: [idinvalidation = 15, idsite = 1, period = day(2019-12-11 - 2019-12-11), name = donee0512c03f7c20af6ef96a8d792c6bb9f, segment = actions>=2]. -Processing invalidation: [idinvalidation = 18, idsite = 1, period = day(2019-12-10 - 2019-12-10), name = donee0512c03f7c20af6ef96a8d792c6bb9f, segment = actions>=2]. +Processing invalidation: [idinvalidation = 269, idsite = 1, period = day(2019-12-12 - 2019-12-12), name = donee0512c03f7c20af6ef96a8d792c6bb9f, segment = actions>=2]. +Processing invalidation: [idinvalidation = 268, idsite = 1, period = day(2019-12-11 - 2019-12-11), name = donee0512c03f7c20af6ef96a8d792c6bb9f, segment = actions>=2]. +Processing invalidation: [idinvalidation = 267, idsite = 1, period = day(2019-12-10 - 2019-12-10), name = donee0512c03f7c20af6ef96a8d792c6bb9f, segment = actions>=2]. Starting archiving for ?module=API&method=CoreAdminHome.archiveReports&idSite=1&period=day&date=2019-12-12&format=json&segment=actions%3E%3D2&trigger=archivephp Starting archiving for ?module=API&method=CoreAdminHome.archiveReports&idSite=1&period=day&date=2019-12-11&format=json&segment=actions%3E%3D2&trigger=archivephp Starting archiving for ?module=API&method=CoreAdminHome.archiveReports&idSite=1&period=day&date=2019-12-10&format=json&segment=actions%3E%3D2&trigger=archivephp Archived website id 1, period = day, date = 2019-12-12, segment = 'actions>=2', 0 visits found. Time elapsed: %fs Archived website id 1, period = day, date = 2019-12-11, segment = 'actions>=2', 0 visits found. Time elapsed: %fs Archived website id 1, period = day, date = 2019-12-10, segment = 'actions>=2', 0 visits found. Time elapsed: %fs -Processing invalidation: [idinvalidation = 10, idsite = 1, period = week(2019-12-09 - 2019-12-15), name = donee0512c03f7c20af6ef96a8d792c6bb9f, segment = actions>=2]. -Processing invalidation: [idinvalidation = 25, idsite = 1, period = day(2019-12-02 - 2019-12-02), name = donee0512c03f7c20af6ef96a8d792c6bb9f, segment = actions>=2]. +Processing invalidation: [idinvalidation = 266, idsite = 1, period = week(2019-12-09 - 2019-12-15), name = donee0512c03f7c20af6ef96a8d792c6bb9f, segment = actions>=2]. +Processing invalidation: [idinvalidation = 257, idsite = 1, period = day(2019-12-02 - 2019-12-02), name = donee0512c03f7c20af6ef96a8d792c6bb9f, segment = actions>=2]. No next invalidated archive. Starting archiving for ?module=API&method=CoreAdminHome.archiveReports&idSite=1&period=week&date=2019-12-09&format=json&segment=actions%3E%3D2&trigger=archivephp Starting archiving for ?module=API&method=CoreAdminHome.archiveReports&idSite=1&period=day&date=2019-12-02&format=json&segment=actions%3E%3D2&trigger=archivephp Archived website id 1, period = week, date = 2019-12-09, segment = 'actions>=2', 0 visits found. Time elapsed: %fs Archived website id 1, period = day, date = 2019-12-02, segment = 'actions>=2', 0 visits found. Time elapsed: %fs -Processing invalidation: [idinvalidation = 26, idsite = 1, period = week(2019-12-02 - 2019-12-08), name = donee0512c03f7c20af6ef96a8d792c6bb9f, segment = actions>=2]. +Processing invalidation: [idinvalidation = 258, idsite = 1, period = week(2019-12-02 - 2019-12-08), name = donee0512c03f7c20af6ef96a8d792c6bb9f, segment = actions>=2]. No next invalidated archive. Starting archiving for ?module=API&method=CoreAdminHome.archiveReports&idSite=1&period=week&date=2019-12-02&format=json&segment=actions%3E%3D2&trigger=archivephp Archived website id 1, period = week, date = 2019-12-02, segment = 'actions>=2', 0 visits found. Time elapsed: %fs -Processing invalidation: [idinvalidation = 11, idsite = 1, period = month(2019-12-01 - 2019-12-31), name = donee0512c03f7c20af6ef96a8d792c6bb9f, segment = actions>=2]. +Processing invalidation: [idinvalidation = 256, idsite = 1, period = month(2019-12-01 - 2019-12-31), name = donee0512c03f7c20af6ef96a8d792c6bb9f, segment = actions>=2]. No next invalidated archive. Starting archiving for ?module=API&method=CoreAdminHome.archiveReports&idSite=1&period=month&date=2019-12-01&format=json&segment=actions%3E%3D2&trigger=archivephp Archived website id 1, period = month, date = 2019-12-01, segment = 'actions>=2', 0 visits found. Time elapsed: %fs -Processing invalidation: [idinvalidation = 12, idsite = 1, period = year(2019-01-01 - 2019-12-31), name = donee0512c03f7c20af6ef96a8d792c6bb9f, segment = actions>=2]. +Processing invalidation: [idinvalidation = 65, idsite = 1, period = year(2019-01-01 - 2019-12-31), name = donee0512c03f7c20af6ef96a8d792c6bb9f, segment = actions>=2]. No next invalidated archive. Starting archiving for ?module=API&method=CoreAdminHome.archiveReports&idSite=1&period=year&date=2019-01-01&format=json&segment=actions%3E%3D2&trigger=archivephp Archived website id 1, period = year, date = 2019-01-01, segment = 'actions>=2', 0 visits found. Time elapsed: %fs @@ -777,7 +774,7 @@ LOG; $sequence = new Sequence(ArchiveTableCreator::getNumericTable(Date::factory('2019-12-10'))); $sequence->create(); - $archiver = new CronArchive(null, $logger); + $archiver = new CronArchive($logger); $archiveFilter = new CronArchive\ArchiveFilter(); $archiver->setArchiveFilter($archiveFilter); @@ -867,7 +864,7 @@ LOG; $logger = new FakeLogger(); - $archiver = new CronArchive(null, $logger); + $archiver = new CronArchive($logger); $archiver->shouldArchiveSpecifiedSites = array(99999, 1); $archiver->init(); $archiver->run(); diff --git a/tests/PHPUnit/Integration/DataAccess/ArchiveInvalidatorTest.php b/tests/PHPUnit/Integration/DataAccess/ArchiveInvalidatorTest.php index 8d1e75db00..f71d63f54d 100644 --- a/tests/PHPUnit/Integration/DataAccess/ArchiveInvalidatorTest.php +++ b/tests/PHPUnit/Integration/DataAccess/ArchiveInvalidatorTest.php @@ -97,8 +97,8 @@ class ArchiveInvalidatorTest extends IntegrationTestCase $items = $list->getAll(); $expected = [ - '{"idSites":[1],"pluginName":"ExamplePlugin","report":null,"startDate":null}', - '{"idSites":[1,4,5],"pluginName":"MyOtherPlugin","report":null,"startDate":null}', + '{"idSites":[1],"pluginName":"ExamplePlugin","report":null,"startDate":null,"segment":null}', + '{"idSites":[1,4,5],"pluginName":"MyOtherPlugin","report":null,"startDate":null,"segment":null}', ]; $this->assertEquals($expected, $items); @@ -115,7 +115,7 @@ class ArchiveInvalidatorTest extends IntegrationTestCase $items = $list->getAll(); $expected = [ - '{"idSites":[1,4,5],"pluginName":"MyOtherPlugin","report":null,"startDate":null}', + '{"idSites":[1,4,5],"pluginName":"MyOtherPlugin","report":null,"startDate":null,"segment":null}', ]; $this->assertEquals($expected, $items); @@ -149,8 +149,8 @@ class ArchiveInvalidatorTest extends IntegrationTestCase $items = $list->getAll(); $expected = [ - '{"idSites":[1,4,5],"pluginName":"ExamplePlugin","report":null,"startDate":null}', - '{"idSites":[1,4,5],"pluginName":"ExamplePlugin","report":"myOtherReport","startDate":null}', + '{"idSites":[1,4,5],"pluginName":"ExamplePlugin","report":null,"startDate":null,"segment":null}', + '{"idSites":[1,4,5],"pluginName":"ExamplePlugin","report":"myOtherReport","startDate":null,"segment":null}', ]; $this->assertEquals($expected, $items); @@ -404,7 +404,7 @@ class ArchiveInvalidatorTest extends IntegrationTestCase $this->assertSameReports($expected, $reports); } - public function test_markArchivesAsInvalidated_shouldForgetInvalidatedSitesAndDates() + public function test_markArchivesAsInvalidated_shouldForgetInvalidatedSitesAndDates_IfPeriodIsDay() { $this->rememberReportsForManySitesAndDates(); @@ -415,7 +415,7 @@ class ArchiveInvalidatorTest extends IntegrationTestCase Date::factory('2010-10-10'), ); - $this->invalidator->markArchivesAsInvalidated($idSites, $dates, 'week'); + $this->invalidator->markArchivesAsInvalidated($idSites, $dates, 'day'); $reports = $this->invalidator->getRememberedArchivedReportsThatShouldBeInvalidated(); $expected = array( @@ -1651,8 +1651,8 @@ class ArchiveInvalidatorTest extends IntegrationTestCase 'period' => '1', 'name' => 'done5f4f9bafeda3443c3c2d4b2ef4dffadc.VisitsSummary', 'report' => 'some.Report', - 'dates' => '2020-05-04,2020-05-04|2020-05-05,2020-05-05|2020-05-06,2020-05-06|2020-05-07,2020-05-07|2020-05-08,2020-05-08|2020-05-09,2020-05-09|2020-05-10,2020-05-10|2020-05-11,2020-05-11|2020-05-12,2020-05-12|2020-05-13,2020-05-13|2020-05-14,2020-05-14|2020-05-15,2020-05-15|2020-05-16,2020-05-16|2020-05-17,2020-05-17|2020-05-18,2020-05-18|2020-05-19,2020-05-19|2020-05-20,2020-05-20|2020-05-21,2020-05-21|2020-05-22,2020-05-22|2020-05-23,2020-05-23|2020-05-24,2020-05-24|2020-05-25,2020-05-25|2020-05-26,2020-05-26|2020-05-27,2020-05-27|2020-05-28,2020-05-28|2020-05-29,2020-05-29|2020-05-30,2020-05-30|2020-05-31,2020-05-31|2020-06-01,2020-06-01|2020-06-02,2020-06-02|2020-06-03,2020-06-03|2020-06-04,2020-06-04|2020-06-05,2020-06-05|2020-06-06,2020-06-06|2020-06-07,2020-06-07|2020-06-08,2020-06-08|2020-06-09,2020-06-09|2020-06-10,2020-06-10|2020-06-11,2020-06-11|2020-06-12,2020-06-12|2020-06-13,2020-06-13|2020-06-14,2020-06-14', - 'count' => '42', + 'dates' => '2020-04-30,2020-04-30|2020-05-01,2020-05-01|2020-05-02,2020-05-02|2020-05-03,2020-05-03|2020-05-04,2020-05-04|2020-05-05,2020-05-05|2020-05-06,2020-05-06|2020-05-07,2020-05-07|2020-05-08,2020-05-08|2020-05-09,2020-05-09|2020-05-10,2020-05-10|2020-05-11,2020-05-11|2020-05-12,2020-05-12|2020-05-13,2020-05-13|2020-05-14,2020-05-14|2020-05-15,2020-05-15|2020-05-16,2020-05-16|2020-05-17,2020-05-17|2020-05-18,2020-05-18|2020-05-19,2020-05-19|2020-05-20,2020-05-20|2020-05-21,2020-05-21|2020-05-22,2020-05-22|2020-05-23,2020-05-23|2020-05-24,2020-05-24|2020-05-25,2020-05-25|2020-05-26,2020-05-26|2020-05-27,2020-05-27|2020-05-28,2020-05-28|2020-05-29,2020-05-29|2020-05-30,2020-05-30|2020-05-31,2020-05-31|2020-06-01,2020-06-01|2020-06-02,2020-06-02|2020-06-03,2020-06-03|2020-06-04,2020-06-04|2020-06-05,2020-06-05|2020-06-06,2020-06-06|2020-06-07,2020-06-07|2020-06-08,2020-06-08|2020-06-09,2020-06-09|2020-06-10,2020-06-10|2020-06-11,2020-06-11|2020-06-12,2020-06-12|2020-06-13,2020-06-13|2020-06-14,2020-06-14', + 'count' => '46', ), array ( 'idsite' => '11', @@ -1667,8 +1667,8 @@ class ArchiveInvalidatorTest extends IntegrationTestCase 'period' => '2', 'name' => 'done5f4f9bafeda3443c3c2d4b2ef4dffadc.VisitsSummary', 'report' => 'some.Report', - 'dates' => '2020-05-04,2020-05-10|2020-05-11,2020-05-17|2020-05-18,2020-05-24|2020-05-25,2020-05-31|2020-06-01,2020-06-07|2020-06-08,2020-06-14', - 'count' => '6', + 'dates' => '2020-04-27,2020-05-03|2020-05-04,2020-05-10|2020-05-11,2020-05-17|2020-05-18,2020-05-24|2020-05-25,2020-05-31|2020-06-01,2020-06-07|2020-06-08,2020-06-14', + 'count' => '7', ), array ( 'idsite' => '11', @@ -1683,8 +1683,8 @@ class ArchiveInvalidatorTest extends IntegrationTestCase 'period' => '3', 'name' => 'done5f4f9bafeda3443c3c2d4b2ef4dffadc.VisitsSummary', 'report' => 'some.Report', - 'dates' => '2020-05-01,2020-05-31|2020-06-01,2020-06-30', - 'count' => '2', + 'dates' => '2020-04-01,2020-04-30|2020-05-01,2020-05-31|2020-06-01,2020-06-30', + 'count' => '3', ), array ( 'idsite' => '11', diff --git a/tests/PHPUnit/Integration/PluginTest.php b/tests/PHPUnit/Integration/PluginTest.php index 756749ee01..7ab46a8945 100644 --- a/tests/PHPUnit/Integration/PluginTest.php +++ b/tests/PHPUnit/Integration/PluginTest.php @@ -84,11 +84,11 @@ class PluginTest extends IntegrationTestCase $item = reset($items); $item = json_decode($item, $assocc = true); - $date = end($item); - if (empty($date)) { - return $date; + if (empty($item['startDate'])) { + return null; } + $date = $item['startDate']; return Date::factory($date)->getDatetime(); } diff --git a/tests/PHPUnit/System/ArchiveInvalidationTest.php b/tests/PHPUnit/System/ArchiveInvalidationTest.php index bb1e9e6083..95d756c2ce 100644 --- a/tests/PHPUnit/System/ArchiveInvalidationTest.php +++ b/tests/PHPUnit/System/ArchiveInvalidationTest.php @@ -52,7 +52,7 @@ class ArchiveInvalidationTest extends SystemTestCase return array( array($apiToCall, array('idSite' => self::$fixture->idSite2, - 'testSuffix' => 'Website' . self::$fixture->idSite2 . "_NewDataShouldNotAppear_BecauseDayWasNotInvalidated", + 'testSuffix' => 'Website' . self::$fixture->idSite2 . $this->suffix, 'date' => self::$fixture->dateTimeFirstDateWebsite2, 'periods' => 'day', 'segment' => 'pageUrl=@category/', @@ -135,7 +135,8 @@ class ArchiveInvalidationTest extends SystemTestCase $r = new Request("module=API&method=CoreAdminHome.invalidateArchivedReports&idSites=" . self::$fixture->idSite1 . "&dates=" . $dateToInvalidate1->format('Y-m-d')); $this->assertApiResponseHasNoError($r->process()); - // week reports only are invalidated and we test our daily report will still show old data. + // week reports only are invalidated. we test our daily report will show new data, even though weekly reports only are invalidated, + // because when we track data, it invalidates day periods as well. $this->invalidateTestArchive(self::$fixture->idSite2, 'week', self::$fixture->dateTimeFirstDateWebsite2); } diff --git a/tests/PHPUnit/System/expected/test_Archive_InvalidationWebsite2_NewDataShouldAppear__Actions.getPageUrls_day.xml b/tests/PHPUnit/System/expected/test_Archive_InvalidationWebsite2_NewDataShouldAppear__Actions.getPageUrls_day.xml new file mode 100644 index 0000000000..e91a63b485 --- /dev/null +++ b/tests/PHPUnit/System/expected/test_Archive_InvalidationWebsite2_NewDataShouldAppear__Actions.getPageUrls_day.xml @@ -0,0 +1,173 @@ + + + + + + 6 + 9 + 0 + 0 + 0 + + + 2 + 18 + 2 + 0 + 0 + 0 + 0 + 0% + 0% + pageUrl=^http%253A%252F%252Fpiwik.net%252Fcategory + + + + 2 + 3 + 3 + 0 + 0 + 0 + + + 2 + 2 + 18 + 2 + 0 + 0 + 0% + 0% + http://example.org/category/Page1 + pageUrl==http%253A%252F%252Fexample.org%252Fcategory%252FPage1 + + + + 2 + 3 + 3 + 0 + 0 + 0 + + + 0 + 0% + 0% + http://example.org/category/Page2 + pageUrl==http%253A%252F%252Fexample.org%252Fcategory%252FPage2 + + + + 1 + 2 + 2 + 0 + 0 + 0 + + + 0 + 0% + 0% + http://example.org/category/NewPage + pageUrl==http%253A%252F%252Fexample.org%252Fcategory%252FNewPage + + + + 1 + 1 + 1 + 0 + 0 + 0 + + + 0 + 0% + 0% + http://example.org/category/Page3 + pageUrl==http%253A%252F%252Fexample.org%252Fcategory%252FPage3 + + + + + + 2 + 3 + 3 + 0 + 0 + 0 + + + 0 + 0 + 0 + 0% + 0% + http://example.org/Contact + pageUrl==http%253A%252F%252Fexample.org%252FContact + + + + 2 + 3 + 3 + 0 + 0 + 0 + + + 0 + 0 + 0 + 0% + 0% + http://example.org/Home + pageUrl==http%253A%252F%252Fexample.org%252FHome + + + + 2 + 3 + 0 + 0 + 0 + + + 2 + 0 + 0 + 0 + 0% + 100% + pageUrl=^http%253A%252F%252Fpiwik.net%252FContact + + + + 2 + 3 + 3 + 0 + 0 + 0 + + + 2 + 2 + 0 + 0% + 100% + http://example.org/Contact/ThankYou + pageUrl==http%253A%252F%252Fexample.org%252FContact%252FThankYou + + + + + + + + + \ No newline at end of file diff --git a/tests/PHPUnit/System/expected/test_Archive_InvalidationWebsite2_NewDataShouldAppear__VisitsSummary.get_day.xml b/tests/PHPUnit/System/expected/test_Archive_InvalidationWebsite2_NewDataShouldAppear__VisitsSummary.get_day.xml new file mode 100644 index 0000000000..bfd375c5c9 --- /dev/null +++ b/tests/PHPUnit/System/expected/test_Archive_InvalidationWebsite2_NewDataShouldAppear__VisitsSummary.get_day.xml @@ -0,0 +1,20 @@ + + + + 2 + 0 + 2 + 18 + 0 + 0 + 2 + 12 + 0% + 9 + 1 + + + + + + \ No newline at end of file diff --git a/tests/PHPUnit/System/expected/test_Archive_InvalidationWebsite2_NewDataShouldNotAppear_BecauseDayWasNotInvalidated__Actions.getPageUrls_day.xml b/tests/PHPUnit/System/expected/test_Archive_InvalidationWebsite2_NewDataShouldNotAppear_BecauseDayWasNotInvalidated__Actions.getPageUrls_day.xml index b3a70db9d0..e91a63b485 100644 --- a/tests/PHPUnit/System/expected/test_Archive_InvalidationWebsite2_NewDataShouldNotAppear_BecauseDayWasNotInvalidated__Actions.getPageUrls_day.xml +++ b/tests/PHPUnit/System/expected/test_Archive_InvalidationWebsite2_NewDataShouldNotAppear_BecauseDayWasNotInvalidated__Actions.getPageUrls_day.xml @@ -3,16 +3,16 @@ - 3 - 3 + 6 + 9 0 0 0 - 1 - 6 - 1 + 2 + 18 + 2 0 0 0 @@ -23,18 +23,18 @@ - 1 - 1 - 1 + 2 + 3 + 3 0 0 0 - 1 - 1 - 6 - 1 + 2 + 2 + 18 + 2 0 0 0% @@ -44,9 +44,9 @@ - 1 - 1 - 1 + 2 + 3 + 3 0 0 0 @@ -58,6 +58,22 @@ http://example.org/category/Page2 pageUrl==http%253A%252F%252Fexample.org%252Fcategory%252FPage2 + + + 1 + 2 + 2 + 0 + 0 + 0 + + + 0 + 0% + 0% + http://example.org/category/NewPage + pageUrl==http%253A%252F%252Fexample.org%252Fcategory%252FNewPage + 1 @@ -78,9 +94,9 @@ - 1 - 1 - 1 + 2 + 3 + 3 0 0 0 @@ -96,9 +112,9 @@ - 1 - 1 - 1 + 2 + 3 + 3 0 0 0 @@ -114,14 +130,14 @@ - 1 - 1 + 2 + 3 0 0 0 - 1 + 2 0 0 0 @@ -131,16 +147,16 @@ - 1 - 1 - 1 + 2 + 3 + 3 0 0 0 - 1 - 1 + 2 + 2 0 0% 100% diff --git a/tests/PHPUnit/System/expected/test_Archive_InvalidationWebsite2_NewDataShouldNotAppear_BecauseDayWasNotInvalidated__VisitsSummary.get_day.xml b/tests/PHPUnit/System/expected/test_Archive_InvalidationWebsite2_NewDataShouldNotAppear_BecauseDayWasNotInvalidated__VisitsSummary.get_day.xml index 241f0a7a02..bfd375c5c9 100644 --- a/tests/PHPUnit/System/expected/test_Archive_InvalidationWebsite2_NewDataShouldNotAppear_BecauseDayWasNotInvalidated__VisitsSummary.get_day.xml +++ b/tests/PHPUnit/System/expected/test_Archive_InvalidationWebsite2_NewDataShouldNotAppear_BecauseDayWasNotInvalidated__VisitsSummary.get_day.xml @@ -1,16 +1,16 @@ - 1 + 2 0 - 1 - 6 + 2 + 18 0 0 - 1 - 6 + 2 + 12 0% - 6 + 9 1 diff --git a/tests/PHPUnit/System/expected/test_Archive_InvalidationWebsite2_NewDataShouldNotAppear__Actions.getPageUrls_day.xml b/tests/PHPUnit/System/expected/test_Archive_InvalidationWebsite2_NewDataShouldNotAppear__Actions.getPageUrls_day.xml new file mode 100644 index 0000000000..b3a70db9d0 --- /dev/null +++ b/tests/PHPUnit/System/expected/test_Archive_InvalidationWebsite2_NewDataShouldNotAppear__Actions.getPageUrls_day.xml @@ -0,0 +1,157 @@ + + + + + + 3 + 3 + 0 + 0 + 0 + + + 1 + 6 + 1 + 0 + 0 + 0 + 0 + 0% + 0% + pageUrl=^http%253A%252F%252Fpiwik.net%252Fcategory + + + + 1 + 1 + 1 + 0 + 0 + 0 + + + 1 + 1 + 6 + 1 + 0 + 0 + 0% + 0% + http://example.org/category/Page1 + pageUrl==http%253A%252F%252Fexample.org%252Fcategory%252FPage1 + + + + 1 + 1 + 1 + 0 + 0 + 0 + + + 0 + 0% + 0% + http://example.org/category/Page2 + pageUrl==http%253A%252F%252Fexample.org%252Fcategory%252FPage2 + + + + 1 + 1 + 1 + 0 + 0 + 0 + + + 0 + 0% + 0% + http://example.org/category/Page3 + pageUrl==http%253A%252F%252Fexample.org%252Fcategory%252FPage3 + + + + + + 1 + 1 + 1 + 0 + 0 + 0 + + + 0 + 0 + 0 + 0% + 0% + http://example.org/Contact + pageUrl==http%253A%252F%252Fexample.org%252FContact + + + + 1 + 1 + 1 + 0 + 0 + 0 + + + 0 + 0 + 0 + 0% + 0% + http://example.org/Home + pageUrl==http%253A%252F%252Fexample.org%252FHome + + + + 1 + 1 + 0 + 0 + 0 + + + 1 + 0 + 0 + 0 + 0% + 100% + pageUrl=^http%253A%252F%252Fpiwik.net%252FContact + + + + 1 + 1 + 1 + 0 + 0 + 0 + + + 1 + 1 + 0 + 0% + 100% + http://example.org/Contact/ThankYou + pageUrl==http%253A%252F%252Fexample.org%252FContact%252FThankYou + + + + + + + + + \ No newline at end of file diff --git a/tests/PHPUnit/System/expected/test_Archive_InvalidationWebsite2_NewDataShouldNotAppear__VisitsSummary.get_day.xml b/tests/PHPUnit/System/expected/test_Archive_InvalidationWebsite2_NewDataShouldNotAppear__VisitsSummary.get_day.xml new file mode 100644 index 0000000000..241f0a7a02 --- /dev/null +++ b/tests/PHPUnit/System/expected/test_Archive_InvalidationWebsite2_NewDataShouldNotAppear__VisitsSummary.get_day.xml @@ -0,0 +1,20 @@ + + + + 1 + 0 + 1 + 6 + 0 + 0 + 1 + 6 + 0% + 6 + 1 + + + + + + \ No newline at end of file -- cgit v1.2.3