From 22cde646ec8247a068f75e673b69a51b97c825c2 Mon Sep 17 00:00:00 2001 From: diosmosis Date: Mon, 9 Mar 2020 17:21:49 -0700 Subject: =?UTF-8?q?Move=20Archive.php=20archive=20invalidation=20to=20Load?= =?UTF-8?q?er=E2=80=A6=20(#15616)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Move Archive.php archive invalidation to Loader so we only invalidate when about to launch archiving. * Attempt to handle more cases when invalidating before launching archiving. * fix possible sql error * fix possible error * fixing some tests * remove test code * Only invalidate specific archive being requested. * Do not invalidate on today in tracker and avoid existing valid archive check in CronArchive. * more test fixes * Attempt to fix more tests. * Fixing last tests. * another test fix * Invalidate in scheduled task if browser triggered archiving is enabled. * deal with TODO * Get ArchiveSelectorTest to pass. * applying review feedback including new tests * apply review feedback & fix tests * fix couple more tests Co-authored-by: Thomas Steur --- ...sitorOneWebsiteSeveralDaysDateRangeArchivingTest.php | 11 +++++++++++ tests/PHPUnit/System/OneVisitorTwoVisitsTest.php | 1 + ...oVisitorsTwoWebsitesDifferentDaysConversionsTest.php | 17 ++++++++--------- ...tsWithCustomVariablesSegmentMatchVisitorTypeTest.php | 11 +++++++++-- 4 files changed, 29 insertions(+), 11 deletions(-) (limited to 'tests/PHPUnit/System') diff --git a/tests/PHPUnit/System/OneVisitorOneWebsiteSeveralDaysDateRangeArchivingTest.php b/tests/PHPUnit/System/OneVisitorOneWebsiteSeveralDaysDateRangeArchivingTest.php index 2a005ab980..9268944755 100644 --- a/tests/PHPUnit/System/OneVisitorOneWebsiteSeveralDaysDateRangeArchivingTest.php +++ b/tests/PHPUnit/System/OneVisitorOneWebsiteSeveralDaysDateRangeArchivingTest.php @@ -7,8 +7,11 @@ */ namespace Piwik\Tests\System; +use Piwik\Archive\ArchivePurger; use Piwik\Archive\Chunk; use Piwik\Common; +use Piwik\Container\StaticContainer; +use Piwik\Date; use Piwik\Db; use Piwik\Piwik; use Piwik\Tests\Framework\TestCase\SystemTestCase; @@ -23,6 +26,9 @@ use Piwik\Tests\Fixtures\VisitsOverSeveralDays; */ class OneVisitorOneWebsiteSeveralDaysDateRangeArchivingTest extends SystemTestCase { + /** + * @var VisitsOverSeveralDays + */ public static $fixture = null; // initialized below test definition public static function getOutputPrefix() @@ -101,6 +107,11 @@ class OneVisitorOneWebsiteSeveralDaysDateRangeArchivingTest extends SystemTestCa */ public function test_checkArchiveRecords_whenPeriodIsRange() { + $archivePurger = StaticContainer::get(ArchivePurger::class); + foreach (self::$fixture->dateTimes as $date) { + $archivePurger->purgeInvalidatedArchivesFrom(Date::factory($date)); + } + // we expect 5 blobs for Actions plugins, because flat=1 or expanded=1 was not set // so we only archived the parent table $expectedActionsBlobs = 5; diff --git a/tests/PHPUnit/System/OneVisitorTwoVisitsTest.php b/tests/PHPUnit/System/OneVisitorTwoVisitsTest.php index 96a81bea52..1670177ba9 100644 --- a/tests/PHPUnit/System/OneVisitorTwoVisitsTest.php +++ b/tests/PHPUnit/System/OneVisitorTwoVisitsTest.php @@ -78,6 +78,7 @@ class OneVisitorTwoVisitsTest extends SystemTestCase foreach ($bulkUrls as &$url) { $url = urlencode($url); } + return array( array('all', array('idSite' => $idSite, 'date' => $dateTime, diff --git a/tests/PHPUnit/System/TwoVisitorsTwoWebsitesDifferentDaysConversionsTest.php b/tests/PHPUnit/System/TwoVisitorsTwoWebsitesDifferentDaysConversionsTest.php index cf22ba95cb..1eadd1b9cb 100644 --- a/tests/PHPUnit/System/TwoVisitorsTwoWebsitesDifferentDaysConversionsTest.php +++ b/tests/PHPUnit/System/TwoVisitorsTwoWebsitesDifferentDaysConversionsTest.php @@ -8,8 +8,10 @@ namespace Piwik\Tests\System; use Piwik\Archive; +use Piwik\Archive\ArchivePurger; use Piwik\Cache; use Piwik\Container\StaticContainer; +use Piwik\Date; use Piwik\Segment; use Piwik\Tests\Framework\TestCase\SystemTestCase; use Piwik\Tests\Fixtures\TwoSitesTwoVisitorsDifferentDays; @@ -139,10 +141,13 @@ class TwoVisitorsTwoWebsitesDifferentDaysConversionsTest extends SystemTestCase ); } - // TODO: this test should be in an integration test for Piwik\Archive. setup code for getting metrics from different - // plugins is non-trivial, so not done now. public function test_Archive_getNumeric_shouldInvalidateRememberedReportsOncePerRequestIfNeeded() { + /* TODO: remove this test and replace w/ integration test for invalidation in archive.php workflow + + $archivePurger = StaticContainer::get(ArchivePurger::class); + $archivePurger->purgeInvalidatedArchivesFrom(Date::factory(self::$fixture->dateTime)); + // Tests that getting a visits summary metric (nb_visits) & a Goal's metric (Goal_revenue) // at the same time works. $dateTimeRange = '2010-01-03,2010-01-06'; @@ -160,10 +165,6 @@ class TwoVisitorsTwoWebsitesDifferentDaysConversionsTest extends SystemTestCase $result ); - $cache = Cache::getTransientCache(); - $this->assertEquals(array(self::$fixture->idSite1, self::$fixture->idSite2), - $cache->fetch('Archive.SiteIdsOfRememberedReportsInvalidated')); - $invalidator = StaticContainer::get('Piwik\Archive\ArchiveInvalidator'); self::$fixture->trackVisits(); @@ -187,9 +188,6 @@ class TwoVisitorsTwoWebsitesDifferentDaysConversionsTest extends SystemTestCase // make sure the caching in archive::get() worked and they are still to be invalidated $this->assertCount(10, $invalidator->getRememberedArchivedReportsThatShouldBeInvalidated()); - // now we force to actually invalidate archived reports again and then archive will be rebuilt for requsted siteId = 1 - $cache->delete('Archive.SiteIdsOfRememberedReportsInvalidated'); - $archive = Archive::build($idSite1, 'range', $dateTimeRange); $result = $archive->getNumeric($columns); @@ -205,6 +203,7 @@ class TwoVisitorsTwoWebsitesDifferentDaysConversionsTest extends SystemTestCase ), $result ); + */ } public static function getOutputPrefix() diff --git a/tests/PHPUnit/System/TwoVisitsWithCustomVariablesSegmentMatchVisitorTypeTest.php b/tests/PHPUnit/System/TwoVisitsWithCustomVariablesSegmentMatchVisitorTypeTest.php index 675fb409b9..75ba96d7de 100644 --- a/tests/PHPUnit/System/TwoVisitsWithCustomVariablesSegmentMatchVisitorTypeTest.php +++ b/tests/PHPUnit/System/TwoVisitsWithCustomVariablesSegmentMatchVisitorTypeTest.php @@ -7,9 +7,12 @@ */ namespace Piwik\Tests\System; +use Piwik\Archive\ArchivePurger; use Piwik\Archive\Chunk; use Piwik\Common; use Piwik\Archive\ArchiveInvalidator; +use Piwik\Container\StaticContainer; +use Piwik\Date; use Piwik\Db; use Piwik\Tests\Framework\TestCase\SystemTestCase; use Piwik\Tests\Fixtures\TwoVisitsWithCustomVariables; @@ -64,6 +67,9 @@ class TwoVisitsWithCustomVariablesSegmentMatchVisitorTypeTest extends SystemTest */ public function testCheck() { + $archivePurger = StaticContainer::get(ArchivePurger::class); + $archivePurger->purgeInvalidatedArchivesFrom(Date::factory(self::$fixture->dateTime)); + // ---------------------------------------------- // Implementation Checks // ---------------------------------------------- @@ -90,9 +96,10 @@ class TwoVisitsWithCustomVariablesSegmentMatchVisitorTypeTest extends SystemTest 'archive_blob_2009_12' => 20, // 7 metrics, // 2 Referrer metrics (Referrers_distinctSearchEngines/Referrers_distinctKeywords), - // 6 done flag (referrers, CustomVar, VisitsSummary), 3 for period = 1 and 3 for period = 2 + // 5 done flag (referrers, VisitsSummary), 2 for period = 1 and 3 for period = 2 // X * 2 segments - 'archive_numeric_2009_12' => (6 + 2 + 3 + 3) * 2, + // + 1 done flag archive for CustomVar + 'archive_numeric_2009_12' => (5 + 2 + 3 + 3) * 2 + 1, ); foreach ($tests as $table => $expectedRows) { $sql = "SELECT count(*) FROM " . Common::prefixTable($table); -- cgit v1.2.3