diff options
author | diosmosis <diosmosis@users.noreply.github.com> | 2019-12-24 14:24:44 +0300 |
---|---|---|
committer | GitHub <noreply@github.com> | 2019-12-24 14:24:44 +0300 |
commit | 42454eb56e02899d2282e1eb6bf81983e666d933 (patch) | |
tree | f62b48f2c26f8c1a5bd0519ec43264b14a239db8 /tests/PHPUnit | |
parent | 2915d75f69f2340ee6dcf1a9f4401b658fb8c966 (diff) |
Lock when archiving and avoid invalidating sites that have archiving in progress (#15272)
* Use a lock when archiving and do not invalidate when archiving is in progress.
* Add and fix tests + modify workflow.
* forgot to add file and remove TODO
* Remove use of argument.
* Add back min archive time processed code and start on tests for it.
* Finish new LoaderTest.
* Fix new tests.
Diffstat (limited to 'tests/PHPUnit')
4 files changed, 284 insertions, 1 deletions
diff --git a/tests/PHPUnit/Integration/ArchiveProcessor/ArchivingStatusTest.php b/tests/PHPUnit/Integration/ArchiveProcessor/ArchivingStatusTest.php new file mode 100644 index 0000000000..e7a873ffea --- /dev/null +++ b/tests/PHPUnit/Integration/ArchiveProcessor/ArchivingStatusTest.php @@ -0,0 +1,87 @@ +<?php +/** + * Piwik - free/libre analytics platform + * + * @link https://matomo.org + * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later + * + */ + +namespace Piwik\Tests\Integration\ArchiveProcessor; + +use Piwik\ArchiveProcessor\ArchivingStatus; +use Piwik\ArchiveProcessor\Parameters; +use Piwik\Common; +use Piwik\Container\StaticContainer; +use Piwik\Db; +use Piwik\Period\Factory; +use Piwik\Segment; +use Piwik\Site; +use Piwik\Tests\Framework\Fixture; +use Piwik\Tests\Framework\TestCase\IntegrationTestCase; + +class ArchivingStatusTest extends IntegrationTestCase +{ + public function setUp() + { + parent::setUp(); + + Fixture::createWebsite('2010-02-03 00:00:00'); + Fixture::createWebsite('2010-02-03 00:00:00'); + } + + public function test_archiveStartedAndArchiveFinished_workflow() + { + /** @var ArchivingStatus $archivingStatus */ + $archivingStatus = StaticContainer::get(ArchivingStatus::class); + + $params = new Parameters(new Site(1), Factory::build('month', '2012-02-04'), new Segment('', [1])); + $archivingStatus->archiveStarted($params); + + $lock = $archivingStatus->getCurrentArchivingLock(); + $this->assertNotEmpty($lock); + $this->assertTrue($lock->isLocked()); + + $this->assertEquals([ + [ + 'key' => 'Archiving.1.fab0afcc3a068ecd91b14d50f3bc911d.' . getmypid(), + 'expiry_time' => time() + ArchivingStatus::DEFAULT_ARCHIVING_TTL, + ], + ], $this->getLockKeysAndTtls()); + + $archivingStatus->archiveFinished(); + + $this->assertEquals([], $this->getLockKeysAndTtls()); + } + + public function test_getSitesCurrentlyArchiving_returnsAllSitesArchiving() + { + /** @var ArchivingStatus $archivingStatus */ + $archivingStatus = StaticContainer::get(ArchivingStatus::class); + + $params = new Parameters(new Site(1), Factory::build('month', '2012-02-04'), new Segment('', [1])); + $archivingStatus->archiveStarted($params); + + $params = new Parameters(new Site(1), Factory::build('month', '2012-02-04'), new Segment('browserCode==ff', [1])); + $archivingStatus->archiveStarted($params); + + $params = new Parameters(new Site(2), Factory::build('month', '2012-02-04'), new Segment('', [1])); + $archivingStatus->archiveStarted($params); + + $this->assertEquals([ + 1, 2, + ], $archivingStatus->getSitesCurrentlyArchiving()); + + $archivingStatus->archiveFinished(); + $archivingStatus->archiveFinished(); + + $this->assertEquals([ + 1, + ], $archivingStatus->getSitesCurrentlyArchiving()); + } + + private function getLockKeysAndTtls() + { + return Db::fetchAll("SELECT `key`, expiry_time FROM `" . Common::prefixTable('locks') . '`'); + } +}
\ No newline at end of file diff --git a/tests/PHPUnit/Integration/ArchiveProcessor/LoaderTest.php b/tests/PHPUnit/Integration/ArchiveProcessor/LoaderTest.php new file mode 100644 index 0000000000..e99d495931 --- /dev/null +++ b/tests/PHPUnit/Integration/ArchiveProcessor/LoaderTest.php @@ -0,0 +1,122 @@ +<?php +/** + * Piwik - free/libre analytics platform + * + * @link https://matomo.org + * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later + * + */ + +namespace Piwik\Tests\Integration\ArchiveProcessor; + + +use Piwik\ArchiveProcessor\Parameters; +use Piwik\ArchiveProcessor\Loader; +use Piwik\Common; +use Piwik\Config; +use Piwik\DataAccess\ArchiveTableCreator; +use Piwik\DataAccess\ArchiveWriter; +use Piwik\Date; +use Piwik\Db; +use Piwik\Period\Factory; +use Piwik\Segment; +use Piwik\Site; +use Piwik\Tests\Framework\Fixture; +use Piwik\Tests\Framework\TestCase\IntegrationTestCase; + +class LoaderTest extends IntegrationTestCase +{ + protected static function beforeTableDataCached() + { + parent::beforeTableDataCached(); + + Fixture::createWebsite('2012-02-03 00:00:00'); + } + + public function test_loadExistingArchiveIdFromDb_returnsFalsesIfNoArchiveFound() + { + $params = new Parameters(new Site(1), Factory::build('day', '2015-03-03'), new Segment('', [1])); + $loader = new Loader($params); + + $archiveInfo = $loader->loadExistingArchiveIdFromDb(); + + $this->assertEquals([false, false, false], $archiveInfo); + } + + /** + * @dataProvider getTestDataForLoadExistingArchiveIdFromDbDebugConfig + */ + public function test_loadExistingArchiveIdFromDb_returnsFalsesPeriodIsForcedToArchive($periodType, $configSetting) + { + $date = $periodType == 'range' ? '2015-03-03,2015-03-04' : '2015-03-03'; + $params = new Parameters(new Site(1), Factory::build($periodType, $date), new Segment('', [1])); + $this->insertArchive($params); + + $loader = new Loader($params); + + $archiveInfo = $loader->loadExistingArchiveIdFromDb(); + $this->assertNotEquals([false, false, false], $archiveInfo); + + Config::getInstance()->Debug[$configSetting] = 1; + + $archiveInfo = $loader->loadExistingArchiveIdFromDb(); + $this->assertEquals([false, false, false], $archiveInfo); + } + + public function getTestDataForLoadExistingArchiveIdFromDbDebugConfig() + { + return [ + ['day', 'always_archive_data_day'], + ['week', 'always_archive_data_period'], + ['month', 'always_archive_data_period'], + ['year', 'always_archive_data_period'], + ['range', 'always_archive_data_range'], + ]; + } + + public function test_loadExistingArchiveIdFromDb_returnsArchiveIfArchiveInThePast() + { + $params = new Parameters(new Site(1), Factory::build('month', '2015-03-03'), new Segment('', [1])); + $this->insertArchive($params); + + $loader = new Loader($params); + + $archiveInfo = $loader->loadExistingArchiveIdFromDb(); + $this->assertEquals(['1', '10', '0'], $archiveInfo); + } + + public function test_loadExistingArchiveIdFromDb_returnsArchiveIfForACurrentPeriod_AndNewEnough() + { + $params = new Parameters(new Site(1), Factory::build('day', 'now'), new Segment('', [1])); + $this->insertArchive($params, $tsArchived = time() - 1); + + $loader = new Loader($params); + + $archiveInfo = $loader->loadExistingArchiveIdFromDb(); + $this->assertEquals(['1', '10', '0'], $archiveInfo); + } + + public function test_loadExistingArchiveIdFromDb_returnsNoArchiveIfForACurrentPeriod_AndNoneAreNewEnough() + { + $params = new Parameters(new Site(1), Factory::build('month', 'now'), new Segment('', [1])); + $this->insertArchive($params, $tsArchived = time() - 3 * 3600); + + $loader = new Loader($params); + + $archiveInfo = $loader->loadExistingArchiveIdFromDb(); + $this->assertEquals([false, false, false], $archiveInfo); + } + + private function insertArchive(Parameters $params, $tsArchived = null, $visits = 10) + { + $archiveWriter = new ArchiveWriter($params); + $archiveWriter->initNewArchive(); + $archiveWriter->insertRecord('nb_visits', $visits); + $archiveWriter->finalizeArchive(); + + if ($tsArchived) { + Db::query("UPDATE " . ArchiveTableCreator::getNumericTable($params->getPeriod()->getDateStart()) . " SET ts_archived = ?", + [Date::factory($tsArchived)->getDatetime()]); + } + } +}
\ No newline at end of file diff --git a/tests/PHPUnit/Integration/DataAccess/ArchiveInvalidatorTest.php b/tests/PHPUnit/Integration/DataAccess/ArchiveInvalidatorTest.php index 530bd6e112..36365b7c49 100644 --- a/tests/PHPUnit/Integration/DataAccess/ArchiveInvalidatorTest.php +++ b/tests/PHPUnit/Integration/DataAccess/ArchiveInvalidatorTest.php @@ -8,7 +8,10 @@ namespace Piwik\Tests\Integration\DataAccess; +use Piwik\ArchiveProcessor\ArchivingStatus; +use Piwik\ArchiveProcessor\Parameters; use Piwik\ArchiveProcessor\Rules; +use Piwik\Container\StaticContainer; use Piwik\CronArchive\SitesToReprocessDistributedList; use Piwik\DataAccess\ArchiveTableCreator; use Piwik\DataAccess\ArchiveWriter; @@ -16,9 +19,11 @@ use Piwik\DataAccess\Model; use Piwik\Date; use Piwik\Db; use Piwik\Option; +use Piwik\Period\Factory; use Piwik\Piwik; use Piwik\Plugins\CoreAdminHome\Tasks\ArchivesToPurgeDistributedList; use Piwik\Plugins\PrivacyManager\PrivacyManager; +use Piwik\Site; use Piwik\Tests\Framework\Fixture; use Piwik\Tests\Framework\TestCase\IntegrationTestCase; use Piwik\Archive\ArchiveInvalidator; @@ -71,7 +76,7 @@ class ArchiveInvalidatorTest extends IntegrationTestCase { parent::setUp(); - $this->invalidator = new ArchiveInvalidator(new Model()); + $this->invalidator = new ArchiveInvalidator(new Model(), StaticContainer::get(ArchivingStatus::class)); } public function test_rememberToInvalidateArchivedReportsLater_shouldCreateAnEntryInCaseThereIsNoneYet() @@ -196,6 +201,39 @@ class ArchiveInvalidatorTest extends IntegrationTestCase $this->assertSame($expected, $reports); } + public function test_markArchivesAsInvalidated_shouldSkipArchivingForInProgressSites() + { + /** @var ArchivingStatus $archivingStatus */ + $archivingStatus = StaticContainer::get(ArchivingStatus::class); + + $params = new Parameters(new Site(7), Factory::build('month', '2012-02-03'), new Segment('', [2])); + $archivingStatus->archiveStarted($params); + + $params = new Parameters(new Site(5), Factory::build('day', '2012-02-03'), new Segment('', [7])); + $archivingStatus->archiveStarted($params); + + $this->rememberReportsForManySitesAndDates(); + + $idSites = array(2, 10, 7, 5); + $dates = array( + Date::factory('2014-04-05'), + Date::factory('2014-04-08'), + Date::factory('2014-05-05'), + ); + + $this->invalidator->markArchivesAsInvalidated($idSites, $dates, 'week'); + $reports = $this->invalidator->getRememberedArchivedReportsThatShouldBeInvalidated(); + + $expected = array( + '2014-04-05' => [1,4,7], + '2014-04-06' => [3], + '2014-05-05' => [5], + '2014-04-08' => [7], + '2014-05-08' => [7], + ); + $this->assertSame($expected, $reports); + } + private function rememberReport($idSite, $date) { $date = Date::factory($date); diff --git a/tests/PHPUnit/Integration/DataAccess/LogAggregatorTest.php b/tests/PHPUnit/Integration/DataAccess/LogAggregatorTest.php index 8b7ac6024c..968b98e73c 100644 --- a/tests/PHPUnit/Integration/DataAccess/LogAggregatorTest.php +++ b/tests/PHPUnit/Integration/DataAccess/LogAggregatorTest.php @@ -8,13 +8,18 @@ namespace Piwik\Tests\Integration\DataAccess; +use Piwik\ArchiveProcessor\ArchivingStatus; use Piwik\ArchiveProcessor\Parameters; +use Piwik\Common; +use Piwik\Container\StaticContainer; use Piwik\DataAccess\LogAggregator; use Piwik\Date; +use Piwik\Db; use Piwik\Period; use Piwik\Segment; use Piwik\Site; use Piwik\Tests\Fixtures\OneVisitorTwoVisits; +use Piwik\Tests\Framework\Fixture; use Piwik\Tests\Framework\TestCase\IntegrationTestCase; /** @@ -135,6 +140,37 @@ class LogAggregatorTest extends IntegrationTestCase ]; $this->assertEquals($expected, $result); } + + public function test_logAggregatorUpdatesArchiveStatusExpireTime() + { + $t = Fixture::getTracker(self::$fixture->idSite, '2010-03-06 14:22:33'); + $t->setUrl('http://example.com/here/we/go'); + $t->doTrackPageView('here we go'); + + $params = new Parameters(new Site(self::$fixture->idSite), Period\Factory::build('day', self::$fixture->dateTime), new Segment('', [self::$fixture->idSite])); + + $archiveStatus = StaticContainer::get(ArchivingStatus::class); + $archiveStatus->archiveStarted($params); + + $locks = $this->getAllLocks(); + $this->assertCount(1, $locks); + $expireTime = $locks[0]['expiry_time']; + + sleep(1); + + $this->logAggregator->queryVisitsByDimension(['visit_total_time']); + + $locks = $this->getAllLocks(); + $this->assertCount(1, $locks); + $expireTimeNew = $locks[0]['expiry_time']; + + $this->assertGreaterThan($expireTime, $expireTimeNew); + } + + private function getAllLocks() + { + return Db::fetchAll("SELECT `key`, expiry_time FROM `" . Common::prefixTable('locks') . '`'); + } } LogAggregatorTest::$fixture = new OneVisitorTwoVisits(); |