diff options
author | diosmosis <diosmosis@users.noreply.github.com> | 2019-12-24 14:24:44 +0300 |
---|---|---|
committer | GitHub <noreply@github.com> | 2019-12-24 14:24:44 +0300 |
commit | 42454eb56e02899d2282e1eb6bf81983e666d933 (patch) | |
tree | f62b48f2c26f8c1a5bd0519ec43264b14a239db8 | |
parent | 2915d75f69f2340ee6dcf1a9f4401b658fb8c966 (diff) |
Lock when archiving and avoid invalidating sites that have archiving in progress (#15272)
* Use a lock when archiving and do not invalidate when archiving is in progress.
* Add and fix tests + modify workflow.
* forgot to add file and remove TODO
* Remove use of argument.
* Add back min archive time processed code and start on tests for it.
* Finish new LoaderTest.
* Fix new tests.
-rw-r--r-- | config/global.php | 2 | ||||
-rw-r--r-- | core/Archive/ArchiveInvalidator.php | 10 | ||||
-rw-r--r-- | core/ArchiveProcessor/ArchivingStatus.php | 107 | ||||
-rw-r--r-- | core/ArchiveProcessor/Loader.php | 45 | ||||
-rw-r--r-- | core/ArchiveProcessor/Rules.php | 2 | ||||
-rw-r--r-- | core/Concurrency/Lock.php | 9 | ||||
-rw-r--r-- | core/Concurrency/LockBackend.php | 12 | ||||
-rw-r--r-- | core/DataAccess/ArchiveSelector.php | 7 | ||||
-rw-r--r-- | core/DataAccess/ArchivingDbAdapter.php | 107 | ||||
-rw-r--r-- | core/DataAccess/LogAggregator.php | 10 | ||||
-rw-r--r-- | core/DataAccess/Model.php | 9 | ||||
-rw-r--r-- | tests/PHPUnit/Integration/ArchiveProcessor/ArchivingStatusTest.php | 87 | ||||
-rw-r--r-- | tests/PHPUnit/Integration/ArchiveProcessor/LoaderTest.php | 122 | ||||
-rw-r--r-- | tests/PHPUnit/Integration/DataAccess/ArchiveInvalidatorTest.php | 40 | ||||
-rw-r--r-- | tests/PHPUnit/Integration/DataAccess/LogAggregatorTest.php | 36 |
15 files changed, 580 insertions, 25 deletions
diff --git a/config/global.php b/config/global.php index 9d8666efd0..5499b0920a 100644 --- a/config/global.php +++ b/config/global.php @@ -217,5 +217,5 @@ return array( \Piwik\CronArchive\Performance\Logger::class => DI\object()->constructorParameter('logger', DI\get('archiving.performance.logger')), - 'Piwik\Concurrency\LockBackend' => DI\object(\Piwik\Concurrency\LockBackend\MySqlLockBackend::class) + \Piwik\Concurrency\LockBackend::class => \DI\get(\Piwik\Concurrency\LockBackend\MySqlLockBackend::class), ); diff --git a/core/Archive/ArchiveInvalidator.php b/core/Archive/ArchiveInvalidator.php index 4504cd5446..649f1cff02 100644 --- a/core/Archive/ArchiveInvalidator.php +++ b/core/Archive/ArchiveInvalidator.php @@ -10,6 +10,7 @@ namespace Piwik\Archive; use Piwik\Archive\ArchiveInvalidator\InvalidationResult; +use Piwik\ArchiveProcessor\ArchivingStatus; use Piwik\CronArchive\SitesToReprocessDistributedList; use Piwik\DataAccess\ArchiveTableCreator; use Piwik\DataAccess\Model; @@ -54,9 +55,15 @@ class ArchiveInvalidator */ private $model; - public function __construct(Model $model) + /** + * @var ArchivingStatus + */ + private $archivingStatus; + + public function __construct(Model $model, ArchivingStatus $archivingStatus) { $this->model = $model; + $this->archivingStatus = $archivingStatus; } public function rememberToInvalidateArchivedReportsLater($idSite, Date $date) @@ -207,6 +214,7 @@ class ArchiveInvalidator } $periodDates = $this->getUniqueDates($periodDates); + $this->markArchivesInvalidated($idSites, $periodDates, $segment); $yearMonths = array_keys($periodDates); diff --git a/core/ArchiveProcessor/ArchivingStatus.php b/core/ArchiveProcessor/ArchivingStatus.php new file mode 100644 index 0000000000..898786db8a --- /dev/null +++ b/core/ArchiveProcessor/ArchivingStatus.php @@ -0,0 +1,107 @@ +<?php +/** + * Piwik - free/libre analytics platform + * + * @link https://matomo.org + * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later + * + */ + +namespace Piwik\ArchiveProcessor; + +use Piwik\Concurrency\Lock; +use Piwik\Concurrency\LockBackend; +use Piwik\Container\StaticContainer; +use Piwik\SettingsPiwik; + +class ArchivingStatus +{ + const LOCK_KEY_PREFIX = 'Archiving'; + const DEFAULT_ARCHIVING_TTL = 7200; // 2 hours + + /** + * @var LockBackend + */ + private $lockBackend; + + /** + * @var int + */ + private $archivingTTLSecs; + + /** + * @var Lock[] + */ + private $lockStack = []; + + public function __construct(LockBackend $lockBackend, $archivingTTLSecs = self::DEFAULT_ARCHIVING_TTL) + { + $this->lockBackend = $lockBackend; + $this->archivingTTLSecs = $archivingTTLSecs; + } + + public function archiveStarted(Parameters $params) + { + $lock = $this->makeArchivingLock($params); + $lock->acquireLock($this->getInstanceProcessId(), $this->archivingTTLSecs); + array_push($this->lockStack, $lock); + } + + public function archiveFinished() + { + $lock = array_pop($this->lockStack); + $lock->unlock(); + } + + public function getCurrentArchivingLock() + { + if (empty($this->lockStack)) { + return null; + } + return end($this->lockStack); + } + + public function getSitesCurrentlyArchiving() + { + $lockMeta = new Lock($this->lockBackend, self::LOCK_KEY_PREFIX . '.'); + $acquiredLocks = $lockMeta->getAllAcquiredLockKeys(); + + $sitesCurrentlyArchiving = []; + foreach ($acquiredLocks as $lockKey) { + $parts = explode('.', $lockKey); + if (!isset($parts[1])) { + continue; + } + $sitesCurrentlyArchiving[] = (int) $parts[1]; + } + $sitesCurrentlyArchiving = array_unique($sitesCurrentlyArchiving); + $sitesCurrentlyArchiving = array_values($sitesCurrentlyArchiving); + + return $sitesCurrentlyArchiving; + } + + /** + * @return Lock + */ + private function makeArchivingLock(Parameters $params) + { + $doneFlag = Rules::getDoneStringFlagFor([$params->getSite()->getId()], $params->getSegment(), + $params->getPeriod()->getLabel(), $params->getRequestedPlugin()); + + $lockKeyParts = [ + self::LOCK_KEY_PREFIX, + $params->getSite()->getId(), + + // md5 to keep it within the 70 char limit in the table + md5($params->getPeriod()->getId() . $params->getPeriod()->getRangeString() . $doneFlag), + ]; + + $lockKeyPrefix = implode('.', $lockKeyParts); + return new Lock(StaticContainer::get(LockBackend::class), $lockKeyPrefix, $this->archivingTTLSecs); + } + + private function getInstanceProcessId() + { + return SettingsPiwik::getPiwikInstanceId() . '.' . getmypid(); + } +}
\ No newline at end of file diff --git a/core/ArchiveProcessor/Loader.php b/core/ArchiveProcessor/Loader.php index f08e63003b..4564146bd9 100644 --- a/core/ArchiveProcessor/Loader.php +++ b/core/ArchiveProcessor/Loader.php @@ -8,15 +8,12 @@ */ namespace Piwik\ArchiveProcessor; -use Piwik\Archive; use Piwik\Cache; -use Piwik\CacheId; -use Piwik\Common; use Piwik\Config; +use Piwik\Container\StaticContainer; use Piwik\Context; use Piwik\DataAccess\ArchiveSelector; use Piwik\Date; -use Piwik\Period; use Piwik\Piwik; /** @@ -73,12 +70,21 @@ class Loader return $idArchive; } - list($visits, $visitsConverted) = $this->prepareCoreMetricsArchive($visits, $visitsConverted); - list($idArchive, $visits) = $this->prepareAllPluginsArchive($visits, $visitsConverted); + /** @var ArchivingStatus $archivingStatus */ + $archivingStatus = StaticContainer::get(ArchivingStatus::class); + $archivingStatus->archiveStarted($this->params); + + try { + list($visits, $visitsConverted) = $this->prepareCoreMetricsArchive($visits, $visitsConverted); + list($idArchive, $visits) = $this->prepareAllPluginsArchive($visits, $visitsConverted); + } finally { + $archivingStatus->archiveFinished(); + } if ($this->isThereSomeVisits($visits) || PluginsArchiver::doesAnyPluginArchiveWithoutVisits()) { return $idArchive; } + return false; } @@ -158,9 +164,11 @@ class Loader * Returns the idArchive if the archive is available in the database for the requested plugin. * Returns false if the archive needs to be processed. * + * (public for tests) + * * @return array */ - protected function loadExistingArchiveIdFromDb() + public function loadExistingArchiveIdFromDb() { $noArchiveFound = array(false, false, false); @@ -168,7 +176,8 @@ class Loader return $noArchiveFound; } - $idAndVisits = ArchiveSelector::getArchiveIdAndVisits($this->params); + $minDatetimeArchiveProcessedUTC = $this->getMinTimeArchiveProcessed(); + $idAndVisits = ArchiveSelector::getArchiveIdAndVisits($this->params, $minDatetimeArchiveProcessedUTC); if (!$idAndVisits) { return $noArchiveFound; @@ -177,6 +186,26 @@ class Loader return $idAndVisits; } + /** + * Returns the minimum archive processed datetime to look at. Only public for tests. + * + * @return int|bool Datetime timestamp, or false if must look at any archive available + */ + protected function getMinTimeArchiveProcessed() + { + $endDateTimestamp = self::determineIfArchivePermanent($this->params->getDateEnd()); + if ($endDateTimestamp) { + // past archive + return $endDateTimestamp; + } + $dateStart = $this->params->getDateStart(); + $period = $this->params->getPeriod(); + $segment = $this->params->getSegment(); + $site = $this->params->getSite(); + // in-progress archive + return Rules::getMinTimeProcessedForInProgressArchive($dateStart, $period, $segment, $site); + } + protected static function determineIfArchivePermanent(Date $dateEnd) { $now = time(); diff --git a/core/ArchiveProcessor/Rules.php b/core/ArchiveProcessor/Rules.php index 90d8d46a23..f054ce5d74 100644 --- a/core/ArchiveProcessor/Rules.php +++ b/core/ArchiveProcessor/Rules.php @@ -112,7 +112,7 @@ class Rules return $doneFlags; } - public static function getMinTimeProcessedForTemporaryArchive( + public static function getMinTimeProcessedForInProgressArchive( Date $dateStart, \Piwik\Period $period, Segment $segment, Site $site) { $todayArchiveTimeToLive = self::getPeriodArchiveTimeToLiveDefault($period->getLabel()); diff --git a/core/Concurrency/Lock.php b/core/Concurrency/Lock.php index e3200310bd..f63bec640a 100644 --- a/core/Concurrency/Lock.php +++ b/core/Concurrency/Lock.php @@ -23,12 +23,19 @@ class Lock private $lockKey = null; private $lockValue = null; + private $defaultTtl = null; - public function __construct(LockBackend $backend, $lockKeyStart) + public function __construct(LockBackend $backend, $lockKeyStart, $defaultTtl = null) { $this->backend = $backend; $this->lockKeyStart = $lockKeyStart; $this->lockKey = $this->lockKeyStart; + $this->defaultTtl = $defaultTtl; + } + + public function reexpireLock() + { + $this->expireLock($this->defaultTtl); } public function getNumberOfAcquiredLocks() diff --git a/core/Concurrency/LockBackend.php b/core/Concurrency/LockBackend.php index 7f67cbafa5..4ca3467657 100644 --- a/core/Concurrency/LockBackend.php +++ b/core/Concurrency/LockBackend.php @@ -12,15 +12,15 @@ namespace Piwik\Concurrency; interface LockBackend { /** - * TODO + * Returns lock keys matching a pattern. * * @param $pattern - * @return mixed + * @return string[] */ public function getKeysMatchingPattern($pattern); /** - * TODO + * Set a key value if the key is not already set. * * @param $lockKey * @param $lockValue @@ -30,7 +30,7 @@ interface LockBackend public function setIfNotExists($lockKey, $lockValue, $ttlInSeconds); /** - * TODO + * Get the lock value for a key if any. * * @param $lockKey * @return mixed @@ -38,7 +38,7 @@ interface LockBackend public function get($lockKey); /** - * TODO + * Delete the lock with key = $lockKey if the lock has the given value. * * @param $lockKey * @param $lockValue @@ -47,7 +47,7 @@ interface LockBackend public function deleteIfKeyHasValue($lockKey, $lockValue); /** - * TODO + * Update expiration for a lock if the lock with the specified key has the given value. * * @param $lockKey * @param $lockValue diff --git a/core/DataAccess/ArchiveSelector.php b/core/DataAccess/ArchiveSelector.php index 5e73f1cb8f..d8063c33c0 100644 --- a/core/DataAccess/ArchiveSelector.php +++ b/core/DataAccess/ArchiveSelector.php @@ -61,6 +61,11 @@ class ArchiveSelector $numericTable = ArchiveTableCreator::getNumericTable($dateStart); + $minDatetimeIsoArchiveProcessedUTC = null; + if ($minDatetimeArchiveProcessedUTC) { + $minDatetimeIsoArchiveProcessedUTC = Date::factory($minDatetimeArchiveProcessedUTC)->getDatetime(); + } + $requestedPlugin = $params->getRequestedPlugin(); $segment = $params->getSegment(); $plugins = array("VisitsSummary", $requestedPlugin); @@ -68,7 +73,7 @@ class ArchiveSelector $doneFlags = Rules::getDoneFlags($plugins, $segment); $doneFlagValues = Rules::getSelectableDoneFlagValues(); - $results = self::getModel()->getArchiveIdAndVisits($numericTable, $idSite, $period, $dateStartIso, $dateEndIso, $doneFlags, $doneFlagValues); + $results = self::getModel()->getArchiveIdAndVisits($numericTable, $idSite, $period, $dateStartIso, $dateEndIso, $minDatetimeIsoArchiveProcessedUTC, $doneFlags, $doneFlagValues); if (empty($results)) { return false; diff --git a/core/DataAccess/ArchivingDbAdapter.php b/core/DataAccess/ArchivingDbAdapter.php new file mode 100644 index 0000000000..e0ebfbe339 --- /dev/null +++ b/core/DataAccess/ArchivingDbAdapter.php @@ -0,0 +1,107 @@ +<?php +/** + * Piwik - free/libre analytics platform + * + * @link https://matomo.org + * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later + * + */ + +namespace Piwik\DataAccess; + +use Piwik\Concurrency\Lock; +use Piwik\Db\AdapterInterface; +use Psr\Log\LoggerInterface; + +class ArchivingDbAdapter +{ + /** + * @var AdapterInterface|\Zend_Db_Adapter_Abstract + */ + private $wrapped; + + /** + * @var Lock + */ + private $archivingLock; + + /** + * @var LoggerInterface + */ + private $logger; + + public function __construct($wrapped, Lock $archivingLock = null, LoggerInterface $logger = null) + { + $this->wrapped = $wrapped; + $this->archivingLock = $archivingLock; + $this->logger = $logger; + } + + public function __call($name, $arguments) + { + return call_user_func_array([$this->wrapped, $name], $arguments); + } + + public function exec($sql) + { + $this->reexpireLock(); + $this->logSql($sql); + + return call_user_func_array([$this->wrapped, __FUNCTION__], func_get_args()); + } + + public function query($sql) + { + $this->reexpireLock(); + $this->logSql($sql); + + return call_user_func_array([$this->wrapped, __FUNCTION__], func_get_args()); + } + + public function fetchAll($sql) + { + $this->reexpireLock(); + $this->logSql($sql); + + return call_user_func_array([$this->wrapped, __FUNCTION__], func_get_args()); + } + + public function fetchRow($sql) + { + $this->reexpireLock(); + $this->logSql($sql); + + return call_user_func_array([$this->wrapped, __FUNCTION__], func_get_args()); + } + + public function fetchOne($sql) + { + $this->reexpireLock(); + $this->logSql($sql); + + return call_user_func_array([$this->wrapped, __FUNCTION__], func_get_args()); + } + + public function fetchAssoc($sql) + { + $this->reexpireLock(); + $this->logSql($sql); + + return call_user_func_array([$this->wrapped, __FUNCTION__], func_get_args()); + } + + private function logSql($sql) + { + // Log on DEBUG level all SQL archiving queries + if ($this->logger) { + $this->logger->debug($sql); + } + } + + private function reexpireLock() + { + if ($this->archivingLock) { + $this->archivingLock->reexpireLock(); + } + } +}
\ No newline at end of file diff --git a/core/DataAccess/LogAggregator.php b/core/DataAccess/LogAggregator.php index 4c169e30c7..0fef1ba31c 100644 --- a/core/DataAccess/LogAggregator.php +++ b/core/DataAccess/LogAggregator.php @@ -8,8 +8,10 @@ */ namespace Piwik\DataAccess; +use Piwik\ArchiveProcessor\ArchivingStatus; use Piwik\ArchiveProcessor\Parameters; use Piwik\Common; +use Piwik\Concurrency\Lock; use Piwik\Config; use Piwik\Container\StaticContainer; use Piwik\DataArray; @@ -372,9 +374,6 @@ class LogAggregator $query['sql'] = 'SELECT /* ' . $this->dateStart->toString() . ',' . $this->dateEnd->toString() . ' sites ' . implode(',', array_map('intval', $this->sites)) . ' segmenthash ' . $this->getSegment()->getHash(). ' */' . substr($query['sql'], strlen($select)); } - // Log on DEBUG level all SQL archiving queries - $this->logger->debug($query['sql']); - return $query; } @@ -1161,6 +1160,9 @@ class LogAggregator public function getDb() { - return Db::getReader(); + /** @var ArchivingStatus $archivingStatus */ + $archivingStatus = StaticContainer::get(ArchivingStatus::class); + $archivingLock = $archivingStatus->getCurrentArchivingLock(); + return new ArchivingDbAdapter(Db::getReader(), $archivingLock, $this->logger); } } diff --git a/core/DataAccess/Model.php b/core/DataAccess/Model.php index 73e2c46ba9..ab32487f8d 100644 --- a/core/DataAccess/Model.php +++ b/core/DataAccess/Model.php @@ -215,7 +215,7 @@ class Model return $deletedRows; } - public function getArchiveIdAndVisits($numericTable, $idSite, $period, $dateStartIso, $dateEndIso, $doneFlags, $doneFlagValues) + public function getArchiveIdAndVisits($numericTable, $idSite, $period, $dateStartIso, $dateEndIso, $minDatetimeIsoArchiveProcessedUTC, $doneFlags, $doneFlagValues) { $bindSQL = array($idSite, $dateStartIso, @@ -225,6 +225,12 @@ class Model $sqlWhereArchiveName = self::getNameCondition($doneFlags, $doneFlagValues); + $timeStampWhere = ''; + if ($minDatetimeIsoArchiveProcessedUTC) { + $timeStampWhere = " AND ts_archived >= ? "; + $bindSQL[] = $minDatetimeIsoArchiveProcessedUTC; + } + $sqlQuery = "SELECT idarchive, value, name, date1 as startDate FROM $numericTable WHERE idsite = ? AND date1 = ? @@ -233,6 +239,7 @@ class Model AND ( ($sqlWhereArchiveName) OR name = '" . ArchiveSelector::NB_VISITS_RECORD_LOOKED_UP . "' OR name = '" . ArchiveSelector::NB_VISITS_CONVERTED_RECORD_LOOKED_UP . "') + $timeStampWhere ORDER BY idarchive DESC"; $results = Db::fetchAll($sqlQuery, $bindSQL); diff --git a/tests/PHPUnit/Integration/ArchiveProcessor/ArchivingStatusTest.php b/tests/PHPUnit/Integration/ArchiveProcessor/ArchivingStatusTest.php new file mode 100644 index 0000000000..e7a873ffea --- /dev/null +++ b/tests/PHPUnit/Integration/ArchiveProcessor/ArchivingStatusTest.php @@ -0,0 +1,87 @@ +<?php +/** + * Piwik - free/libre analytics platform + * + * @link https://matomo.org + * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later + * + */ + +namespace Piwik\Tests\Integration\ArchiveProcessor; + +use Piwik\ArchiveProcessor\ArchivingStatus; +use Piwik\ArchiveProcessor\Parameters; +use Piwik\Common; +use Piwik\Container\StaticContainer; +use Piwik\Db; +use Piwik\Period\Factory; +use Piwik\Segment; +use Piwik\Site; +use Piwik\Tests\Framework\Fixture; +use Piwik\Tests\Framework\TestCase\IntegrationTestCase; + +class ArchivingStatusTest extends IntegrationTestCase +{ + public function setUp() + { + parent::setUp(); + + Fixture::createWebsite('2010-02-03 00:00:00'); + Fixture::createWebsite('2010-02-03 00:00:00'); + } + + public function test_archiveStartedAndArchiveFinished_workflow() + { + /** @var ArchivingStatus $archivingStatus */ + $archivingStatus = StaticContainer::get(ArchivingStatus::class); + + $params = new Parameters(new Site(1), Factory::build('month', '2012-02-04'), new Segment('', [1])); + $archivingStatus->archiveStarted($params); + + $lock = $archivingStatus->getCurrentArchivingLock(); + $this->assertNotEmpty($lock); + $this->assertTrue($lock->isLocked()); + + $this->assertEquals([ + [ + 'key' => 'Archiving.1.fab0afcc3a068ecd91b14d50f3bc911d.' . getmypid(), + 'expiry_time' => time() + ArchivingStatus::DEFAULT_ARCHIVING_TTL, + ], + ], $this->getLockKeysAndTtls()); + + $archivingStatus->archiveFinished(); + + $this->assertEquals([], $this->getLockKeysAndTtls()); + } + + public function test_getSitesCurrentlyArchiving_returnsAllSitesArchiving() + { + /** @var ArchivingStatus $archivingStatus */ + $archivingStatus = StaticContainer::get(ArchivingStatus::class); + + $params = new Parameters(new Site(1), Factory::build('month', '2012-02-04'), new Segment('', [1])); + $archivingStatus->archiveStarted($params); + + $params = new Parameters(new Site(1), Factory::build('month', '2012-02-04'), new Segment('browserCode==ff', [1])); + $archivingStatus->archiveStarted($params); + + $params = new Parameters(new Site(2), Factory::build('month', '2012-02-04'), new Segment('', [1])); + $archivingStatus->archiveStarted($params); + + $this->assertEquals([ + 1, 2, + ], $archivingStatus->getSitesCurrentlyArchiving()); + + $archivingStatus->archiveFinished(); + $archivingStatus->archiveFinished(); + + $this->assertEquals([ + 1, + ], $archivingStatus->getSitesCurrentlyArchiving()); + } + + private function getLockKeysAndTtls() + { + return Db::fetchAll("SELECT `key`, expiry_time FROM `" . Common::prefixTable('locks') . '`'); + } +}
\ No newline at end of file diff --git a/tests/PHPUnit/Integration/ArchiveProcessor/LoaderTest.php b/tests/PHPUnit/Integration/ArchiveProcessor/LoaderTest.php new file mode 100644 index 0000000000..e99d495931 --- /dev/null +++ b/tests/PHPUnit/Integration/ArchiveProcessor/LoaderTest.php @@ -0,0 +1,122 @@ +<?php +/** + * Piwik - free/libre analytics platform + * + * @link https://matomo.org + * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later + * + */ + +namespace Piwik\Tests\Integration\ArchiveProcessor; + + +use Piwik\ArchiveProcessor\Parameters; +use Piwik\ArchiveProcessor\Loader; +use Piwik\Common; +use Piwik\Config; +use Piwik\DataAccess\ArchiveTableCreator; +use Piwik\DataAccess\ArchiveWriter; +use Piwik\Date; +use Piwik\Db; +use Piwik\Period\Factory; +use Piwik\Segment; +use Piwik\Site; +use Piwik\Tests\Framework\Fixture; +use Piwik\Tests\Framework\TestCase\IntegrationTestCase; + +class LoaderTest extends IntegrationTestCase +{ + protected static function beforeTableDataCached() + { + parent::beforeTableDataCached(); + + Fixture::createWebsite('2012-02-03 00:00:00'); + } + + public function test_loadExistingArchiveIdFromDb_returnsFalsesIfNoArchiveFound() + { + $params = new Parameters(new Site(1), Factory::build('day', '2015-03-03'), new Segment('', [1])); + $loader = new Loader($params); + + $archiveInfo = $loader->loadExistingArchiveIdFromDb(); + + $this->assertEquals([false, false, false], $archiveInfo); + } + + /** + * @dataProvider getTestDataForLoadExistingArchiveIdFromDbDebugConfig + */ + public function test_loadExistingArchiveIdFromDb_returnsFalsesPeriodIsForcedToArchive($periodType, $configSetting) + { + $date = $periodType == 'range' ? '2015-03-03,2015-03-04' : '2015-03-03'; + $params = new Parameters(new Site(1), Factory::build($periodType, $date), new Segment('', [1])); + $this->insertArchive($params); + + $loader = new Loader($params); + + $archiveInfo = $loader->loadExistingArchiveIdFromDb(); + $this->assertNotEquals([false, false, false], $archiveInfo); + + Config::getInstance()->Debug[$configSetting] = 1; + + $archiveInfo = $loader->loadExistingArchiveIdFromDb(); + $this->assertEquals([false, false, false], $archiveInfo); + } + + public function getTestDataForLoadExistingArchiveIdFromDbDebugConfig() + { + return [ + ['day', 'always_archive_data_day'], + ['week', 'always_archive_data_period'], + ['month', 'always_archive_data_period'], + ['year', 'always_archive_data_period'], + ['range', 'always_archive_data_range'], + ]; + } + + public function test_loadExistingArchiveIdFromDb_returnsArchiveIfArchiveInThePast() + { + $params = new Parameters(new Site(1), Factory::build('month', '2015-03-03'), new Segment('', [1])); + $this->insertArchive($params); + + $loader = new Loader($params); + + $archiveInfo = $loader->loadExistingArchiveIdFromDb(); + $this->assertEquals(['1', '10', '0'], $archiveInfo); + } + + public function test_loadExistingArchiveIdFromDb_returnsArchiveIfForACurrentPeriod_AndNewEnough() + { + $params = new Parameters(new Site(1), Factory::build('day', 'now'), new Segment('', [1])); + $this->insertArchive($params, $tsArchived = time() - 1); + + $loader = new Loader($params); + + $archiveInfo = $loader->loadExistingArchiveIdFromDb(); + $this->assertEquals(['1', '10', '0'], $archiveInfo); + } + + public function test_loadExistingArchiveIdFromDb_returnsNoArchiveIfForACurrentPeriod_AndNoneAreNewEnough() + { + $params = new Parameters(new Site(1), Factory::build('month', 'now'), new Segment('', [1])); + $this->insertArchive($params, $tsArchived = time() - 3 * 3600); + + $loader = new Loader($params); + + $archiveInfo = $loader->loadExistingArchiveIdFromDb(); + $this->assertEquals([false, false, false], $archiveInfo); + } + + private function insertArchive(Parameters $params, $tsArchived = null, $visits = 10) + { + $archiveWriter = new ArchiveWriter($params); + $archiveWriter->initNewArchive(); + $archiveWriter->insertRecord('nb_visits', $visits); + $archiveWriter->finalizeArchive(); + + if ($tsArchived) { + Db::query("UPDATE " . ArchiveTableCreator::getNumericTable($params->getPeriod()->getDateStart()) . " SET ts_archived = ?", + [Date::factory($tsArchived)->getDatetime()]); + } + } +}
\ No newline at end of file diff --git a/tests/PHPUnit/Integration/DataAccess/ArchiveInvalidatorTest.php b/tests/PHPUnit/Integration/DataAccess/ArchiveInvalidatorTest.php index 530bd6e112..36365b7c49 100644 --- a/tests/PHPUnit/Integration/DataAccess/ArchiveInvalidatorTest.php +++ b/tests/PHPUnit/Integration/DataAccess/ArchiveInvalidatorTest.php @@ -8,7 +8,10 @@ namespace Piwik\Tests\Integration\DataAccess; +use Piwik\ArchiveProcessor\ArchivingStatus; +use Piwik\ArchiveProcessor\Parameters; use Piwik\ArchiveProcessor\Rules; +use Piwik\Container\StaticContainer; use Piwik\CronArchive\SitesToReprocessDistributedList; use Piwik\DataAccess\ArchiveTableCreator; use Piwik\DataAccess\ArchiveWriter; @@ -16,9 +19,11 @@ use Piwik\DataAccess\Model; use Piwik\Date; use Piwik\Db; use Piwik\Option; +use Piwik\Period\Factory; use Piwik\Piwik; use Piwik\Plugins\CoreAdminHome\Tasks\ArchivesToPurgeDistributedList; use Piwik\Plugins\PrivacyManager\PrivacyManager; +use Piwik\Site; use Piwik\Tests\Framework\Fixture; use Piwik\Tests\Framework\TestCase\IntegrationTestCase; use Piwik\Archive\ArchiveInvalidator; @@ -71,7 +76,7 @@ class ArchiveInvalidatorTest extends IntegrationTestCase { parent::setUp(); - $this->invalidator = new ArchiveInvalidator(new Model()); + $this->invalidator = new ArchiveInvalidator(new Model(), StaticContainer::get(ArchivingStatus::class)); } public function test_rememberToInvalidateArchivedReportsLater_shouldCreateAnEntryInCaseThereIsNoneYet() @@ -196,6 +201,39 @@ class ArchiveInvalidatorTest extends IntegrationTestCase $this->assertSame($expected, $reports); } + public function test_markArchivesAsInvalidated_shouldSkipArchivingForInProgressSites() + { + /** @var ArchivingStatus $archivingStatus */ + $archivingStatus = StaticContainer::get(ArchivingStatus::class); + + $params = new Parameters(new Site(7), Factory::build('month', '2012-02-03'), new Segment('', [2])); + $archivingStatus->archiveStarted($params); + + $params = new Parameters(new Site(5), Factory::build('day', '2012-02-03'), new Segment('', [7])); + $archivingStatus->archiveStarted($params); + + $this->rememberReportsForManySitesAndDates(); + + $idSites = array(2, 10, 7, 5); + $dates = array( + Date::factory('2014-04-05'), + Date::factory('2014-04-08'), + Date::factory('2014-05-05'), + ); + + $this->invalidator->markArchivesAsInvalidated($idSites, $dates, 'week'); + $reports = $this->invalidator->getRememberedArchivedReportsThatShouldBeInvalidated(); + + $expected = array( + '2014-04-05' => [1,4,7], + '2014-04-06' => [3], + '2014-05-05' => [5], + '2014-04-08' => [7], + '2014-05-08' => [7], + ); + $this->assertSame($expected, $reports); + } + private function rememberReport($idSite, $date) { $date = Date::factory($date); diff --git a/tests/PHPUnit/Integration/DataAccess/LogAggregatorTest.php b/tests/PHPUnit/Integration/DataAccess/LogAggregatorTest.php index 8b7ac6024c..968b98e73c 100644 --- a/tests/PHPUnit/Integration/DataAccess/LogAggregatorTest.php +++ b/tests/PHPUnit/Integration/DataAccess/LogAggregatorTest.php @@ -8,13 +8,18 @@ namespace Piwik\Tests\Integration\DataAccess; +use Piwik\ArchiveProcessor\ArchivingStatus; use Piwik\ArchiveProcessor\Parameters; +use Piwik\Common; +use Piwik\Container\StaticContainer; use Piwik\DataAccess\LogAggregator; use Piwik\Date; +use Piwik\Db; use Piwik\Period; use Piwik\Segment; use Piwik\Site; use Piwik\Tests\Fixtures\OneVisitorTwoVisits; +use Piwik\Tests\Framework\Fixture; use Piwik\Tests\Framework\TestCase\IntegrationTestCase; /** @@ -135,6 +140,37 @@ class LogAggregatorTest extends IntegrationTestCase ]; $this->assertEquals($expected, $result); } + + public function test_logAggregatorUpdatesArchiveStatusExpireTime() + { + $t = Fixture::getTracker(self::$fixture->idSite, '2010-03-06 14:22:33'); + $t->setUrl('http://example.com/here/we/go'); + $t->doTrackPageView('here we go'); + + $params = new Parameters(new Site(self::$fixture->idSite), Period\Factory::build('day', self::$fixture->dateTime), new Segment('', [self::$fixture->idSite])); + + $archiveStatus = StaticContainer::get(ArchivingStatus::class); + $archiveStatus->archiveStarted($params); + + $locks = $this->getAllLocks(); + $this->assertCount(1, $locks); + $expireTime = $locks[0]['expiry_time']; + + sleep(1); + + $this->logAggregator->queryVisitsByDimension(['visit_total_time']); + + $locks = $this->getAllLocks(); + $this->assertCount(1, $locks); + $expireTimeNew = $locks[0]['expiry_time']; + + $this->assertGreaterThan($expireTime, $expireTimeNew); + } + + private function getAllLocks() + { + return Db::fetchAll("SELECT `key`, expiry_time FROM `" . Common::prefixTable('locks') . '`'); + } } LogAggregatorTest::$fixture = new OneVisitorTwoVisits(); |