Lock when archiving and avoid invalidating sites that have archiving in progress (#15272)

* Use a lock when archiving and do not invalidate when archiving is in progress.

* Add and fix tests + modify workflow.

* forgot to add file and remove TODO

* Remove use of argument.

* Add back min archive time processed code and start on tests for it.

* Finish new LoaderTest.

* Fix new tests.

15 files changed, 580 insertions, 25 deletions

diff --git a/config/global.php b/config/global.php
index 9d8666efd0..5499b0920a 100644
--- a/config/global.php
+++ b/config/global.php
@@ -217,5 +217,5 @@ return array(
 
     \Piwik\CronArchive\Performance\Logger::class => DI\object()->constructorParameter('logger', DI\get('archiving.performance.logger')),
 
-    'Piwik\Concurrency\LockBackend' => DI\object(\Piwik\Concurrency\LockBackend\MySqlLockBackend::class)
+    \Piwik\Concurrency\LockBackend::class => \DI\get(\Piwik\Concurrency\LockBackend\MySqlLockBackend::class),
 );
diff --git a/core/Archive/ArchiveInvalidator.php b/core/Archive/ArchiveInvalidator.php
index 4504cd5446..649f1cff02 100644
--- a/core/Archive/ArchiveInvalidator.php
+++ b/core/Archive/ArchiveInvalidator.php
@@ -10,6 +10,7 @@
 namespace Piwik\Archive;
 
 use Piwik\Archive\ArchiveInvalidator\InvalidationResult;
+use Piwik\ArchiveProcessor\ArchivingStatus;
 use Piwik\CronArchive\SitesToReprocessDistributedList;
 use Piwik\DataAccess\ArchiveTableCreator;
 use Piwik\DataAccess\Model;
@@ -54,9 +55,15 @@ class ArchiveInvalidator
      */
     private $model;
 
-    public function __construct(Model $model)
+    /**
+     * @var ArchivingStatus
+     */
+    private $archivingStatus;
+
+    public function __construct(Model $model, ArchivingStatus $archivingStatus)
     {
         $this->model = $model;
+        $this->archivingStatus = $archivingStatus;
     }
 
     public function rememberToInvalidateArchivedReportsLater($idSite, Date $date)
@@ -207,6 +214,7 @@ class ArchiveInvalidator
         }
 
         $periodDates = $this->getUniqueDates($periodDates);
+
         $this->markArchivesInvalidated($idSites, $periodDates, $segment);
 
         $yearMonths = array_keys($periodDates);
diff --git a/core/ArchiveProcessor/ArchivingStatus.php b/core/ArchiveProcessor/ArchivingStatus.php
new file mode 100644
index 0000000000..898786db8a
--- /dev/null
+++ b/core/ArchiveProcessor/ArchivingStatus.php
@@ -0,0 +1,107 @@
+<?php
+/**
+ * Piwik - free/libre analytics platform
+ *
+ * @link https://matomo.org
+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
+ *
+ */
+
+namespace Piwik\ArchiveProcessor;
+
+use Piwik\Concurrency\Lock;
+use Piwik\Concurrency\LockBackend;
+use Piwik\Container\StaticContainer;
+use Piwik\SettingsPiwik;
+
+class ArchivingStatus
+{
+    const LOCK_KEY_PREFIX = 'Archiving';
+    const DEFAULT_ARCHIVING_TTL = 7200; // 2 hours
+
+    /**
+     * @var LockBackend
+     */
+    private $lockBackend;
+
+    /**
+     * @var int
+     */
+    private $archivingTTLSecs;
+
+    /**
+     * @var Lock[]
+     */
+    private $lockStack = [];
+
+    public function __construct(LockBackend $lockBackend, $archivingTTLSecs = self::DEFAULT_ARCHIVING_TTL)
+    {
+        $this->lockBackend = $lockBackend;
+        $this->archivingTTLSecs = $archivingTTLSecs;
+    }
+
+    public function archiveStarted(Parameters $params)
+    {
+        $lock = $this->makeArchivingLock($params);
+        $lock->acquireLock($this->getInstanceProcessId(), $this->archivingTTLSecs);
+        array_push($this->lockStack, $lock);
+    }
+
+    public function archiveFinished()
+    {
+        $lock = array_pop($this->lockStack);
+        $lock->unlock();
+    }
+
+    public function getCurrentArchivingLock()
+    {
+        if (empty($this->lockStack)) {
+            return null;
+        }
+        return end($this->lockStack);
+    }
+
+    public function getSitesCurrentlyArchiving()
+    {
+        $lockMeta = new Lock($this->lockBackend, self::LOCK_KEY_PREFIX . '.');
+        $acquiredLocks = $lockMeta->getAllAcquiredLockKeys();
+
+        $sitesCurrentlyArchiving = [];
+        foreach ($acquiredLocks as $lockKey) {
+            $parts = explode('.', $lockKey);
+            if (!isset($parts[1])) {
+                continue;
+            }
+            $sitesCurrentlyArchiving[] = (int) $parts[1];
+        }
+        $sitesCurrentlyArchiving = array_unique($sitesCurrentlyArchiving);
+        $sitesCurrentlyArchiving = array_values($sitesCurrentlyArchiving);
+
+        return $sitesCurrentlyArchiving;
+    }
+
+    /**
+     * @return Lock
+     */
+    private function makeArchivingLock(Parameters $params)
+    {
+        $doneFlag = Rules::getDoneStringFlagFor([$params->getSite()->getId()], $params->getSegment(),
+            $params->getPeriod()->getLabel(), $params->getRequestedPlugin());
+
+        $lockKeyParts = [
+            self::LOCK_KEY_PREFIX,
+            $params->getSite()->getId(),
+
+            // md5 to keep it within the 70 char limit in the table
+            md5($params->getPeriod()->getId() . $params->getPeriod()->getRangeString() . $doneFlag),
+        ];
+
+        $lockKeyPrefix = implode('.', $lockKeyParts);
+        return new Lock(StaticContainer::get(LockBackend::class), $lockKeyPrefix, $this->archivingTTLSecs);
+    }
+
+    private function getInstanceProcessId()
+    {
+        return SettingsPiwik::getPiwikInstanceId() . '.' . getmypid();
+    }
+}
\ No newline at end of file
diff --git a/core/ArchiveProcessor/Loader.php b/core/ArchiveProcessor/Loader.php
index f08e63003b..4564146bd9 100644
--- a/core/ArchiveProcessor/Loader.php
+++ b/core/ArchiveProcessor/Loader.php
@@ -8,15 +8,12 @@
  */
 namespace Piwik\ArchiveProcessor;
 
-use Piwik\Archive;
 use Piwik\Cache;
-use Piwik\CacheId;
-use Piwik\Common;
 use Piwik\Config;
+use Piwik\Container\StaticContainer;
 use Piwik\Context;
 use Piwik\DataAccess\ArchiveSelector;
 use Piwik\Date;
-use Piwik\Period;
 use Piwik\Piwik;
 
 /**
@@ -73,12 +70,21 @@ class Loader
             return $idArchive;
         }
 
-        list($visits, $visitsConverted) = $this->prepareCoreMetricsArchive($visits, $visitsConverted);
-        list($idArchive, $visits) = $this->prepareAllPluginsArchive($visits, $visitsConverted);
+        /** @var ArchivingStatus $archivingStatus */
+        $archivingStatus = StaticContainer::get(ArchivingStatus::class);
+        $archivingStatus->archiveStarted($this->params);
+
+        try {
+            list($visits, $visitsConverted) = $this->prepareCoreMetricsArchive($visits, $visitsConverted);
+            list($idArchive, $visits) = $this->prepareAllPluginsArchive($visits, $visitsConverted);
+        } finally {
+            $archivingStatus->archiveFinished();
+        }
 
         if ($this->isThereSomeVisits($visits) || PluginsArchiver::doesAnyPluginArchiveWithoutVisits()) {
             return $idArchive;
         }
+
         return false;
     }
 
@@ -158,9 +164,11 @@ class Loader
      * Returns the idArchive if the archive is available in the database for the requested plugin.
      * Returns false if the archive needs to be processed.
      *
+     * (public for tests)
+     *
      * @return array
      */
-    protected function loadExistingArchiveIdFromDb()
+    public function loadExistingArchiveIdFromDb()
     {
         $noArchiveFound = array(false, false, false);
 
@@ -168,7 +176,8 @@ class Loader
             return $noArchiveFound;
         }
 
-        $idAndVisits = ArchiveSelector::getArchiveIdAndVisits($this->params);
+        $minDatetimeArchiveProcessedUTC = $this->getMinTimeArchiveProcessed();
+        $idAndVisits = ArchiveSelector::getArchiveIdAndVisits($this->params, $minDatetimeArchiveProcessedUTC);
 
         if (!$idAndVisits) {
             return $noArchiveFound;
@@ -177,6 +186,26 @@ class Loader
         return $idAndVisits;
     }
 
+    /**
+     * Returns the minimum archive processed datetime to look at. Only public for tests.
+     *
+     * @return int|bool  Datetime timestamp, or false if must look at any archive available
+     */
+    protected function getMinTimeArchiveProcessed()
+    {
+        $endDateTimestamp = self::determineIfArchivePermanent($this->params->getDateEnd());
+        if ($endDateTimestamp) {
+            // past archive
+            return $endDateTimestamp;
+        }
+        $dateStart = $this->params->getDateStart();
+        $period    = $this->params->getPeriod();
+        $segment   = $this->params->getSegment();
+        $site      = $this->params->getSite();
+        // in-progress archive
+        return Rules::getMinTimeProcessedForInProgressArchive($dateStart, $period, $segment, $site);
+    }
+
     protected static function determineIfArchivePermanent(Date $dateEnd)
     {
         $now = time();
diff --git a/core/ArchiveProcessor/Rules.php b/core/ArchiveProcessor/Rules.php
index 90d8d46a23..f054ce5d74 100644
--- a/core/ArchiveProcessor/Rules.php
+++ b/core/ArchiveProcessor/Rules.php
@@ -112,7 +112,7 @@ class Rules
         return $doneFlags;
     }
 
-    public static function getMinTimeProcessedForTemporaryArchive(
+    public static function getMinTimeProcessedForInProgressArchive(
         Date $dateStart, \Piwik\Period $period, Segment $segment, Site $site)
     {
         $todayArchiveTimeToLive = self::getPeriodArchiveTimeToLiveDefault($period->getLabel());
diff --git a/core/Concurrency/Lock.php b/core/Concurrency/Lock.php
index e3200310bd..f63bec640a 100644
--- a/core/Concurrency/Lock.php
+++ b/core/Concurrency/Lock.php
@@ -23,12 +23,19 @@ class Lock
 
     private $lockKey   = null;
     private $lockValue = null;
+    private $defaultTtl = null;
 
-    public function __construct(LockBackend $backend, $lockKeyStart)
+    public function __construct(LockBackend $backend, $lockKeyStart, $defaultTtl = null)
     {
         $this->backend = $backend;
         $this->lockKeyStart = $lockKeyStart;
         $this->lockKey = $this->lockKeyStart;
+        $this->defaultTtl = $defaultTtl;
+    }
+
+    public function reexpireLock()
+    {
+        $this->expireLock($this->defaultTtl);
     }
 
     public function getNumberOfAcquiredLocks()
diff --git a/core/Concurrency/LockBackend.php b/core/Concurrency/LockBackend.php
index 7f67cbafa5..4ca3467657 100644
--- a/core/Concurrency/LockBackend.php
+++ b/core/Concurrency/LockBackend.php
@@ -12,15 +12,15 @@ namespace Piwik\Concurrency;
 interface LockBackend
 {
     /**
-     * TODO
+     * Returns lock keys matching a pattern.
      *
      * @param $pattern
-     * @return mixed
+     * @return string[]
      */
     public function getKeysMatchingPattern($pattern);
 
     /**
-     * TODO
+     * Set a key value if the key is not already set.
      *
      * @param $lockKey
      * @param $lockValue
@@ -30,7 +30,7 @@ interface LockBackend
     public function setIfNotExists($lockKey, $lockValue, $ttlInSeconds);
 
     /**
-     * TODO
+     * Get the lock value for a key if any.
      *
      * @param $lockKey
      * @return mixed
@@ -38,7 +38,7 @@ interface LockBackend
     public function get($lockKey);
 
     /**
-     * TODO
+     * Delete the lock with key = $lockKey if the lock has the given value.
      *
      * @param $lockKey
      * @param $lockValue
@@ -47,7 +47,7 @@ interface LockBackend
     public function deleteIfKeyHasValue($lockKey, $lockValue);
 
     /**
-     * TODO
+     * Update expiration for a lock if the lock with the specified key has the given value.
      *
      * @param $lockKey
      * @param $lockValue
diff --git a/core/DataAccess/ArchiveSelector.php b/core/DataAccess/ArchiveSelector.php
index 5e73f1cb8f..d8063c33c0 100644
--- a/core/DataAccess/ArchiveSelector.php
+++ b/core/DataAccess/ArchiveSelector.php
@@ -61,6 +61,11 @@ class ArchiveSelector
 
         $numericTable = ArchiveTableCreator::getNumericTable($dateStart);
 
+        $minDatetimeIsoArchiveProcessedUTC = null;
+        if ($minDatetimeArchiveProcessedUTC) {
+            $minDatetimeIsoArchiveProcessedUTC = Date::factory($minDatetimeArchiveProcessedUTC)->getDatetime();
+        }
+
         $requestedPlugin = $params->getRequestedPlugin();
         $segment         = $params->getSegment();
         $plugins = array("VisitsSummary", $requestedPlugin);
@@ -68,7 +73,7 @@ class ArchiveSelector
         $doneFlags      = Rules::getDoneFlags($plugins, $segment);
         $doneFlagValues = Rules::getSelectableDoneFlagValues();
 
-        $results = self::getModel()->getArchiveIdAndVisits($numericTable, $idSite, $period, $dateStartIso, $dateEndIso, $doneFlags, $doneFlagValues);
+        $results = self::getModel()->getArchiveIdAndVisits($numericTable, $idSite, $period, $dateStartIso, $dateEndIso, $minDatetimeIsoArchiveProcessedUTC, $doneFlags, $doneFlagValues);
 
         if (empty($results)) {
             return false;
diff --git a/core/DataAccess/ArchivingDbAdapter.php b/core/DataAccess/ArchivingDbAdapter.php
new file mode 100644
index 0000000000..e0ebfbe339
--- /dev/null
+++ b/core/DataAccess/ArchivingDbAdapter.php
@@ -0,0 +1,107 @@
+<?php
+/**
+ * Piwik - free/libre analytics platform
+ *
+ * @link https://matomo.org
+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
+ *
+ */
+
+namespace Piwik\DataAccess;
+
+use Piwik\Concurrency\Lock;
+use Piwik\Db\AdapterInterface;
+use Psr\Log\LoggerInterface;
+
+class ArchivingDbAdapter
+{
+    /**
+     * @var AdapterInterface|\Zend_Db_Adapter_Abstract
+     */
+    private $wrapped;
+
+    /**
+     * @var Lock
+     */
+    private $archivingLock;
+
+    /**
+     * @var LoggerInterface
+     */
+    private $logger;
+
+    public function __construct($wrapped, Lock $archivingLock = null, LoggerInterface $logger = null)
+    {
+        $this->wrapped = $wrapped;
+        $this->archivingLock = $archivingLock;
+        $this->logger = $logger;
+    }
+
+    public function __call($name, $arguments)
+    {
+        return call_user_func_array([$this->wrapped, $name], $arguments);
+    }
+
+    public function exec($sql)
+    {
+        $this->reexpireLock();
+        $this->logSql($sql);
+
+        return call_user_func_array([$this->wrapped, __FUNCTION__], func_get_args());
+    }
+
+    public function query($sql)
+    {
+        $this->reexpireLock();
+        $this->logSql($sql);
+
+        return call_user_func_array([$this->wrapped, __FUNCTION__], func_get_args());
+    }
+
+    public function fetchAll($sql)
+    {
+        $this->reexpireLock();
+        $this->logSql($sql);
+
+        return call_user_func_array([$this->wrapped, __FUNCTION__], func_get_args());
+    }
+
+    public function fetchRow($sql)
+    {
+        $this->reexpireLock();
+        $this->logSql($sql);
+
+        return call_user_func_array([$this->wrapped, __FUNCTION__], func_get_args());
+    }
+
+    public function fetchOne($sql)
+    {
+        $this->reexpireLock();
+        $this->logSql($sql);
+
+        return call_user_func_array([$this->wrapped, __FUNCTION__], func_get_args());
+    }
+
+    public function fetchAssoc($sql)
+    {
+        $this->reexpireLock();
+        $this->logSql($sql);
+
+        return call_user_func_array([$this->wrapped, __FUNCTION__], func_get_args());
+    }
+
+    private function logSql($sql)
+    {
+        // Log on DEBUG level all SQL archiving queries
+        if ($this->logger) {
+            $this->logger->debug($sql);
+        }
+    }
+
+    private function reexpireLock()
+    {
+        if ($this->archivingLock) {
+            $this->archivingLock->reexpireLock();
+        }
+    }
+}
\ No newline at end of file
diff --git a/core/DataAccess/LogAggregator.php b/core/DataAccess/LogAggregator.php
index 4c169e30c7..0fef1ba31c 100644
--- a/core/DataAccess/LogAggregator.php
+++ b/core/DataAccess/LogAggregator.php
@@ -8,8 +8,10 @@
  */
 namespace Piwik\DataAccess;
 
+use Piwik\ArchiveProcessor\ArchivingStatus;
 use Piwik\ArchiveProcessor\Parameters;
 use Piwik\Common;
+use Piwik\Concurrency\Lock;
 use Piwik\Config;
 use Piwik\Container\StaticContainer;
 use Piwik\DataArray;
@@ -372,9 +374,6 @@ class LogAggregator
             $query['sql'] = 'SELECT /* ' . $this->dateStart->toString() . ',' . $this->dateEnd->toString() . ' sites ' . implode(',', array_map('intval', $this->sites)) . ' segmenthash ' . $this->getSegment()->getHash(). ' */' . substr($query['sql'], strlen($select));
         }
  
-    	// Log on DEBUG level all SQL archiving queries
-        $this->logger->debug($query['sql']);
-
         return $query;
     }
 
@@ -1161,6 +1160,9 @@ class LogAggregator
 
     public function getDb()
     {
-        return Db::getReader();
+        /** @var ArchivingStatus $archivingStatus */
+        $archivingStatus = StaticContainer::get(ArchivingStatus::class);
+        $archivingLock = $archivingStatus->getCurrentArchivingLock();
+        return new ArchivingDbAdapter(Db::getReader(), $archivingLock, $this->logger);
     }
 }
diff --git a/core/DataAccess/Model.php b/core/DataAccess/Model.php
index 73e2c46ba9..ab32487f8d 100644
--- a/core/DataAccess/Model.php
+++ b/core/DataAccess/Model.php
@@ -215,7 +215,7 @@ class Model
         return $deletedRows;
     }
 
-    public function getArchiveIdAndVisits($numericTable, $idSite, $period, $dateStartIso, $dateEndIso, $doneFlags, $doneFlagValues)
+    public function getArchiveIdAndVisits($numericTable, $idSite, $period, $dateStartIso, $dateEndIso, $minDatetimeIsoArchiveProcessedUTC, $doneFlags, $doneFlagValues)
     {
         $bindSQL = array($idSite,
             $dateStartIso,
@@ -225,6 +225,12 @@ class Model
 
         $sqlWhereArchiveName = self::getNameCondition($doneFlags, $doneFlagValues);
 
+        $timeStampWhere = '';
+        if ($minDatetimeIsoArchiveProcessedUTC) {
+            $timeStampWhere = " AND ts_archived >= ? ";
+            $bindSQL[]      = $minDatetimeIsoArchiveProcessedUTC;
+        }
+
         $sqlQuery = "SELECT idarchive, value, name, date1 as startDate FROM $numericTable
                      WHERE idsite = ?
                          AND date1 = ?
@@ -233,6 +239,7 @@ class Model
                          AND ( ($sqlWhereArchiveName)
                                OR name = '" . ArchiveSelector::NB_VISITS_RECORD_LOOKED_UP . "'
                                OR name = '" . ArchiveSelector::NB_VISITS_CONVERTED_RECORD_LOOKED_UP . "')
+                         $timeStampWhere
                      ORDER BY idarchive DESC";
         $results = Db::fetchAll($sqlQuery, $bindSQL);
 
diff --git a/tests/PHPUnit/Integration/ArchiveProcessor/ArchivingStatusTest.php b/tests/PHPUnit/Integration/ArchiveProcessor/ArchivingStatusTest.php
new file mode 100644
index 0000000000..e7a873ffea
--- /dev/null
+++ b/tests/PHPUnit/Integration/ArchiveProcessor/ArchivingStatusTest.php
@@ -0,0 +1,87 @@
+<?php
+/**
+ * Piwik - free/libre analytics platform
+ *
+ * @link https://matomo.org
+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
+ *
+ */
+
+namespace Piwik\Tests\Integration\ArchiveProcessor;
+
+use Piwik\ArchiveProcessor\ArchivingStatus;
+use Piwik\ArchiveProcessor\Parameters;
+use Piwik\Common;
+use Piwik\Container\StaticContainer;
+use Piwik\Db;
+use Piwik\Period\Factory;
+use Piwik\Segment;
+use Piwik\Site;
+use Piwik\Tests\Framework\Fixture;
+use Piwik\Tests\Framework\TestCase\IntegrationTestCase;
+
+class ArchivingStatusTest extends IntegrationTestCase
+{
+    public function setUp()
+    {
+        parent::setUp();
+
+        Fixture::createWebsite('2010-02-03 00:00:00');
+        Fixture::createWebsite('2010-02-03 00:00:00');
+    }
+
+    public function test_archiveStartedAndArchiveFinished_workflow()
+    {
+        /** @var ArchivingStatus $archivingStatus */
+        $archivingStatus = StaticContainer::get(ArchivingStatus::class);
+
+        $params = new Parameters(new Site(1), Factory::build('month', '2012-02-04'), new Segment('', [1]));
+        $archivingStatus->archiveStarted($params);
+
+        $lock = $archivingStatus->getCurrentArchivingLock();
+        $this->assertNotEmpty($lock);
+        $this->assertTrue($lock->isLocked());
+
+        $this->assertEquals([
+            [
+                'key' => 'Archiving.1.fab0afcc3a068ecd91b14d50f3bc911d.' . getmypid(),
+                'expiry_time' => time() + ArchivingStatus::DEFAULT_ARCHIVING_TTL,
+            ],
+        ], $this->getLockKeysAndTtls());
+
+        $archivingStatus->archiveFinished();
+
+        $this->assertEquals([], $this->getLockKeysAndTtls());
+    }
+
+    public function test_getSitesCurrentlyArchiving_returnsAllSitesArchiving()
+    {
+        /** @var ArchivingStatus $archivingStatus */
+        $archivingStatus = StaticContainer::get(ArchivingStatus::class);
+
+        $params = new Parameters(new Site(1), Factory::build('month', '2012-02-04'), new Segment('', [1]));
+        $archivingStatus->archiveStarted($params);
+
+        $params = new Parameters(new Site(1), Factory::build('month', '2012-02-04'), new Segment('browserCode==ff', [1]));
+        $archivingStatus->archiveStarted($params);
+
+        $params = new Parameters(new Site(2), Factory::build('month', '2012-02-04'), new Segment('', [1]));
+        $archivingStatus->archiveStarted($params);
+
+        $this->assertEquals([
+            1, 2,
+        ], $archivingStatus->getSitesCurrentlyArchiving());
+
+        $archivingStatus->archiveFinished();
+        $archivingStatus->archiveFinished();
+
+        $this->assertEquals([
+            1,
+        ], $archivingStatus->getSitesCurrentlyArchiving());
+    }
+
+    private function getLockKeysAndTtls()
+    {
+        return Db::fetchAll("SELECT `key`, expiry_time FROM `" . Common::prefixTable('locks') . '`');
+    }
+}
\ No newline at end of file
diff --git a/tests/PHPUnit/Integration/ArchiveProcessor/LoaderTest.php b/tests/PHPUnit/Integration/ArchiveProcessor/LoaderTest.php
new file mode 100644
index 0000000000..e99d495931
--- /dev/null
+++ b/tests/PHPUnit/Integration/ArchiveProcessor/LoaderTest.php
@@ -0,0 +1,122 @@
+<?php
+/**
+ * Piwik - free/libre analytics platform
+ *
+ * @link https://matomo.org
+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
+ *
+ */
+
+namespace Piwik\Tests\Integration\ArchiveProcessor;
+
+
+use Piwik\ArchiveProcessor\Parameters;
+use Piwik\ArchiveProcessor\Loader;
+use Piwik\Common;
+use Piwik\Config;
+use Piwik\DataAccess\ArchiveTableCreator;
+use Piwik\DataAccess\ArchiveWriter;
+use Piwik\Date;
+use Piwik\Db;
+use Piwik\Period\Factory;
+use Piwik\Segment;
+use Piwik\Site;
+use Piwik\Tests\Framework\Fixture;
+use Piwik\Tests\Framework\TestCase\IntegrationTestCase;
+
+class LoaderTest extends IntegrationTestCase
+{
+    protected static function beforeTableDataCached()
+    {
+        parent::beforeTableDataCached();
+
+        Fixture::createWebsite('2012-02-03 00:00:00');
+    }
+
+    public function test_loadExistingArchiveIdFromDb_returnsFalsesIfNoArchiveFound()
+    {
+        $params = new Parameters(new Site(1), Factory::build('day', '2015-03-03'), new Segment('', [1]));
+        $loader = new Loader($params);
+
+        $archiveInfo = $loader->loadExistingArchiveIdFromDb();
+
+        $this->assertEquals([false, false, false], $archiveInfo);
+    }
+
+    /**
+     * @dataProvider getTestDataForLoadExistingArchiveIdFromDbDebugConfig
+     */
+    public function test_loadExistingArchiveIdFromDb_returnsFalsesPeriodIsForcedToArchive($periodType, $configSetting)
+    {
+        $date = $periodType == 'range' ? '2015-03-03,2015-03-04' : '2015-03-03';
+        $params = new Parameters(new Site(1), Factory::build($periodType, $date), new Segment('', [1]));
+        $this->insertArchive($params);
+
+        $loader = new Loader($params);
+
+        $archiveInfo = $loader->loadExistingArchiveIdFromDb();
+        $this->assertNotEquals([false, false, false], $archiveInfo);
+
+        Config::getInstance()->Debug[$configSetting] = 1;
+
+        $archiveInfo = $loader->loadExistingArchiveIdFromDb();
+        $this->assertEquals([false, false, false], $archiveInfo);
+    }
+
+    public function getTestDataForLoadExistingArchiveIdFromDbDebugConfig()
+    {
+        return [
+            ['day', 'always_archive_data_day'],
+            ['week', 'always_archive_data_period'],
+            ['month', 'always_archive_data_period'],
+            ['year', 'always_archive_data_period'],
+            ['range', 'always_archive_data_range'],
+        ];
+    }
+
+    public function test_loadExistingArchiveIdFromDb_returnsArchiveIfArchiveInThePast()
+    {
+        $params = new Parameters(new Site(1), Factory::build('month', '2015-03-03'), new Segment('', [1]));
+        $this->insertArchive($params);
+
+        $loader = new Loader($params);
+
+        $archiveInfo = $loader->loadExistingArchiveIdFromDb();
+        $this->assertEquals(['1', '10', '0'], $archiveInfo);
+    }
+
+    public function test_loadExistingArchiveIdFromDb_returnsArchiveIfForACurrentPeriod_AndNewEnough()
+    {
+        $params = new Parameters(new Site(1), Factory::build('day', 'now'), new Segment('', [1]));
+        $this->insertArchive($params, $tsArchived = time() - 1);
+
+        $loader = new Loader($params);
+
+        $archiveInfo = $loader->loadExistingArchiveIdFromDb();
+        $this->assertEquals(['1', '10', '0'], $archiveInfo);
+    }
+
+    public function test_loadExistingArchiveIdFromDb_returnsNoArchiveIfForACurrentPeriod_AndNoneAreNewEnough()
+    {
+        $params = new Parameters(new Site(1), Factory::build('month', 'now'), new Segment('', [1]));
+        $this->insertArchive($params, $tsArchived = time() - 3 * 3600);
+
+        $loader = new Loader($params);
+
+        $archiveInfo = $loader->loadExistingArchiveIdFromDb();
+        $this->assertEquals([false, false, false], $archiveInfo);
+    }
+
+    private function insertArchive(Parameters $params, $tsArchived = null, $visits = 10)
+    {
+        $archiveWriter = new ArchiveWriter($params);
+        $archiveWriter->initNewArchive();
+        $archiveWriter->insertRecord('nb_visits', $visits);
+        $archiveWriter->finalizeArchive();
+
+        if ($tsArchived) {
+            Db::query("UPDATE " . ArchiveTableCreator::getNumericTable($params->getPeriod()->getDateStart()) . " SET ts_archived = ?",
+                [Date::factory($tsArchived)->getDatetime()]);
+        }
+    }
+}
\ No newline at end of file
diff --git a/tests/PHPUnit/Integration/DataAccess/ArchiveInvalidatorTest.php b/tests/PHPUnit/Integration/DataAccess/ArchiveInvalidatorTest.php
index 530bd6e112..36365b7c49 100644
--- a/tests/PHPUnit/Integration/DataAccess/ArchiveInvalidatorTest.php
+++ b/tests/PHPUnit/Integration/DataAccess/ArchiveInvalidatorTest.php
@@ -8,7 +8,10 @@
 
 namespace Piwik\Tests\Integration\DataAccess;
 
+use Piwik\ArchiveProcessor\ArchivingStatus;
+use Piwik\ArchiveProcessor\Parameters;
 use Piwik\ArchiveProcessor\Rules;
+use Piwik\Container\StaticContainer;
 use Piwik\CronArchive\SitesToReprocessDistributedList;
 use Piwik\DataAccess\ArchiveTableCreator;
 use Piwik\DataAccess\ArchiveWriter;
@@ -16,9 +19,11 @@ use Piwik\DataAccess\Model;
 use Piwik\Date;
 use Piwik\Db;
 use Piwik\Option;
+use Piwik\Period\Factory;
 use Piwik\Piwik;
 use Piwik\Plugins\CoreAdminHome\Tasks\ArchivesToPurgeDistributedList;
 use Piwik\Plugins\PrivacyManager\PrivacyManager;
+use Piwik\Site;
 use Piwik\Tests\Framework\Fixture;
 use Piwik\Tests\Framework\TestCase\IntegrationTestCase;
 use Piwik\Archive\ArchiveInvalidator;
@@ -71,7 +76,7 @@ class ArchiveInvalidatorTest extends IntegrationTestCase
     {
         parent::setUp();
 
-        $this->invalidator = new ArchiveInvalidator(new Model());
+        $this->invalidator = new ArchiveInvalidator(new Model(), StaticContainer::get(ArchivingStatus::class));
     }
 
     public function test_rememberToInvalidateArchivedReportsLater_shouldCreateAnEntryInCaseThereIsNoneYet()
@@ -196,6 +201,39 @@ class ArchiveInvalidatorTest extends IntegrationTestCase
         $this->assertSame($expected, $reports);
     }
 
+    public function test_markArchivesAsInvalidated_shouldSkipArchivingForInProgressSites()
+    {
+        /** @var ArchivingStatus $archivingStatus */
+        $archivingStatus = StaticContainer::get(ArchivingStatus::class);
+
+        $params = new Parameters(new Site(7), Factory::build('month', '2012-02-03'), new Segment('', [2]));
+        $archivingStatus->archiveStarted($params);
+
+        $params = new Parameters(new Site(5), Factory::build('day', '2012-02-03'), new Segment('', [7]));
+        $archivingStatus->archiveStarted($params);
+
+        $this->rememberReportsForManySitesAndDates();
+
+        $idSites = array(2, 10, 7, 5);
+        $dates = array(
+            Date::factory('2014-04-05'),
+            Date::factory('2014-04-08'),
+            Date::factory('2014-05-05'),
+        );
+
+        $this->invalidator->markArchivesAsInvalidated($idSites, $dates, 'week');
+        $reports = $this->invalidator->getRememberedArchivedReportsThatShouldBeInvalidated();
+
+        $expected = array(
+            '2014-04-05' => [1,4,7],
+            '2014-04-06' => [3],
+            '2014-05-05' => [5],
+            '2014-04-08' => [7],
+            '2014-05-08' => [7],
+        );
+        $this->assertSame($expected, $reports);
+    }
+
     private function rememberReport($idSite, $date)
     {
         $date = Date::factory($date);
diff --git a/tests/PHPUnit/Integration/DataAccess/LogAggregatorTest.php b/tests/PHPUnit/Integration/DataAccess/LogAggregatorTest.php
index 8b7ac6024c..968b98e73c 100644
--- a/tests/PHPUnit/Integration/DataAccess/LogAggregatorTest.php
+++ b/tests/PHPUnit/Integration/DataAccess/LogAggregatorTest.php
@@ -8,13 +8,18 @@
 
 namespace Piwik\Tests\Integration\DataAccess;
 
+use Piwik\ArchiveProcessor\ArchivingStatus;
 use Piwik\ArchiveProcessor\Parameters;
+use Piwik\Common;
+use Piwik\Container\StaticContainer;
 use Piwik\DataAccess\LogAggregator;
 use Piwik\Date;
+use Piwik\Db;
 use Piwik\Period;
 use Piwik\Segment;
 use Piwik\Site;
 use Piwik\Tests\Fixtures\OneVisitorTwoVisits;
+use Piwik\Tests\Framework\Fixture;
 use Piwik\Tests\Framework\TestCase\IntegrationTestCase;
 
 /**
@@ -135,6 +140,37 @@ class LogAggregatorTest extends IntegrationTestCase
         ];
         $this->assertEquals($expected, $result);
     }
+
+    public function test_logAggregatorUpdatesArchiveStatusExpireTime()
+    {
+        $t = Fixture::getTracker(self::$fixture->idSite, '2010-03-06 14:22:33');
+        $t->setUrl('http://example.com/here/we/go');
+        $t->doTrackPageView('here we go');
+
+        $params = new Parameters(new Site(self::$fixture->idSite), Period\Factory::build('day', self::$fixture->dateTime), new Segment('', [self::$fixture->idSite]));
+
+        $archiveStatus = StaticContainer::get(ArchivingStatus::class);
+        $archiveStatus->archiveStarted($params);
+
+        $locks = $this->getAllLocks();
+        $this->assertCount(1, $locks);
+        $expireTime = $locks[0]['expiry_time'];
+
+        sleep(1);
+
+        $this->logAggregator->queryVisitsByDimension(['visit_total_time']);
+
+        $locks = $this->getAllLocks();
+        $this->assertCount(1, $locks);
+        $expireTimeNew = $locks[0]['expiry_time'];
+
+        $this->assertGreaterThan($expireTime, $expireTimeNew);
+    }
+
+    private function getAllLocks()
+    {
+        return Db::fetchAll("SELECT `key`, expiry_time FROM `" . Common::prefixTable('locks') . '`');
+    }
 }
 
 LogAggregatorTest::$fixture = new OneVisitorTwoVisits();