Lock when archiving and avoid invalidating sites that have archiving in progress (#15272)

* Use a lock when archiving and do not invalidate when archiving is in progress.

* Add and fix tests + modify workflow.

* forgot to add file and remove TODO

* Remove use of argument.

* Add back min archive time processed code and start on tests for it.

* Finish new LoaderTest.

* Fix new tests.

4 files changed, 284 insertions, 1 deletions

diff --git a/tests/PHPUnit/Integration/ArchiveProcessor/ArchivingStatusTest.php b/tests/PHPUnit/Integration/ArchiveProcessor/ArchivingStatusTest.php
new file mode 100644
index 0000000000..e7a873ffea
--- /dev/null
+++ b/tests/PHPUnit/Integration/ArchiveProcessor/ArchivingStatusTest.php
@@ -0,0 +1,87 @@
+<?php
+/**
+ * Piwik - free/libre analytics platform
+ *
+ * @link https://matomo.org
+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
+ *
+ */
+
+namespace Piwik\Tests\Integration\ArchiveProcessor;
+
+use Piwik\ArchiveProcessor\ArchivingStatus;
+use Piwik\ArchiveProcessor\Parameters;
+use Piwik\Common;
+use Piwik\Container\StaticContainer;
+use Piwik\Db;
+use Piwik\Period\Factory;
+use Piwik\Segment;
+use Piwik\Site;
+use Piwik\Tests\Framework\Fixture;
+use Piwik\Tests\Framework\TestCase\IntegrationTestCase;
+
+class ArchivingStatusTest extends IntegrationTestCase
+{
+    public function setUp()
+    {
+        parent::setUp();
+
+        Fixture::createWebsite('2010-02-03 00:00:00');
+        Fixture::createWebsite('2010-02-03 00:00:00');
+    }
+
+    public function test_archiveStartedAndArchiveFinished_workflow()
+    {
+        /** @var ArchivingStatus $archivingStatus */
+        $archivingStatus = StaticContainer::get(ArchivingStatus::class);
+
+        $params = new Parameters(new Site(1), Factory::build('month', '2012-02-04'), new Segment('', [1]));
+        $archivingStatus->archiveStarted($params);
+
+        $lock = $archivingStatus->getCurrentArchivingLock();
+        $this->assertNotEmpty($lock);
+        $this->assertTrue($lock->isLocked());
+
+        $this->assertEquals([
+            [
+                'key' => 'Archiving.1.fab0afcc3a068ecd91b14d50f3bc911d.' . getmypid(),
+                'expiry_time' => time() + ArchivingStatus::DEFAULT_ARCHIVING_TTL,
+            ],
+        ], $this->getLockKeysAndTtls());
+
+        $archivingStatus->archiveFinished();
+
+        $this->assertEquals([], $this->getLockKeysAndTtls());
+    }
+
+    public function test_getSitesCurrentlyArchiving_returnsAllSitesArchiving()
+    {
+        /** @var ArchivingStatus $archivingStatus */
+        $archivingStatus = StaticContainer::get(ArchivingStatus::class);
+
+        $params = new Parameters(new Site(1), Factory::build('month', '2012-02-04'), new Segment('', [1]));
+        $archivingStatus->archiveStarted($params);
+
+        $params = new Parameters(new Site(1), Factory::build('month', '2012-02-04'), new Segment('browserCode==ff', [1]));
+        $archivingStatus->archiveStarted($params);
+
+        $params = new Parameters(new Site(2), Factory::build('month', '2012-02-04'), new Segment('', [1]));
+        $archivingStatus->archiveStarted($params);
+
+        $this->assertEquals([
+            1, 2,
+        ], $archivingStatus->getSitesCurrentlyArchiving());
+
+        $archivingStatus->archiveFinished();
+        $archivingStatus->archiveFinished();
+
+        $this->assertEquals([
+            1,
+        ], $archivingStatus->getSitesCurrentlyArchiving());
+    }
+
+    private function getLockKeysAndTtls()
+    {
+        return Db::fetchAll("SELECT `key`, expiry_time FROM `" . Common::prefixTable('locks') . '`');
+    }
+}
\ No newline at end of file
diff --git a/tests/PHPUnit/Integration/ArchiveProcessor/LoaderTest.php b/tests/PHPUnit/Integration/ArchiveProcessor/LoaderTest.php
new file mode 100644
index 0000000000..e99d495931
--- /dev/null
+++ b/tests/PHPUnit/Integration/ArchiveProcessor/LoaderTest.php
@@ -0,0 +1,122 @@
+<?php
+/**
+ * Piwik - free/libre analytics platform
+ *
+ * @link https://matomo.org
+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
+ *
+ */
+
+namespace Piwik\Tests\Integration\ArchiveProcessor;
+
+
+use Piwik\ArchiveProcessor\Parameters;
+use Piwik\ArchiveProcessor\Loader;
+use Piwik\Common;
+use Piwik\Config;
+use Piwik\DataAccess\ArchiveTableCreator;
+use Piwik\DataAccess\ArchiveWriter;
+use Piwik\Date;
+use Piwik\Db;
+use Piwik\Period\Factory;
+use Piwik\Segment;
+use Piwik\Site;
+use Piwik\Tests\Framework\Fixture;
+use Piwik\Tests\Framework\TestCase\IntegrationTestCase;
+
+class LoaderTest extends IntegrationTestCase
+{
+    protected static function beforeTableDataCached()
+    {
+        parent::beforeTableDataCached();
+
+        Fixture::createWebsite('2012-02-03 00:00:00');
+    }
+
+    public function test_loadExistingArchiveIdFromDb_returnsFalsesIfNoArchiveFound()
+    {
+        $params = new Parameters(new Site(1), Factory::build('day', '2015-03-03'), new Segment('', [1]));
+        $loader = new Loader($params);
+
+        $archiveInfo = $loader->loadExistingArchiveIdFromDb();
+
+        $this->assertEquals([false, false, false], $archiveInfo);
+    }
+
+    /**
+     * @dataProvider getTestDataForLoadExistingArchiveIdFromDbDebugConfig
+     */
+    public function test_loadExistingArchiveIdFromDb_returnsFalsesPeriodIsForcedToArchive($periodType, $configSetting)
+    {
+        $date = $periodType == 'range' ? '2015-03-03,2015-03-04' : '2015-03-03';
+        $params = new Parameters(new Site(1), Factory::build($periodType, $date), new Segment('', [1]));
+        $this->insertArchive($params);
+
+        $loader = new Loader($params);
+
+        $archiveInfo = $loader->loadExistingArchiveIdFromDb();
+        $this->assertNotEquals([false, false, false], $archiveInfo);
+
+        Config::getInstance()->Debug[$configSetting] = 1;
+
+        $archiveInfo = $loader->loadExistingArchiveIdFromDb();
+        $this->assertEquals([false, false, false], $archiveInfo);
+    }
+
+    public function getTestDataForLoadExistingArchiveIdFromDbDebugConfig()
+    {
+        return [
+            ['day', 'always_archive_data_day'],
+            ['week', 'always_archive_data_period'],
+            ['month', 'always_archive_data_period'],
+            ['year', 'always_archive_data_period'],
+            ['range', 'always_archive_data_range'],
+        ];
+    }
+
+    public function test_loadExistingArchiveIdFromDb_returnsArchiveIfArchiveInThePast()
+    {
+        $params = new Parameters(new Site(1), Factory::build('month', '2015-03-03'), new Segment('', [1]));
+        $this->insertArchive($params);
+
+        $loader = new Loader($params);
+
+        $archiveInfo = $loader->loadExistingArchiveIdFromDb();
+        $this->assertEquals(['1', '10', '0'], $archiveInfo);
+    }
+
+    public function test_loadExistingArchiveIdFromDb_returnsArchiveIfForACurrentPeriod_AndNewEnough()
+    {
+        $params = new Parameters(new Site(1), Factory::build('day', 'now'), new Segment('', [1]));
+        $this->insertArchive($params, $tsArchived = time() - 1);
+
+        $loader = new Loader($params);
+
+        $archiveInfo = $loader->loadExistingArchiveIdFromDb();
+        $this->assertEquals(['1', '10', '0'], $archiveInfo);
+    }
+
+    public function test_loadExistingArchiveIdFromDb_returnsNoArchiveIfForACurrentPeriod_AndNoneAreNewEnough()
+    {
+        $params = new Parameters(new Site(1), Factory::build('month', 'now'), new Segment('', [1]));
+        $this->insertArchive($params, $tsArchived = time() - 3 * 3600);
+
+        $loader = new Loader($params);
+
+        $archiveInfo = $loader->loadExistingArchiveIdFromDb();
+        $this->assertEquals([false, false, false], $archiveInfo);
+    }
+
+    private function insertArchive(Parameters $params, $tsArchived = null, $visits = 10)
+    {
+        $archiveWriter = new ArchiveWriter($params);
+        $archiveWriter->initNewArchive();
+        $archiveWriter->insertRecord('nb_visits', $visits);
+        $archiveWriter->finalizeArchive();
+
+        if ($tsArchived) {
+            Db::query("UPDATE " . ArchiveTableCreator::getNumericTable($params->getPeriod()->getDateStart()) . " SET ts_archived = ?",
+                [Date::factory($tsArchived)->getDatetime()]);
+        }
+    }
+}
\ No newline at end of file
diff --git a/tests/PHPUnit/Integration/DataAccess/ArchiveInvalidatorTest.php b/tests/PHPUnit/Integration/DataAccess/ArchiveInvalidatorTest.php
index 530bd6e112..36365b7c49 100644
--- a/tests/PHPUnit/Integration/DataAccess/ArchiveInvalidatorTest.php
+++ b/tests/PHPUnit/Integration/DataAccess/ArchiveInvalidatorTest.php
@@ -8,7 +8,10 @@
 
 namespace Piwik\Tests\Integration\DataAccess;
 
+use Piwik\ArchiveProcessor\ArchivingStatus;
+use Piwik\ArchiveProcessor\Parameters;
 use Piwik\ArchiveProcessor\Rules;
+use Piwik\Container\StaticContainer;
 use Piwik\CronArchive\SitesToReprocessDistributedList;
 use Piwik\DataAccess\ArchiveTableCreator;
 use Piwik\DataAccess\ArchiveWriter;
@@ -16,9 +19,11 @@ use Piwik\DataAccess\Model;
 use Piwik\Date;
 use Piwik\Db;
 use Piwik\Option;
+use Piwik\Period\Factory;
 use Piwik\Piwik;
 use Piwik\Plugins\CoreAdminHome\Tasks\ArchivesToPurgeDistributedList;
 use Piwik\Plugins\PrivacyManager\PrivacyManager;
+use Piwik\Site;
 use Piwik\Tests\Framework\Fixture;
 use Piwik\Tests\Framework\TestCase\IntegrationTestCase;
 use Piwik\Archive\ArchiveInvalidator;
@@ -71,7 +76,7 @@ class ArchiveInvalidatorTest extends IntegrationTestCase
     {
         parent::setUp();
 
-        $this->invalidator = new ArchiveInvalidator(new Model());
+        $this->invalidator = new ArchiveInvalidator(new Model(), StaticContainer::get(ArchivingStatus::class));
     }
 
     public function test_rememberToInvalidateArchivedReportsLater_shouldCreateAnEntryInCaseThereIsNoneYet()
@@ -196,6 +201,39 @@ class ArchiveInvalidatorTest extends IntegrationTestCase
         $this->assertSame($expected, $reports);
     }
 
+    public function test_markArchivesAsInvalidated_shouldSkipArchivingForInProgressSites()
+    {
+        /** @var ArchivingStatus $archivingStatus */
+        $archivingStatus = StaticContainer::get(ArchivingStatus::class);
+
+        $params = new Parameters(new Site(7), Factory::build('month', '2012-02-03'), new Segment('', [2]));
+        $archivingStatus->archiveStarted($params);
+
+        $params = new Parameters(new Site(5), Factory::build('day', '2012-02-03'), new Segment('', [7]));
+        $archivingStatus->archiveStarted($params);
+
+        $this->rememberReportsForManySitesAndDates();
+
+        $idSites = array(2, 10, 7, 5);
+        $dates = array(
+            Date::factory('2014-04-05'),
+            Date::factory('2014-04-08'),
+            Date::factory('2014-05-05'),
+        );
+
+        $this->invalidator->markArchivesAsInvalidated($idSites, $dates, 'week');
+        $reports = $this->invalidator->getRememberedArchivedReportsThatShouldBeInvalidated();
+
+        $expected = array(
+            '2014-04-05' => [1,4,7],
+            '2014-04-06' => [3],
+            '2014-05-05' => [5],
+            '2014-04-08' => [7],
+            '2014-05-08' => [7],
+        );
+        $this->assertSame($expected, $reports);
+    }
+
     private function rememberReport($idSite, $date)
     {
         $date = Date::factory($date);
diff --git a/tests/PHPUnit/Integration/DataAccess/LogAggregatorTest.php b/tests/PHPUnit/Integration/DataAccess/LogAggregatorTest.php
index 8b7ac6024c..968b98e73c 100644
--- a/tests/PHPUnit/Integration/DataAccess/LogAggregatorTest.php
+++ b/tests/PHPUnit/Integration/DataAccess/LogAggregatorTest.php
@@ -8,13 +8,18 @@
 
 namespace Piwik\Tests\Integration\DataAccess;
 
+use Piwik\ArchiveProcessor\ArchivingStatus;
 use Piwik\ArchiveProcessor\Parameters;
+use Piwik\Common;
+use Piwik\Container\StaticContainer;
 use Piwik\DataAccess\LogAggregator;
 use Piwik\Date;
+use Piwik\Db;
 use Piwik\Period;
 use Piwik\Segment;
 use Piwik\Site;
 use Piwik\Tests\Fixtures\OneVisitorTwoVisits;
+use Piwik\Tests\Framework\Fixture;
 use Piwik\Tests\Framework\TestCase\IntegrationTestCase;
 
 /**
@@ -135,6 +140,37 @@ class LogAggregatorTest extends IntegrationTestCase
         ];
         $this->assertEquals($expected, $result);
     }
+
+    public function test_logAggregatorUpdatesArchiveStatusExpireTime()
+    {
+        $t = Fixture::getTracker(self::$fixture->idSite, '2010-03-06 14:22:33');
+        $t->setUrl('http://example.com/here/we/go');
+        $t->doTrackPageView('here we go');
+
+        $params = new Parameters(new Site(self::$fixture->idSite), Period\Factory::build('day', self::$fixture->dateTime), new Segment('', [self::$fixture->idSite]));
+
+        $archiveStatus = StaticContainer::get(ArchivingStatus::class);
+        $archiveStatus->archiveStarted($params);
+
+        $locks = $this->getAllLocks();
+        $this->assertCount(1, $locks);
+        $expireTime = $locks[0]['expiry_time'];
+
+        sleep(1);
+
+        $this->logAggregator->queryVisitsByDimension(['visit_total_time']);
+
+        $locks = $this->getAllLocks();
+        $this->assertCount(1, $locks);
+        $expireTimeNew = $locks[0]['expiry_time'];
+
+        $this->assertGreaterThan($expireTime, $expireTimeNew);
+    }
+
+    private function getAllLocks()
+    {
+        return Db::fetchAll("SELECT `key`, expiry_time FROM `" . Common::prefixTable('locks') . '`');
+    }
 }
 
 LogAggregatorTest::$fixture = new OneVisitorTwoVisits();