Move Archive.php archive invalidation to Loader… (#15616)3.13.4-b1

* Move Archive.php archive invalidation to Loader so we only invalidate when about to launch archiving.

* Attempt to handle more cases when invalidating before launching archiving.

* fix possible sql error

* fix possible error

* fixing some tests

* remove test code

* Only invalidate specific archive being requested.

* Do not invalidate on today in tracker and avoid existing valid archive check in CronArchive.

* more test fixes

* Attempt to fix more tests.

* Fixing last tests.

* another test fix

* Invalidate in scheduled task if browser triggered archiving is enabled.

* deal with TODO

* Get ArchiveSelectorTest to pass.

* applying review feedback including new tests

* apply review feedback & fix tests

* fix couple more tests

Co-authored-by: Thomas Steur <tsteur@users.noreply.github.com>

11 files changed, 565 insertions, 470 deletions

diff --git a/tests/PHPUnit/Integration/ArchiveProcessor/LoaderTest.php b/tests/PHPUnit/Integration/ArchiveProcessor/LoaderTest.php
index e99d495931..e354a2b988 100644
--- a/tests/PHPUnit/Integration/ArchiveProcessor/LoaderTest.php
+++ b/tests/PHPUnit/Integration/ArchiveProcessor/LoaderTest.php
@@ -10,10 +10,12 @@
 namespace Piwik\Tests\Integration\ArchiveProcessor;
 
 
+use Piwik\Archive\ArchiveInvalidator;
 use Piwik\ArchiveProcessor\Parameters;
 use Piwik\ArchiveProcessor\Loader;
 use Piwik\Common;
 use Piwik\Config;
+use Piwik\Container\StaticContainer;
 use Piwik\DataAccess\ArchiveTableCreator;
 use Piwik\DataAccess\ArchiveWriter;
 use Piwik\Date;
@@ -31,6 +33,7 @@ class LoaderTest extends IntegrationTestCase
         parent::beforeTableDataCached();
 
         Fixture::createWebsite('2012-02-03 00:00:00');
+        Fixture::createWebsite('2012-02-03 00:00:00');
     }
 
     public function test_loadExistingArchiveIdFromDb_returnsFalsesIfNoArchiveFound()
@@ -40,7 +43,7 @@ class LoaderTest extends IntegrationTestCase
 
         $archiveInfo = $loader->loadExistingArchiveIdFromDb();
 
-        $this->assertEquals([false, false, false], $archiveInfo);
+        $this->assertEquals([false, false, false, false], $archiveInfo);
     }
 
     /**
@@ -55,12 +58,12 @@ class LoaderTest extends IntegrationTestCase
         $loader = new Loader($params);
 
         $archiveInfo = $loader->loadExistingArchiveIdFromDb();
-        $this->assertNotEquals([false, false, false], $archiveInfo);
+        $this->assertNotEquals([false, false, false, false], $archiveInfo);
 
         Config::getInstance()->Debug[$configSetting] = 1;
 
         $archiveInfo = $loader->loadExistingArchiveIdFromDb();
-        $this->assertEquals([false, false, false], $archiveInfo);
+        $this->assertEquals([false, false, false, false], $archiveInfo);
     }
 
     public function getTestDataForLoadExistingArchiveIdFromDbDebugConfig()
@@ -82,7 +85,7 @@ class LoaderTest extends IntegrationTestCase
         $loader = new Loader($params);
 
         $archiveInfo = $loader->loadExistingArchiveIdFromDb();
-        $this->assertEquals(['1', '10', '0'], $archiveInfo);
+        $this->assertEquals(['1', '10', '0', true], $archiveInfo);
     }
 
     public function test_loadExistingArchiveIdFromDb_returnsArchiveIfForACurrentPeriod_AndNewEnough()
@@ -93,7 +96,7 @@ class LoaderTest extends IntegrationTestCase
         $loader = new Loader($params);
 
         $archiveInfo = $loader->loadExistingArchiveIdFromDb();
-        $this->assertEquals(['1', '10', '0'], $archiveInfo);
+        $this->assertEquals(['1', '10', '0', true], $archiveInfo);
     }
 
     public function test_loadExistingArchiveIdFromDb_returnsNoArchiveIfForACurrentPeriod_AndNoneAreNewEnough()
@@ -104,7 +107,103 @@ class LoaderTest extends IntegrationTestCase
         $loader = new Loader($params);
 
         $archiveInfo = $loader->loadExistingArchiveIdFromDb();
-        $this->assertEquals([false, false, false], $archiveInfo);
+        $this->assertEquals([false, '10', '0', true], $archiveInfo); // visits are still returned as this was the original behavior
+    }
+
+    /**
+     * @dataProvider getTestDataForGetReportsToInvalidate
+     */
+    public function test_getReportsToInvalidate_returnsCorrectReportsToInvalidate($rememberedReports, $idSite, $period, $date, $segment, $expected)
+    {
+        $invalidator = StaticContainer::get(ArchiveInvalidator::class);
+        foreach ($rememberedReports as $entry) {
+            $invalidator->rememberToInvalidateArchivedReportsLater($entry['idSite'], Date::factory($entry['date']));
+        }
+
+        $params = new Parameters(new Site($idSite), Factory::build($period, $date), new Segment($segment, [$idSite]));
+        $loader = new Loader($params);
+
+        $reportsToInvalidate = $loader->getReportsToInvalidate();
+        $this->assertEquals($expected, $reportsToInvalidate);
+    }
+
+    public function getTestDataForGetReportsToInvalidate()
+    {
+        return [
+            // two dates for one site
+            [
+                [
+                    ['idSite' => 1, 'date' => '2013-04-05'],
+                    ['idSite' => 1, 'date' => '2013-03-05'],
+                    ['idSite' => 2, 'date' => '2013-05-05'],
+                ],
+                1,
+                'day',
+                '2013-04-05',
+                '',
+                [
+                    '2013-04-05' => [1],
+                ],
+            ],
+
+            // no dates for a site
+            [
+                [
+                    ['idSite' => '', 'date' => '2013-04-05'],
+                    ['idSite' => '', 'date' => '2013-04-06'],
+                    ['idSite' => 2, 'date' => '2013-05-05'],
+                ],
+                1,
+                'day',
+                '2013-04-05',
+                'browserCode==ff',
+                [],
+            ],
+
+            // day period not within range
+            [
+                [
+                    ['idSite' => 1, 'date' => '2014-03-04'],
+                    ['idSite' => 1, 'date' => '2014-03-06'],
+                ],
+                1,
+                'day',
+                '2013-03-05',
+                '',
+                [],
+            ],
+
+            // non-day periods
+            [
+                [
+                    ['idSite' => 1, 'date' => '2014-03-01'],
+                    ['idSite' => 1, 'date' => '2014-03-06'],
+                    ['idSite' => 2, 'date' => '2014-03-01'],
+                ],
+                1,
+                'week',
+                '2014-03-01',
+                '',
+                [
+                    '2014-03-01' => [2, 1],
+                ],
+            ],
+            [
+                [
+                    ['idSite' => 1, 'date' => '2014-02-01'],
+                    ['idSite' => 1, 'date' => '2014-03-06'],
+                    ['idSite' => 2, 'date' => '2014-03-05'],
+                    ['idSite' => 2, 'date' => '2014-03-06'],
+                ],
+                1,
+                'month',
+                '2014-03-01',
+                '',
+                [
+                    '2014-03-06' => [2, 1],
+                ],
+            ],
+        ];
     }
 
     private function insertArchive(Parameters $params, $tsArchived = null, $visits = 10)
diff --git a/tests/PHPUnit/Integration/ArchiveTest.php b/tests/PHPUnit/Integration/ArchiveTest.php
deleted file mode 100644
index bd2e2d8246..0000000000
--- a/tests/PHPUnit/Integration/ArchiveTest.php
+++ /dev/null
@@ -1,441 +0,0 @@
-<?php
-/**
- * Piwik - free/libre analytics platform
- *
- * @link https://matomo.org
- * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
- */
-namespace Piwik\Tests\Integration;
-
-use Piwik\API\Proxy;
-use Piwik\API\Request;
-use Piwik\Archive as PiwikArchive;
-use Piwik\ArchiveProcessor;
-use Piwik\ArchiveProcessor\Parameters;
-use Piwik\ArchiveProcessor\Rules;
-use Piwik\Common;
-use Piwik\Config;
-use Piwik\DataAccess\ArchiveSelector;
-use Piwik\DataAccess\ArchiveTableCreator;
-use Piwik\DataAccess\ArchiveWriter;
-use Piwik\DataAccess\LogAggregator;
-use Piwik\Date;
-use Piwik\Db;
-use Piwik\Piwik;
-use Piwik\Plugins\UserLanguage;
-use Piwik\Segment;
-use Piwik\Site;
-use Piwik\Tests\Fixtures\OneVisitorTwoVisits;
-use Piwik\Tests\Framework\Fixture;
-use Piwik\Tests\Framework\TestCase\IntegrationTestCase;
-use Piwik\Period\Factory as PeriodFactory;
-use Piwik\Archive\Chunk;
-
-class Archive extends PiwikArchive
-{
-    public function get($archiveNames, $archiveDataType, $idSubtable = null)
-    {
-        return parent::get($archiveNames, $archiveDataType, $idSubtable);
-    }
-}
-
-class CustomArchiveQueryFactory extends PiwikArchive\ArchiveQueryFactory
-{
-    public function newInstance(\Piwik\Archive\Parameters $params, $forceIndexedBySite, $forceIndexedByDate)
-    {
-        return new Archive($params, $forceIndexedBySite, $forceIndexedByDate);
-    }
-}
-
-/**
- * @group Core
- */
-class ArchiveTest extends IntegrationTestCase
-{
-    /**
-     * @var OneVisitorTwoVisits
-     */
-    public static $fixture;
-
-    public function tearDown()
-    {
-        parent::tearDown();
-
-        unset($_GET['trigger']);
-    }
-
-    protected static function configureFixture($fixture)
-    {
-        $fixture->createSuperUser = true;
-    }
-
-    protected static function beforeTableDataCached()
-    {
-        $date = Date::factory('2010-03-01');
-
-        $archiveTableCreator = new ArchiveTableCreator();
-        $archiveTableCreator->getBlobTable($date);
-        $archiveTableCreator->getNumericTable($date);
-    }
-
-    public function getForceOptionsForForceArchivingOnBrowserRequest()
-    {
-        return array(
-            array(1),
-            array(null)
-        );
-    }
-
-    /**
-     * @dataProvider getForceOptionsForForceArchivingOnBrowserRequest
-     */
-    public function test_ArchivingIsLaunchedForRanges_WhenForceOnBrowserRequest_IsTruthy($optionValue)
-    {
-        $this->archiveDataForIndividualDays();
-
-        Config::getInstance()->General['archiving_range_force_on_browser_request'] = $optionValue;
-        Rules::setBrowserTriggerArchiving(false);
-
-        $data = $this->initiateArchivingForRange();
-
-        $this->assertNotEmpty($data);
-        $this->assertArchiveTablesAreNotEmpty('2010_03');
-    }
-
-    public function test_ArchivingIsNotLaunchedForRanges_WhenForceOnBrowserRequest_IsFalse()
-    {
-        $this->archiveDataForIndividualDays();
-
-        Config::getInstance()->General['archiving_range_force_on_browser_request'] = 0;
-        Rules::setBrowserTriggerArchiving(false);
-
-        $data = $this->initiateArchivingForRange();
-
-        $this->assertEmpty($data);
-        $this->assertArchiveTablesAreEmpty('2010_03');
-    }
-
-    public function test_ArchiveIsLaunched_WhenForceOnBrowserRequest_IsFalse_AndArchivePhpTriggered()
-    {
-        $this->archiveDataForIndividualDays();
-
-        Config::getInstance()->General['archiving_range_force_on_browser_request'] = 0;
-        $_GET['trigger'] = 'archivephp';
-        Rules::setBrowserTriggerArchiving(false);
-
-        $data = $this->initiateArchivingForRange();
-
-        $this->assertNotEmpty($data);
-        $this->assertArchiveTablesAreNotEmpty('2010_03');
-    }
-
-    public function test_ArchiveBlob_ShouldBeAbleToLoadFirstLevelDataArrays()
-    {
-        $this->createManyDifferentArchiveBlobs();
-
-        $archive = $this->getArchive('day', '2013-01-01,2013-01-05');
-        $dataArrays = $archive->get(array('Actions_Actionsurl'), 'blob');
-
-        $this->assertArchiveBlob($dataArrays, '2013-01-01', array('Actions_Actionsurl' => 'test01'));
-        $this->assertArchiveBlob($dataArrays, '2013-01-02', array('Actions_Actionsurl' => 'test02'));
-        $this->assertArchiveBlob($dataArrays, '2013-01-03', array('Actions_Actionsurl' => 'test03'));
-        $this->assertArchiveBlob($dataArrays, '2013-01-04', array('Actions_Actionsurl' => 'test04'));
-        $this->assertArchiveBlob($dataArrays, '2013-01-05', array('Actions_Actionsurl' => 0));
-    }
-
-    public function test_ArchiveBlob_ShouldBeAbleToLoadOneSubtable_NoMatterWhetherTheyAreStoredSeparatelyOrInACombinedSubtableEntry()
-    {
-        $this->createManyDifferentArchiveBlobs();
-
-        $archive = $this->getArchive('day', '2013-01-01,2013-01-05');
-        $dataArrays = $archive->get(array('Actions_Actionsurl'), 'blob', 2);
-
-        $this->assertArchiveBlob($dataArrays, '2013-01-01', array('Actions_Actionsurl_2' => 0));
-        $this->assertArchiveBlob($dataArrays, '2013-01-02', array('Actions_Actionsurl_2' => 'test2'));
-        $this->assertArchiveBlob($dataArrays, '2013-01-03', array('Actions_Actionsurl_2' => 'subtable2'));
-        $this->assertArchiveBlob($dataArrays, '2013-01-04', array('Actions_Actionsurl_2' => 0));
-        $this->assertArchiveBlob($dataArrays, '2013-01-05', array('Actions_Actionsurl_2' => 0));
-
-        // test another one
-        $dataArrays = $archive->get(array('Actions_Actionsurl'), 'blob', 5);
-
-        $this->assertArchiveBlob($dataArrays, '2013-01-01', array('Actions_Actionsurl_5' => 0));
-        $this->assertArchiveBlob($dataArrays, '2013-01-02', array('Actions_Actionsurl_5' => 0));
-        $this->assertArchiveBlob($dataArrays, '2013-01-03', array('Actions_Actionsurl_5' => 'subtable5'));
-        $this->assertArchiveBlob($dataArrays, '2013-01-04', array('Actions_Actionsurl_5' => 'subtable45'));
-        $this->assertArchiveBlob($dataArrays, '2013-01-05', array('Actions_Actionsurl_5' => 0));
-
-        // test one that does not exist
-        $dataArrays = $archive->get(array('Actions_Actionsurl'), 'blob', 999);
-
-        $this->assertArchiveBlob($dataArrays, '2013-01-01', array('Actions_Actionsurl_999' => 0));
-        $this->assertArchiveBlob($dataArrays, '2013-01-02', array('Actions_Actionsurl_999' => 0));
-        $this->assertArchiveBlob($dataArrays, '2013-01-03', array('Actions_Actionsurl_999' => 0));
-        $this->assertArchiveBlob($dataArrays, '2013-01-04', array('Actions_Actionsurl_999' => 0));
-        $this->assertArchiveBlob($dataArrays, '2013-01-05', array('Actions_Actionsurl_999' => 0));
-    }
-
-    public function test_ArchiveBlob_ShouldBeAbleToLoadAllSubtables_NoMatterWhetherTheyAreStoredSeparatelyOrInACombinedSubtableEntry()
-    {
-        $this->createManyDifferentArchiveBlobs();
-
-        $archive = $this->getArchive('day', '2013-01-01,2013-01-06');
-        $dataArrays = $archive->get(array('Actions_Actionsurl'), 'blob', Archive::ID_SUBTABLE_LOAD_ALL_SUBTABLES);
-
-        $this->assertArchiveBlob($dataArrays, '2013-01-01', array('Actions_Actionsurl' => 'test01'));
-        $this->assertArchiveBlob($dataArrays, '2013-01-02', array('Actions_Actionsurl' => 'test02', 'Actions_Actionsurl_1' => 'test1', 'Actions_Actionsurl_2' => 'test2'));
-        $this->assertArchiveBlob($dataArrays, '2013-01-03', array('Actions_Actionsurl' => 'test03', 'Actions_Actionsurl_1' => 'subtable1', 'Actions_Actionsurl_2' => 'subtable2', 'Actions_Actionsurl_5' => 'subtable5'));
-        $this->assertArchiveBlob($dataArrays, '2013-01-04', array('Actions_Actionsurl' => 'test04', 'Actions_Actionsurl_5' => 'subtable45', 'Actions_Actionsurl_6' => 'subtable6'));
-        $this->assertArchiveBlob($dataArrays, '2013-01-05', array('Actions_Actionsurl' => 0));
-        $this->assertArchiveBlob($dataArrays, '2013-01-06', array('Actions_Actionsurl' => 'test06'));
-    }
-
-    public function test_ArchiveBlob_ShouldBeAbleToLoadDifferentArchives_NoMatterWhetherTheyAreStoredSeparatelyOrInACombinedSubtableEntry()
-    {
-        $this->createManyDifferentArchiveBlobs();
-
-        $archive = $this->getArchive('day', '2013-01-01,2013-01-06');
-        $dataArrays = $archive->get(array('Actions_Actionsurl', 'Actions_Actions'), 'blob', 2);
-
-        $this->assertArchiveBlob($dataArrays, '2013-01-01', array('Actions_Actionsurl_2' => 0, 'Actions_Actions_2' => 0));
-        $this->assertArchiveBlob($dataArrays, '2013-01-02', array('Actions_Actions_2' => 'actionsSubtable2', 'Actions_Actionsurl_2' => 'test2'));
-        $this->assertArchiveBlob($dataArrays, '2013-01-03', array('Actions_Actions_2' => 'actionsTest2', 'Actions_Actionsurl_2' => 'subtable2'));
-        $this->assertArchiveBlob($dataArrays, '2013-01-04', array('Actions_Actionsurl_2' => 0, 'Actions_Actions_2' => 0));
-        $this->assertArchiveBlob($dataArrays, '2013-01-05', array('Actions_Actionsurl_2' => 0, 'Actions_Actions_2' => 0));
-        $this->assertArchiveBlob($dataArrays, '2013-01-06', array('Actions_Actionsurl_2' => 0, 'Actions_Actions_2' => 0));
-    }
-
-    /**
-     * @dataProvider findBlobsWithinDifferentChunksDataProvider
-     */
-    public function test_ArchiveBlob_ShouldBeFindBlobs_WithinDifferentChunks($idSubtable, $expectedBlob)
-    {
-        $recordName = 'Actions_Actions';
-
-        $chunk    = new Chunk();
-        $chunk5   = $chunk->getRecordNameForTableId($recordName, $subtableId = 5);
-        $chunk152 = $chunk->getRecordNameForTableId($recordName, $subtableId = 152);
-        $chunk399 = $chunk->getRecordNameForTableId($recordName, $subtableId = 399);
-
-        $this->createArchiveBlobEntry('2013-01-02', array(
-            $recordName => 'actions_02',
-            $chunk5     => serialize(array(1 => 'actionsSubtable1', 2 => 'actionsSubtable2', 5 => 'actionsSubtable5')),
-            $chunk152   => serialize(array(151 => 'actionsSubtable151', 152 => 'actionsSubtable152')),
-            $chunk399   => serialize(array(399 => 'actionsSubtable399'))
-        ));
-
-        $archive = $this->getArchive('day', '2013-01-02,2013-01-02');
-
-        $dataArrays = $archive->get(array('Actions_Actions'), 'blob', $idSubtable);
-        $this->assertArchiveBlob($dataArrays, '2013-01-02', $expectedBlob);
-    }
-
-    public function findBlobsWithinDifferentChunksDataProvider()
-    {
-        return array(
-            array($idSubtable = 2, $expectedBlobs = array('Actions_Actions_2' => 'actionsSubtable2')),
-            array(5, array('Actions_Actions_5' => 'actionsSubtable5')),
-            array(151, array('Actions_Actions_151' => 'actionsSubtable151')),
-            array(152, array('Actions_Actions_152' => 'actionsSubtable152')),
-            array(399, array('Actions_Actions_399' => 'actionsSubtable399')),
-            // this one does not exist
-            array(404, array('Actions_Actions_404' => 0)),
-        );
-    }
-
-    public function testExistingArchivesAreReplaced()
-    {
-        $date = self::$fixture->dateTime;
-        $period = PeriodFactory::makePeriodFromQueryParams('UTC', 'day', $date);
-
-        // request an report to trigger archiving
-        $userLanguageReport = Proxy::getInstance()->call('\\Piwik\\Plugins\\UserLanguage\\API', 'getLanguage', array(
-            'idSite' => 1,
-            'period' => 'day',
-            'date' => $date
-        ));
-
-        $this->assertEquals(1, $userLanguageReport->getRowsCount());
-        $this->assertEquals('UserLanguage_LanguageCode fr', $userLanguageReport->getFirstRow()->getColumn('label'));
-        $this->assertEquals('UserLanguage_LanguageCode fr', $userLanguageReport->getLastRow()->getColumn('label'));
-
-        $parameters = new Parameters(new Site(1), $period, new Segment('', []));
-        $parameters->setRequestedPlugin('UserLanguage');
-
-        $result    = ArchiveSelector::getArchiveIdAndVisits($parameters, $period->getDateStart()->getDateStartUTC());
-        $idArchive = $result ? array_shift($result) : null;
-
-        if (empty($idArchive)) {
-            $this->fail('Archive should be available');
-        }
-
-        // track a new visits now
-        $t = Fixture::getTracker(1, $date, $defaultInit = true);
-        $t->setForceVisitDateTime(Date::factory($date)->addHour(1)->getDatetime());
-        $t->setUrl('http://site.com/index.htm');
-        $t->setBrowserLanguage('pt-br');
-        Fixture::checkResponse($t->doTrackPageView('my_site'));
-
-        $archiveWriter            = new ArchiveWriter($parameters, !!$idArchive);
-        $archiveWriter->idArchive = $idArchive;
-
-        $archiveProcessor = new ArchiveProcessor($parameters, $archiveWriter,
-            new LogAggregator($parameters));
-
-        $archiveProcessor->setNumberOfVisits(1, 1);
-
-        // directly trigger specific archiver for existing archive
-        $archiver = new UserLanguage\Archiver($archiveProcessor);
-        $archiver->aggregateDayReport();
-        $archiveWriter->finalizeArchive();
-
-        // report should be updated
-        $userLanguageReport = Proxy::getInstance()->call('\\Piwik\\Plugins\\UserLanguage\\API', 'getLanguage', array(
-            'idSite' => 1,
-            'period' => 'day',
-            'date' => $date
-        ));
-
-        $this->assertEquals(2, $userLanguageReport->getRowsCount());
-        $this->assertEquals('UserLanguage_LanguageCode fr', $userLanguageReport->getFirstRow()->getColumn('label'));
-        $this->assertEquals('UserLanguage_LanguageCode pt', $userLanguageReport->getLastRow()->getColumn('label'));
-    }
-
-    public function testNoFutureArchiveTablesAreCreatedWithoutArchiving()
-    {
-        $dateTime = '2066-01-01';
-        Config::getInstance()->General['enable_browser_archiving_triggering'] = 0;
-
-        // request API which should not trigger archiving due to config and shouldn't create any archive tables
-        Request::processRequest('VisitsSummary.get', array('idSite' => 1, 'period' => 'day', 'date' => $dateTime));
-
-        $numericTables = Db::get()->fetchAll('SHOW TABLES like "%archive_numeric_2066_%"');
-
-        $this->assertEmpty($numericTables, 'Archive table for future date found');
-    }
-
-    public function testNoFutureArchiveTablesAreCreatedWithArchiving()
-    {
-        $dateTime = '2066-01-01';
-        Config::getInstance()->General['enable_browser_archiving_triggering'] = 1;
-
-        // request API which should not trigger archiving due to config and shouldn't create any archive tables
-        Request::processRequest('VisitsSummary.get', array('idSite' => 1, 'period' => 'day', 'date' => $dateTime));
-
-        $numericTables = Db::get()->fetchAll('SHOW TABLES like "%archive_numeric_2066_%"');
-
-        $this->assertEmpty($numericTables, 'Archive table for future date found');
-    }
-
-    private function createManyDifferentArchiveBlobs()
-    {
-        $recordName1 = 'Actions_Actions';
-        $recordName2 = 'Actions_Actionsurl';
-
-        $chunk = new Chunk();
-        $chunk0_1 = $chunk->getRecordNameForTableId($recordName1, 0);
-        $chunk0_2 = $chunk->getRecordNameForTableId($recordName2, 0);
-
-        $this->createArchiveBlobEntry('2013-01-01', array(
-            $recordName2 => 'test01'
-        ));
-        $this->createArchiveBlobEntry('2013-01-02', array(
-            $recordName2        => 'test02',
-            $recordName2 . '_1' => 'test1', // testing BC where each subtable was stored seperately
-            $recordName2 . '_2' => 'test2', // testing BC
-            $recordName1        => 'actions_02',
-            $chunk0_1 => serialize(array(1 => 'actionsSubtable1', 2 => 'actionsSubtable2', 5 => 'actionsSubtable5'))
-        ));
-        $this->createArchiveBlobEntry('2013-01-03', array(
-            $recordName2 => 'test03',
-            $chunk0_2 => serialize(array(1 => 'subtable1', 2 => 'subtable2', 5 => 'subtable5')),
-            $recordName1 => 'actions_03',
-            $recordName1 . '_1' => 'actionsTest1',
-            $recordName1 . '_2' => 'actionsTest2'
-        ));
-        $this->createArchiveBlobEntry('2013-01-04', array(
-            $recordName2 => 'test04',
-            $recordName2 . '_5' => 'subtable45',
-            $recordName2 . '_6' => 'subtable6'
-        ));
-        $this->createArchiveBlobEntry('2013-01-06', array(
-            $recordName2 => 'test06',
-            $chunk0_2 => serialize(array())
-        ));
-    }
-
-    private function assertArchiveBlob(PiwikArchive\DataCollection $dataCollection, $date, $expectedBlob)
-    {
-        $dateIndex  = $date . ',' . $date;
-        $dataArrays = $dataCollection->get(1, $dateIndex);
-
-        if (!empty($expectedBlob) && 0 !== reset($expectedBlob)) {
-            $this->assertNotEmpty($dataArrays['_metadata']['ts_archived']);
-            $dataArrays['_metadata']['ts_archived'] = true;
-            unset($dataArrays['_metadata']);
-        }
-
-        $this->assertEquals($expectedBlob, $dataArrays);
-    }
-
-    private function createArchiveBlobEntry($date, $blobs)
-    {
-        $oPeriod = PeriodFactory::makePeriodFromQueryParams('UTC', 'day', $date);
-
-        $segment = new Segment(false, array(1));
-        $params  = new Parameters(new Site(1), $oPeriod, $segment);
-        $writer  = new ArchiveWriter($params, false);
-        $writer->initNewArchive();
-        foreach ($blobs as $name => $blob) {
-            $writer->insertBlobRecord($name, $blob);
-        }
-        $writer->finalizeArchive();
-    }
-
-    private function assertArchiveTablesAreNotEmpty($tableMonth)
-    {
-        $this->assertNotEquals(0, $this->getRangeArchiveTableCount('archive_numeric', $tableMonth));
-    }
-
-    private function assertArchiveTablesAreEmpty($tableMonth)
-    {
-        $this->assertEquals(0, $this->getRangeArchiveTableCount('archive_numeric', $tableMonth));
-        $this->assertEquals(0, $this->getRangeArchiveTableCount('archive_blob', $tableMonth));
-    }
-
-    private function getRangeArchiveTableCount($tableType, $tableMonth)
-    {
-        $table = Common::prefixTable($tableType . '_' . $tableMonth);
-        return Db::fetchOne("SELECT COUNT(*) FROM $table WHERE period = " . Piwik::$idPeriods['range']);
-    }
-
-    private function initiateArchivingForRange()
-    {
-        $archive = $this->getArchive('range');
-        return $archive->getNumeric('nb_visits');
-    }
-
-    private function archiveDataForIndividualDays()
-    {
-        $archive = $this->getArchive('day');
-        return $archive->getNumeric('nb_visits');
-    }
-
-    /**
-     * @return Archive
-     */
-    private function getArchive($period, $day = '2010-03-04,2010-03-07')
-    {
-        /** @noinspection PhpIncompatibleReturnTypeInspection */
-        return Archive::build(self::$fixture->idSite, $period, $day);
-    }
-
-    public function provideContainerConfig()
-    {
-        return [
-            PiwikArchive\ArchiveQueryFactory::class => \DI\object(CustomArchiveQueryFactory::class),
-        ];
-    }
-}
-
-ArchiveTest::$fixture = new OneVisitorTwoVisits();
\ No newline at end of file
diff --git a/tests/PHPUnit/Integration/CronArchiveTest.php b/tests/PHPUnit/Integration/CronArchiveTest.php
index fa568d6b9b..a9b9ee7b93 100644
--- a/tests/PHPUnit/Integration/CronArchiveTest.php
+++ b/tests/PHPUnit/Integration/CronArchiveTest.php
@@ -10,7 +10,9 @@ namespace Piwik\Tests\Integration;
 
 use Piwik\Container\StaticContainer;
 use Piwik\CronArchive;
+use Piwik\DataAccess\ArchiveTableCreator;
 use Piwik\Date;
+use Piwik\Db;
 use Piwik\Plugins\CoreAdminHome\tests\Framework\Mock\API;
 use Piwik\Plugins\SegmentEditor\Model;
 use Piwik\Tests\Framework\Fixture;
@@ -104,7 +106,7 @@ class CronArchiveTest extends IntegrationTestCase
         \Piwik\Tests\Framework\Mock\FakeCliMulti::$specifiedResults = array(
             '/method=API.get/' => serialize(array(array('nb_visits' => 1)))
         );
-        
+
         Fixture::createWebsite('2014-12-12 00:01:02');
         SegmentAPI::getInstance()->add('foo', 'actions>=1', 1, true, true);
         $id = SegmentAPI::getInstance()->add('barb', 'actions>=2', 1, true, true);
@@ -230,12 +232,198 @@ LOG;
         $this->assertContains($expected, $logger->output);
     }
 
+    /**
+     * @dataProvider getTestDataForIsThereAValidArchiveForPeriod
+     */
+    public function test_isThereAValidArchiveForPeriod_modifiesLastNCorrectly($archiveRows, $idSite, $period, $date, $expected)
+    {
+        Date::$now = strtotime('2015-03-04 05:05:05');
+
+        Fixture::createWebsite('2014-12-12 00:01:02');
+
+        $this->insertArchiveData($archiveRows);
+
+        $cronArchive = new CronArchive();
+        $result = $cronArchive->isThereAValidArchiveForPeriod($idSite, $period, $date);
+
+        $this->assertEquals($expected, $result);
+    }
+
+    public function getTestDataForIsThereAValidArchiveForPeriod()
+    {
+        return [
+            // single periods that include today (we don't perform the check and just archive since we assume today is invalid)
+            [
+                [
+                    ['idarchive' => 1, 'idsite' => 1, 'period' => 1, 'date1' => '2015-03-04', 'date2' => '2015-03-04', 'name' => 'done', 'value' => 1],
+                ],
+                1,
+                'day',
+                '2015-03-04',
+                [false, null],
+            ],
+            [
+                [],
+                1,
+                'week',
+                '2015-03-04',
+                [false, null],
+            ],
+            [
+                [],
+                1,
+                'month',
+                '2015-03-04',
+                [false, null],
+            ],
+            [
+                [],
+                1,
+                'year',
+                '2015-03-04',
+                [false, null],
+            ],
+
+            // single periods that do not include today
+            [
+                [
+                    ['idarchive' => 1, 'idsite' => 1, 'period' => 1, 'date1' => '2015-03-02', 'date2' => '2015-03-02', 'name' => 'done', 'value' => 1],
+                ],
+                1,
+                'day',
+                '2015-03-02',
+                [true, '2015-03-02'],
+            ],
+            [
+                [
+                    ['idarchive' => 1, 'idsite' => 1, 'period' => 1, 'date1' => '2015-03-01', 'date2' => '2015-03-01', 'name' => 'done', 'value' => 1],
+                ],
+                1,
+                'day',
+                '2015-03-02',
+                [false, '2015-03-02'],
+            ],
+            [
+                [
+                    ['idarchive' => 1, 'idsite' => 1, 'period' => 2, 'date1' => '2015-02-16', 'date2' => '2015-02-22', 'name' => 'done', 'value' => 1],
+                ],
+                1,
+                'week',
+                '2015-02-17',
+                [true, '2015-02-17'],
+            ],
+            [
+                [
+                    ['idarchive' => 1, 'idsite' => 1, 'period' => 2, 'date1' => '2015-02-15', 'date2' => '2015-02-21', 'name' => 'done', 'value' => 4],
+                ],
+                1,
+                'week',
+                '2015-02-17',
+                [false, '2015-02-17'],
+            ],
+
+            // lastN periods
+            [ // last day invalid, some valid in between
+                [
+                    ['idarchive' => 1, 'idsite' => 1, 'period' => 1, 'date1' => '2015-03-04', 'date2' => '2015-03-04', 'name' => 'done', 'value' => 4],
+                    ['idarchive' => 2, 'idsite' => 1, 'period' => 1, 'date1' => '2015-03-03', 'date2' => '2015-03-04', 'name' => 'done', 'value' => 4],
+                    ['idarchive' => 3, 'idsite' => 1, 'period' => 1, 'date1' => '2015-03-02', 'date2' => '2015-03-04', 'name' => 'done', 'value' => 1],
+                    ['idarchive' => 4, 'idsite' => 1, 'period' => 1, 'date1' => '2015-03-01', 'date2' => '2015-03-04', 'name' => 'done', 'value' => 1],
+                    ['idarchive' => 5, 'idsite' => 1, 'period' => 1, 'date1' => '2015-02-28', 'date2' => '2015-03-04', 'name' => 'done', 'value' => 4],
+                ],
+                1,
+                'day',
+                'last5',
+                [false, 'last5'],
+            ],
+            [ // last two invalid, rest valid
+                [
+                    ['idarchive' => 1, 'idsite' => 1, 'period' => 1, 'date1' => '2015-03-04', 'date2' => '2015-03-04', 'name' => 'done', 'value' => 4],
+                    ['idarchive' => 2, 'idsite' => 1, 'period' => 1, 'date1' => '2015-03-03', 'date2' => '2015-03-03', 'name' => 'done', 'value' => 4],
+                    ['idarchive' => 3, 'idsite' => 1, 'period' => 1, 'date1' => '2015-03-02', 'date2' => '2015-03-02', 'name' => 'done', 'value' => 1],
+                    ['idarchive' => 4, 'idsite' => 1, 'period' => 1, 'date1' => '2015-03-01', 'date2' => '2015-03-01', 'name' => 'done', 'value' => 1],
+                    ['idarchive' => 5, 'idsite' => 1, 'period' => 1, 'date1' => '2015-02-28', 'date2' => '2015-02-28', 'name' => 'done', 'value' => 1],
+                ],
+                1,
+                'day',
+                'last5',
+                [false, 'last2'],
+            ],
+            [ // all valid
+                [
+                    ['idarchive' => 1, 'idsite' => 1, 'period' => 1, 'date1' => '2015-03-04', 'date2' => '2015-03-04', 'name' => 'done', 'value' => 1],
+                    ['idarchive' => 2, 'idsite' => 1, 'period' => 1, 'date1' => '2015-03-03', 'date2' => '2015-03-03', 'name' => 'done', 'value' => 1],
+                    ['idarchive' => 3, 'idsite' => 1, 'period' => 1, 'date1' => '2015-03-02', 'date2' => '2015-03-02', 'name' => 'done', 'value' => 1],
+                    ['idarchive' => 4, 'idsite' => 1, 'period' => 1, 'date1' => '2015-03-01', 'date2' => '2015-03-01', 'name' => 'done', 'value' => 1],
+                    ['idarchive' => 5, 'idsite' => 1, 'period' => 1, 'date1' => '2015-02-28', 'date2' => '2015-02-28', 'name' => 'done', 'value' => 1],
+                ],
+                1,
+                'day',
+                'last5',
+                [false, 'last2'],
+            ],
+            [ // month w/ last 3 invalid, today valid
+                [
+                    ['idarchive' => 1, 'idsite' => 1, 'period' => 3, 'date1' => '2015-03-01', 'date2' => '2015-03-31', 'name' => 'done', 'value' => 1],
+                    ['idarchive' => 2, 'idsite' => 1, 'period' => 3, 'date1' => '2015-02-01', 'date2' => '2015-02-28', 'name' => 'done', 'value' => 4],
+                    ['idarchive' => 3, 'idsite' => 1, 'period' => 3, 'date1' => '2015-01-01', 'date2' => '2015-01-31', 'name' => 'done', 'value' => 4],
+                    ['idarchive' => 4, 'idsite' => 1, 'period' => 3, 'date1' => '2014-12-01', 'date2' => '2014-12-31', 'name' => 'done', 'value' => 1],
+                    ['idarchive' => 5, 'idsite' => 1, 'period' => 3, 'date1' => '2014-11-01', 'date2' => '2014-11-30', 'name' => 'done', 'value' => 1],
+                ],
+                1,
+                'month',
+                'last5',
+                [false, 'last3'],
+            ],
+
+            // range periods
+            [ // includes today
+                [
+                    ['idarchive' => 5, 'idsite' => 1, 'period' => 5, 'date1' => '2015-03-02', 'date2' => '2015-03-04', 'name' => 'done', 'value' => 1],
+                ],
+                1,
+                'range',
+                '2015-03-02,2015-03-04',
+                [false, null],
+            ],
+            [ // does not include today, invalid
+                [
+                    ['idarchive' => 1, 'idsite' => 1, 'period' => 5, 'date1' => '2015-03-01', 'date2' => '2015-03-03', 'name' => 'done', 'value' => 4],
+                ],
+                1,
+                'range',
+                '2015-03-01,2015-03-03',
+                [false, '2015-03-01,2015-03-03'],
+            ],
+            [ // does not include today, valid
+                [
+                    ['idarchive' => 1, 'idsite' => 1, 'period' => 5, 'date1' => '2015-03-01', 'date2' => '2015-03-03', 'name' => 'done', 'value' => 1],
+                ],
+                1,
+                'range',
+                '2015-03-01,2015-03-03',
+                [true, '2015-03-01,2015-03-03'],
+            ],
+        ];
+    }
+
     public function provideContainerConfig()
     {
         return array(
             'Piwik\CliMulti' => \DI\object('Piwik\Tests\Framework\Mock\FakeCliMulti')
         );
     }
+
+    private function insertArchiveData($archiveRows)
+    {
+        foreach ($archiveRows as $row) {
+            $table = ArchiveTableCreator::getNumericTable(Date::factory($row['date1']));
+
+            $tsArchived = isset($row['ts_archived']) ? $row['ts_archived'] : Date::now()->getDatetime();
+            Db::query("INSERT INTO `$table` (idarchive, idsite, period, date1, date2, `name`, `value`, ts_archived) VALUES (?, ?, ?, ?, ?, ?, ?, ?)",
+                [$row['idarchive'], $row['idsite'], $row['period'], $row['date1'], $row['date2'], $row['name'], $row['value'], $tsArchived]);
+        }
+    }
 }
 
 class TestCronArchive extends CronArchive
diff --git a/tests/PHPUnit/Integration/DataAccess/ArchiveSelectorTest.php b/tests/PHPUnit/Integration/DataAccess/ArchiveSelectorTest.php
new file mode 100644
index 0000000000..6143534a87
--- /dev/null
+++ b/tests/PHPUnit/Integration/DataAccess/ArchiveSelectorTest.php
@@ -0,0 +1,212 @@
+<?php
+/**
+ * Piwik - free/libre analytics platform
+ *
+ * @link https://matomo.org
+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
+ *
+ */
+
+namespace Piwik\Tests\Integration\DataAccess;
+
+
+use Piwik\ArchiveProcessor\Parameters;
+use Piwik\DataAccess\ArchiveSelector;
+use Piwik\DataAccess\ArchiveTableCreator;
+use Piwik\Date;
+use Piwik\Db;
+use Piwik\Period\Factory;
+use Piwik\Segment;
+use Piwik\Site;
+use Piwik\Tests\Framework\Fixture;
+use Piwik\Tests\Framework\TestCase\IntegrationTestCase;
+
+class ArchiveSelectorTest extends IntegrationTestCase
+{
+    protected static function configureFixture($fixture)
+    {
+        parent::configureFixture($fixture);
+        $fixture->createSuperUser = true;
+    }
+
+    /**
+     * @dataProvider getTestDataForGetArchiveIdAndVisits
+     */
+    public function test_getArchiveIdAndVisits_returnsCorrectResult($archiveRows, $segment, $minDateProcessed, $includeInvalidated, $expected)
+    {
+        Fixture::createWebsite('2010-02-02 00:00:00');
+
+        $this->insertArchiveData($archiveRows);
+
+        $params = new Parameters(new Site(1), Factory::build('day', '2019-10-05'), new Segment($segment, [1]));
+        $result = ArchiveSelector::getArchiveIdAndVisits($params, $minDateProcessed, $includeInvalidated);
+        $this->assertEquals($expected, $result);
+    }
+
+    private function insertArchiveData($archiveRows)
+    {
+        $table = ArchiveTableCreator::getNumericTable(Date::factory('2019-10-01 12:13:14'));
+        foreach ($archiveRows as $row) {
+            $tsArchived = isset($row['ts_archived']) ? $row['ts_archived'] : Date::now()->getDatetime();
+            Db::query("INSERT INTO `$table` (idarchive, idsite, period, date1, date2, `name`, `value`, ts_archived) VALUES (?, ?, ?, ?, ?, ?, ?, ?)",
+                [$row['idarchive'], $row['idsite'], $row['period'], $row['date1'], $row['date2'], $row['name'], $row['value'], $tsArchived]);
+        }
+    }
+
+    public function getTestDataForGetArchiveIdAndVisits()
+    {
+        $minDateProcessed = Date::factory('2020-03-04 00:00:00')->subSeconds(900)->getDatetime();
+        return [
+            // no archive data found
+            [ // nothing in the db
+                [],
+                '',
+                $minDateProcessed,
+                true,
+                [false, false, false, false],
+            ],
+            [
+                [
+                    ['idarchive' => 1, 'idsite' => 1, 'period' => 2, 'date1' => '2019-10-05', 'date2' => '2019-10-05', 'name' => 'done', 'value' => 1],
+                    ['idarchive' => 2, 'idsite' => 1, 'period' => 1, 'date1' => '2019-10-06', 'date2' => '2019-10-06', 'name' => 'done', 'value' => 1],
+                    ['idarchive' => 3, 'idsite' => 2, 'period' => 1, 'date1' => '2019-10-05', 'date2' => '2019-10-05', 'name' => 'done', 'value' => 1],
+                ],
+                '',
+                $minDateProcessed,
+                true,
+                [false, false, false, false],
+            ],
+
+            // value is not valid
+            [
+                [
+                    ['idarchive' => 1, 'idsite' => 1, 'period' => 1, 'date1' => '2019-10-05', 'date2' => '2019-10-05', 'name' => 'done', 'value' => 4],
+                    ['idarchive' => 2, 'idsite' => 1, 'period' => 1, 'date1' => '2019-10-05', 'date2' => '2019-10-05', 'name' => 'done', 'value' => 2],
+                    ['idarchive' => 3, 'idsite' => 1, 'period' => 1, 'date1' => '2019-10-05', 'date2' => '2019-10-05', 'name' => 'done', 'value' => 3],
+                    ['idarchive' => 4, 'idsite' => 1, 'period' => 1, 'date1' => '2019-10-05', 'date2' => '2019-10-05', 'name' => 'done', 'value' => 99],
+                ],
+                '',
+                $minDateProcessed,
+                false,
+                [false, false, false, true],
+            ],
+            [
+                [
+                    ['idarchive' => 1, 'idsite' => 1, 'period' => 1, 'date1' => '2019-10-05', 'date2' => '2019-10-05', 'name' => 'done', 'value' => 4],
+                    ['idarchive' => 1, 'idsite' => 1, 'period' => 1, 'date1' => '2019-10-05', 'date2' => '2019-10-05', 'name' => 'nb_visits', 'value' => 20],
+                    ['idarchive' => 1, 'idsite' => 1, 'period' => 1, 'date1' => '2019-10-05', 'date2' => '2019-10-05', 'name' => 'nb_visits_converted', 'value' => 40],
+                    ['idarchive' => 2, 'idsite' => 1, 'period' => 1, 'date1' => '2019-10-05', 'date2' => '2019-10-05', 'name' => 'done', 'value' => 2],
+                    ['idarchive' => 3, 'idsite' => 1, 'period' => 1, 'date1' => '2019-10-05', 'date2' => '2019-10-05', 'name' => 'done', 'value' => 3],
+                    ['idarchive' => 4, 'idsite' => 1, 'period' => 1, 'date1' => '2019-10-05', 'date2' => '2019-10-05', 'name' => 'done', 'value' => 99],
+                ],
+                '',
+                $minDateProcessed,
+                false,
+                [false, 0, 0, true],
+            ],
+            [
+                [
+                    ['idarchive' => 1, 'idsite' => 1, 'period' => 1, 'date1' => '2019-10-05', 'date2' => '2019-10-05', 'name' => 'done.VisitsSummary', 'value' => 4],
+                    ['idarchive' => 1, 'idsite' => 1, 'period' => 1, 'date1' => '2019-10-05', 'date2' => '2019-10-05', 'name' => 'nb_visits', 'value' => 20],
+                    ['idarchive' => 1, 'idsite' => 1, 'period' => 1, 'date1' => '2019-10-05', 'date2' => '2019-10-05', 'name' => 'nb_visits_converted', 'value' => 40],
+                ],
+                '',
+                $minDateProcessed,
+                false,
+                [false, 20, 40, true],
+            ],
+            [
+                [
+                    ['idarchive' => 1, 'idsite' => 1, 'period' => 1, 'date1' => '2019-10-05', 'date2' => '2019-10-05', 'name' => 'done.VisitsSummary', 'value' => 1],
+                    ['idarchive' => 1, 'idsite' => 1, 'period' => 1, 'date1' => '2019-10-05', 'date2' => '2019-10-05', 'name' => 'nb_visits', 'value' => 30],
+                    ['idarchive' => 1, 'idsite' => 1, 'period' => 1, 'date1' => '2019-10-05', 'date2' => '2019-10-05', 'name' => 'nb_visits_converted', 'value' => 50],
+                ],
+                '',
+                $minDateProcessed,
+                false,
+                [false, 30, 50, true],
+            ],
+            [
+                [
+                    ['idarchive' => 1, 'idsite' => 1, 'period' => 1, 'date1' => '2019-10-05', 'date2' => '2019-10-05', 'name' => 'done.VisitsSummary', 'value' => 1],
+                    ['idarchive' => 1, 'idsite' => 1, 'period' => 1, 'date1' => '2019-10-05', 'date2' => '2019-10-05', 'name' => 'nb_visits', 'value' => 30],
+                    ['idarchive' => 1, 'idsite' => 1, 'period' => 1, 'date1' => '2019-10-05', 'date2' => '2019-10-05', 'name' => 'nb_visits_converted', 'value' => 50],
+                    ['idarchive' => 2, 'idsite' => 1, 'period' => 1, 'date1' => '2019-10-05', 'date2' => '2019-10-05', 'name' => 'done.VisitsSummary', 'value' => 4],
+                ],
+                '',
+                $minDateProcessed,
+                false,
+                [false, 0, 0, true],
+            ],
+
+            // archive is too old
+            [
+                [
+                    ['idarchive' => 1, 'idsite' => 1, 'period' => 1, 'date1' => '2019-10-05', 'date2' => '2019-10-05', 'name' => 'done', 'value' => 1,
+                        'ts_archived' => Date::factory($minDateProcessed)->subSeconds(1)->getDatetime()],
+                ],
+                '',
+                $minDateProcessed,
+                false,
+                [false, false, false, true],
+            ],
+            [
+                [
+                    ['idarchive' => 1, 'idsite' => 1, 'period' => 1, 'date1' => '2019-10-05', 'date2' => '2019-10-05', 'name' => 'done', 'value' => 1,
+                        'ts_archived' => Date::factory($minDateProcessed)->subSeconds(1)->getDatetime()],
+                    ['idarchive' => 1, 'idsite' => 1, 'period' => 1, 'date1' => '2019-10-05', 'date2' => '2019-10-05', 'name' => 'nb_visits', 'value' => 1,
+                        'ts_archived' => Date::factory($minDateProcessed)->subSeconds(1)->getDatetime()],
+                ],
+                '',
+                $minDateProcessed,
+                false,
+                [false, 1, false, true],
+            ],
+
+            // no archive done flags, but metric
+            [
+                [
+                    ['idarchive' => 1, 'idsite' => 1, 'period' => 1, 'date1' => '2019-10-05', 'date2' => '2019-10-05', 'name' => 'nb_visits_converted', 'value' => 10],
+                    ['idarchive' => 1, 'idsite' => 1, 'period' => 1, 'date1' => '2019-10-05', 'date2' => '2019-10-05', 'name' => 'nb_visits', 'value' => 1],
+                ],
+                '',
+                $minDateProcessed,
+                false,
+                [false, false, false, true],
+            ],
+            [
+                [
+                    ['idarchive' => 1, 'idsite' => 1, 'period' => 1, 'date1' => '2019-10-05', 'date2' => '2019-10-05', 'name' => 'nb_visits_converted', 'value' => 10],
+                    ['idarchive' => 1, 'idsite' => 1, 'period' => 1, 'date1' => '2019-10-05', 'date2' => '2019-10-05', 'name' => 'nb_visits', 'value' => 3],
+                    ['idarchive' => 2, 'idsite' => 1, 'period' => 1, 'date1' => '2019-10-05', 'date2' => '2019-10-05', 'name' => 'nb_visits', 'value' => 5],
+                ],
+                '',
+                $minDateProcessed,
+                false,
+                [false, false, false, true],
+            ],
+
+            // archive exists and is usable
+            [
+                [
+                    ['idarchive' => 1, 'idsite' => 1, 'period' => 1, 'date1' => '2019-10-05', 'date2' => '2019-10-05', 'name' => 'done', 'value' => 1],
+                ],
+                '',
+                $minDateProcessed,
+                false,
+                [1, false, false, true],
+            ],
+            [
+                [
+                    ['idarchive' => 1, 'idsite' => 1, 'period' => 1, 'date1' => '2019-10-05', 'date2' => '2019-10-05', 'name' => 'done', 'value' => 1],
+                    ['idarchive' => 1, 'idsite' => 1, 'period' => 1, 'date1' => '2019-10-05', 'date2' => '2019-10-05', 'name' => 'nb_visits', 'value' => 5],
+                    ['idarchive' => 1, 'idsite' => 1, 'period' => 1, 'date1' => '2019-10-05', 'date2' => '2019-10-05', 'name' => 'nb_visits_converted', 'value' => 10],
+                ],
+                '',
+                $minDateProcessed,
+                false,
+                [1, 5, 10, true],
+            ],
+        ];
+    }
+}
\ No newline at end of file
diff --git a/tests/PHPUnit/Integration/Tracker/VisitTest.php b/tests/PHPUnit/Integration/Tracker/VisitTest.php
index 0ccc7f4953..ccd73ce15f 100644
--- a/tests/PHPUnit/Integration/Tracker/VisitTest.php
+++ b/tests/PHPUnit/Integration/Tracker/VisitTest.php
@@ -392,7 +392,7 @@ class VisitTest extends IntegrationTestCase
         $currentActionTime = Date::today()->getDatetime();
         $idsite = API::getInstance()->addSite('name', 'http://piwik.net/');
 
-        $expectedRemembered = array(Date::today()->toString() => [1]);
+        $expectedRemembered = array();
 
         $this->assertRememberedArchivedReportsThatShouldBeInvalidated($idsite, $currentActionTime, $expectedRemembered);
     }
@@ -414,7 +414,6 @@ class VisitTest extends IntegrationTestCase
         // The double-handling below is needed to work around weird behaviour when UTC and UTC+5 are different dates
         // Example: 4:32am on 1 April in UTC+5 is 11:32pm on 31 March in UTC
         $midnight = Date::factoryInTimezone('today', 'UTC+5')->setTimezone('UTC+5');
-        $today = Date::factoryInTimezone('today', 'UTC+5');
 
         $oneHourAfterMidnight = $midnight->addHour(1)->getDatetime();
         $oneHourBeforeMidnight = $midnight->subHour(1)->getDatetime();
@@ -428,11 +427,10 @@ class VisitTest extends IntegrationTestCase
 
         $expectedRemembered = array(
             substr($oneHourAfterMidnight, 0, 10) => array($idsite),
-            $today->toString() => [$idsite],
         );
 
         // if website timezone was von considered both would be today (expected = array())
-        $this->assertRememberedArchivedReportsThatShouldBeInvalidated($idsite, $oneHourAfterMidnight, array($today->toString() => [$idsite]));
+        $this->assertRememberedArchivedReportsThatShouldBeInvalidated($idsite, $oneHourAfterMidnight, array());
         $this->assertRememberedArchivedReportsThatShouldBeInvalidated($idsite, $oneHourBeforeMidnight, $expectedRemembered);
     }
 
diff --git a/tests/PHPUnit/Integration/TrackerTest.php b/tests/PHPUnit/Integration/TrackerTest.php
index 574aa3312c..94f2feefa0 100644
--- a/tests/PHPUnit/Integration/TrackerTest.php
+++ b/tests/PHPUnit/Integration/TrackerTest.php
@@ -369,7 +369,7 @@ class TrackerTest extends IntegrationTestCase
     public function test_archiveInvalidation_differentServerAndWebsiteTimezones()
     {
         // Server timezone is UTC
-        ini_set('date.timezone', 'America/New_York');
+        ini_set('date.timezone', 'UTC');
 
         // Website timezone is New York
         $idSite = Fixture::createWebsite('2014-01-01 00:00:00', 0, false, false,
@@ -385,12 +385,8 @@ class TrackerTest extends IntegrationTestCase
         $this->request->setCurrentTimestamp(Date::$now);
         $this->tracker->trackRequest($this->request);
 
-        $keys = Option::getLike('%report_to_invalidate_2_2019-04-02%');
-        $this->assertCount(1, $keys);
-        $key = key($keys);
-        $this->assertStringEndsWith('report_to_invalidate_2_2019-04-02_' . getmypid(), $key);
-        // make sure today archives are also invalidated
-        $this->assertEquals([$key => '1'], $keys);
+        // make sure today archives are not invalidated
+        $this->assertEquals([], Option::getLike('report_to_invalidate_2_2019-04-02%'));
     }
 
     public function test_TrackingNewVisitOfKnownVisitor()
diff --git a/tests/PHPUnit/System/OneVisitorOneWebsiteSeveralDaysDateRangeArchivingTest.php b/tests/PHPUnit/System/OneVisitorOneWebsiteSeveralDaysDateRangeArchivingTest.php
index 2a005ab980..9268944755 100644
--- a/tests/PHPUnit/System/OneVisitorOneWebsiteSeveralDaysDateRangeArchivingTest.php
+++ b/tests/PHPUnit/System/OneVisitorOneWebsiteSeveralDaysDateRangeArchivingTest.php
@@ -7,8 +7,11 @@
  */
 namespace Piwik\Tests\System;
 
+use Piwik\Archive\ArchivePurger;
 use Piwik\Archive\Chunk;
 use Piwik\Common;
+use Piwik\Container\StaticContainer;
+use Piwik\Date;
 use Piwik\Db;
 use Piwik\Piwik;
 use Piwik\Tests\Framework\TestCase\SystemTestCase;
@@ -23,6 +26,9 @@ use Piwik\Tests\Fixtures\VisitsOverSeveralDays;
  */
 class OneVisitorOneWebsiteSeveralDaysDateRangeArchivingTest extends SystemTestCase
 {
+    /**
+     * @var VisitsOverSeveralDays
+     */
     public static $fixture = null; // initialized below test definition
 
     public static function getOutputPrefix()
@@ -101,6 +107,11 @@ class OneVisitorOneWebsiteSeveralDaysDateRangeArchivingTest extends SystemTestCa
      */
     public function test_checkArchiveRecords_whenPeriodIsRange()
     {
+        $archivePurger = StaticContainer::get(ArchivePurger::class);
+        foreach (self::$fixture->dateTimes as $date) {
+            $archivePurger->purgeInvalidatedArchivesFrom(Date::factory($date));
+        }
+
         // we expect 5 blobs for Actions plugins, because flat=1 or expanded=1 was not set
         // so we only archived the parent table
         $expectedActionsBlobs = 5;
diff --git a/tests/PHPUnit/System/OneVisitorTwoVisitsTest.php b/tests/PHPUnit/System/OneVisitorTwoVisitsTest.php
index 96a81bea52..1670177ba9 100644
--- a/tests/PHPUnit/System/OneVisitorTwoVisitsTest.php
+++ b/tests/PHPUnit/System/OneVisitorTwoVisitsTest.php
@@ -78,6 +78,7 @@ class OneVisitorTwoVisitsTest extends SystemTestCase
         foreach ($bulkUrls as &$url) {
             $url = urlencode($url);
         }
+
         return array(
             array('all', array('idSite' => $idSite,
                                'date' => $dateTime,
diff --git a/tests/PHPUnit/System/TwoVisitorsTwoWebsitesDifferentDaysConversionsTest.php b/tests/PHPUnit/System/TwoVisitorsTwoWebsitesDifferentDaysConversionsTest.php
index cf22ba95cb..1eadd1b9cb 100644
--- a/tests/PHPUnit/System/TwoVisitorsTwoWebsitesDifferentDaysConversionsTest.php
+++ b/tests/PHPUnit/System/TwoVisitorsTwoWebsitesDifferentDaysConversionsTest.php
@@ -8,8 +8,10 @@
 namespace Piwik\Tests\System;
 
 use Piwik\Archive;
+use Piwik\Archive\ArchivePurger;
 use Piwik\Cache;
 use Piwik\Container\StaticContainer;
+use Piwik\Date;
 use Piwik\Segment;
 use Piwik\Tests\Framework\TestCase\SystemTestCase;
 use Piwik\Tests\Fixtures\TwoSitesTwoVisitorsDifferentDays;
@@ -139,10 +141,13 @@ class TwoVisitorsTwoWebsitesDifferentDaysConversionsTest extends SystemTestCase
         );
     }
 
-    // TODO: this test should be in an integration test for Piwik\Archive. setup code for getting metrics from different
-    //       plugins is non-trivial, so not done now.
     public function test_Archive_getNumeric_shouldInvalidateRememberedReportsOncePerRequestIfNeeded()
     {
+        /* TODO: remove this test and replace w/ integration test for invalidation in archive.php workflow
+
+        $archivePurger = StaticContainer::get(ArchivePurger::class);
+        $archivePurger->purgeInvalidatedArchivesFrom(Date::factory(self::$fixture->dateTime));
+
         // Tests that getting a visits summary metric (nb_visits) & a Goal's metric (Goal_revenue)
         // at the same time works.
         $dateTimeRange = '2010-01-03,2010-01-06';
@@ -160,10 +165,6 @@ class TwoVisitorsTwoWebsitesDifferentDaysConversionsTest extends SystemTestCase
             $result
         );
 
-        $cache = Cache::getTransientCache();
-        $this->assertEquals(array(self::$fixture->idSite1, self::$fixture->idSite2),
-                            $cache->fetch('Archive.SiteIdsOfRememberedReportsInvalidated'));
-
         $invalidator = StaticContainer::get('Piwik\Archive\ArchiveInvalidator');
 
         self::$fixture->trackVisits();
@@ -187,9 +188,6 @@ class TwoVisitorsTwoWebsitesDifferentDaysConversionsTest extends SystemTestCase
         // make sure the caching in archive::get() worked and they are still to be invalidated
         $this->assertCount(10, $invalidator->getRememberedArchivedReportsThatShouldBeInvalidated());
 
-        // now we force to actually invalidate archived reports again and then archive will be rebuilt for requsted siteId = 1
-        $cache->delete('Archive.SiteIdsOfRememberedReportsInvalidated');
-
         $archive = Archive::build($idSite1, 'range', $dateTimeRange);
         $result = $archive->getNumeric($columns);
 
@@ -205,6 +203,7 @@ class TwoVisitorsTwoWebsitesDifferentDaysConversionsTest extends SystemTestCase
             ),
             $result
         );
+        */
     }
 
     public static function getOutputPrefix()
diff --git a/tests/PHPUnit/System/TwoVisitsWithCustomVariablesSegmentMatchVisitorTypeTest.php b/tests/PHPUnit/System/TwoVisitsWithCustomVariablesSegmentMatchVisitorTypeTest.php
index 675fb409b9..75ba96d7de 100644
--- a/tests/PHPUnit/System/TwoVisitsWithCustomVariablesSegmentMatchVisitorTypeTest.php
+++ b/tests/PHPUnit/System/TwoVisitsWithCustomVariablesSegmentMatchVisitorTypeTest.php
@@ -7,9 +7,12 @@
  */
 namespace Piwik\Tests\System;
 
+use Piwik\Archive\ArchivePurger;
 use Piwik\Archive\Chunk;
 use Piwik\Common;
 use Piwik\Archive\ArchiveInvalidator;
+use Piwik\Container\StaticContainer;
+use Piwik\Date;
 use Piwik\Db;
 use Piwik\Tests\Framework\TestCase\SystemTestCase;
 use Piwik\Tests\Fixtures\TwoVisitsWithCustomVariables;
@@ -64,6 +67,9 @@ class TwoVisitsWithCustomVariablesSegmentMatchVisitorTypeTest extends SystemTest
      */
     public function testCheck()
     {
+        $archivePurger = StaticContainer::get(ArchivePurger::class);
+        $archivePurger->purgeInvalidatedArchivesFrom(Date::factory(self::$fixture->dateTime));
+
         // ----------------------------------------------
         // Implementation Checks
         // ----------------------------------------------
@@ -90,9 +96,10 @@ class TwoVisitsWithCustomVariablesSegmentMatchVisitorTypeTest extends SystemTest
             'archive_blob_2009_12'    => 20,
             // 7 metrics,
             // 2 Referrer metrics (Referrers_distinctSearchEngines/Referrers_distinctKeywords),
-            // 6 done flag (referrers, CustomVar, VisitsSummary), 3 for period = 1 and 3 for period = 2
+            // 5 done flag (referrers, VisitsSummary), 2 for period = 1 and 3 for period = 2
             // X * 2 segments
-            'archive_numeric_2009_12' => (6 + 2 + 3 + 3) * 2,
+            // + 1 done flag archive for CustomVar
+            'archive_numeric_2009_12' => (5 + 2 + 3 + 3) * 2 + 1,
         );
         foreach ($tests as $table => $expectedRows) {
             $sql = "SELECT count(*) FROM " . Common::prefixTable($table);
diff --git a/tests/PHPUnit/Unit/PeriodTest.php b/tests/PHPUnit/Unit/PeriodTest.php
index a46d6a09b7..bacee84d61 100644
--- a/tests/PHPUnit/Unit/PeriodTest.php
+++ b/tests/PHPUnit/Unit/PeriodTest.php
@@ -251,4 +251,29 @@ class PeriodTest extends \PHPUnit_Framework_TestCase
             return array($period->getLabel(), $period->getRangeString());
         }, $periods);
     }
+
+    /**
+     * @dataProvider getTestDataForIsDateInPeriod
+     */
+    public function test_isDateInPeriod($date, $period, $periodDate, $expected)
+    {
+        $date = Date::factory($date);
+        $period = Period\Factory::build($period, $periodDate);
+
+        $actual = $period->isDateInPeriod($date);
+        $this->assertEquals($expected, $actual);
+    }
+
+    public function getTestDataForIsDateInPeriod()
+    {
+        return [
+            ['2014-02-03 00:00:00', 'day', '2014-02-03 03:04:05', true],
+            ['2014-02-03 00:00:00', 'week', '2014-02-03 03:04:05', true],
+            ['2014-02-03 00:00:00', 'month', '2014-02-03 03:04:05', true],
+            ['2014-02-02 23:59:59', 'day', '2014-02-03 03:04:05', false],
+            ['2014-01-31 23:59:59', 'month', '2014-02-03 03:04:05', false],
+            ['2014-03-01 00:00:00', 'month', '2014-02-03 03:04:05', false],
+            ['2014-03-31 23:59:59', 'month', '2014-03-03 03:04:05', true],
+        ];
+    }
 }
\ No newline at end of file