allow invalidating plugin archives only and archiving past data for plugins (#15889)

* Adding initial new code for cron archive rewrite.

* first pass at removing unused CronArchive code.

* unfinished commit

* fill out archiveinvalidator code

* getting some tests to pass

* unfinished commit

* fixing part of test

* Another test fix.

* another sql change

* fix broken merge or something else that went wrong

* Couple more fixes and extra logs.

* Fixing enough issues to get core archive command to run completely.

* Fix and log change.

* Fixed more segment/test related issues for CronArchiveTest. Includes optimization for no visits for period + segment process from handling.

* another optimization and possible build fix

* no visit optimization

* test fix

* Implement archiving_custom_ranges logic w/ queue based implementation

* fixes to get archivecrontest to work

* add logic to invalidate today period

* fix optimization and some tests

* Fixing more tests.

* Fixing more tests

* debug travis failure

* more test fixes

* more test fixes, removing more unneeded code, handling some TODOs

* Handle more TODOs including creating ArchiveFilter class for some cli options.

* tests and todos

* idarchives are specific to table + start on archivefilter tests

* one test

* more TODOs and tests

* more tests and todo taken care of

* handle more todos

* fixing more tests

* fix comment

* make sure autoarchiving is enabled for segments when cron archive picks them up

* Fixing test.

* apply more pr feedback

* order by date1 asc

* quick refactor

* use batch insert instead of createDummyArchives

* apply rest of pr feedback

* add removed events, add new test, fix an issue (when deleting idarchives older than do not lump all segments together).

* re-add fixed/shared siteids

* fix tests

* incomplete commit

* Insert archive entries into archive_invalidations table.

* Use invalidations table in core:archive and get ArchiveCronTest to pass.

* fixing some tests

* debugging travis

* fix more tests & remove DONE_IN_PROGRESS which is no longer used.

* fix more tests

* Allow forcing plugin specific archive in core:archive.

* When querying from archive data use all available archives including "all" archives and plugin specific archives.

* Adding some code for invalidating specific plugin archives.

* Get archive invalidation test to pass.

* add plugin capability to invalidate command

* Handle plugin only archives in core:archive.

* Add Archive test and get ArchiveCronTest to pass.

* update some expected files

* Fix some more tests.

* incomplete commit

* allow invalidating individual reports

* adding more API for DONE_PARTIAL support

* get archivecrontest to pass

* add archive processor tests

* fix some test randomnes

* when purging keep latest partial archives if there is no newer whole archive

* add rearchivereport method + some unfinished tests

* Add archiveReports API method, fix race condition in test, when archiving single report, always ignore inserting other reports.

* require archivers to handle partial archives themselves entirely instead of trying to do it automatically and allow requested report to be any string

* couple fixes

* Use core config option for last N montsh to invalidate.

* Add test for ArchiveSelector method.

* Ignore archives w/ deactivated plugins.

* Refactor queue looping into new QueueConsumer class.

* apply more review feedback + another fix

* invalidate segments too in reArchiveReport w/ etsts

* remove DONE_IN_PROGRESS, no longer used. use new status in query and add queue consumer test.

* forgot to add file

* delete old unneeded archives when finalizing a new one.

* tweak invalidation archive description

* add plugin archiving tests and get them to pass

* fix test

* many fixes

* fix another test

* update expected test files

* fix more tests

* last test fixes hopefully

* tweak log

* In case a column already exists, do not try to add it in an AddColumns migration or the entire migration will fail and no columns will be added.

* try to fix tests again

* fix again?

* apply some review feedback + fix test

* fix test

* fix another test

* couple fixes

* Remove extra param.

* apply pr feedback

* check for usable archive before invalidating and before initiating archiving

* fixing tests

* fixing tests

* fixing tests

* fix another test issue

* fix archiveinvalidator test

* fix one test and debug another

* more debugging

* fix test

* use twig

* remove no longer needed change

* add back previous logic

* fix tracking is not working

* apply pr feedback and add tests

* fixing tests

* update submodule

* debugging random travis failure

* update test

* more debugging

* more debugging

* another attempt at debugging

* Lets try this fix

* trying to fix the build

* debug

* simpler test

* fix test

* fix test

* fix test

* fix test

* fix test failure

* update screenshots

* update screenshots

Co-authored-by: Thomas Steur <tsteur@users.noreply.github.com>

22 files changed, 1059 insertions, 469 deletions

diff --git a/core/Access.php b/core/Access.php
index b7adcfab33..87043193ad 100644
--- a/core/Access.php
+++ b/core/Access.php
@@ -673,7 +673,7 @@ class Access
 
         try {
             $result = $function();
-        } catch (Exception $ex) {
+        } catch (\Throwable $ex) {
             $access->setSuperUserAccess($isSuperUser);
             if ($shouldResetLogin) {
                 $access->login = null;
diff --git a/core/Archive.php b/core/Archive.php
index 545c4ff757..8f3a5fdbfa 100644
--- a/core/Archive.php
+++ b/core/Archive.php
@@ -14,6 +14,7 @@ use Piwik\Archive\Parameters;
 use Piwik\ArchiveProcessor\Rules;
 use Piwik\Container\StaticContainer;
 use Piwik\DataAccess\ArchiveSelector;
+use Piwik\Plugins\CoreAdminHome\API;
 
 /**
  * The **Archive** class is used to query cached analytics statistics
@@ -524,7 +525,7 @@ class Archive implements ArchiveQuery
         // then we have the archive IDs in $this->idarchives)
         $doneFlags     = array();
         $archiveGroups = array();
-        foreach ($plugins as $plugin) {
+        foreach (array_merge($plugins, ['all']) as $plugin) {
             $doneFlag = $this->getDoneStringForPlugin($plugin, $this->params->getIdSites());
 
             $doneFlags[$doneFlag] = true;
@@ -537,12 +538,13 @@ class Archive implements ArchiveQuery
                 $archiveGroups[] = $archiveGroup;
             }
 
-            $globalDoneFlag = Rules::getDoneFlagArchiveContainsAllPlugins($this->params->getSegment());
-            if ($globalDoneFlag !== $doneFlag) {
-                $doneFlags[$globalDoneFlag] = true;
-            }
+            $doneFlag = Rules::getDoneFlagArchiveContainsOnePlugin($this->params->getSegment(), $plugin);
+            $doneFlags[$doneFlag] = true;
         }
 
+        $globalDoneFlag = Rules::getDoneFlagArchiveContainsAllPlugins($this->params->getSegment());
+        $doneFlags[$globalDoneFlag] = true;
+
         $archiveGroups = array_unique($archiveGroups);
 
         // cache id archives for plugins we haven't processed yet
@@ -583,7 +585,6 @@ class Archive implements ArchiveQuery
                     && Common::getRequestVar('skipArchiveSegmentToday', 0, 'int')
                     && $period->getDateStart()->toString() === Date::factory('now', $site->getTimezone())->toString()
                 ) {
-
                     Log::debug("Skipping archive %s for %s as segment today is disabled", $period->getLabel(), $period->getPrettyString());
                     continue;
                 }
@@ -632,6 +633,8 @@ class Archive implements ArchiveQuery
         foreach ($idarchivesByReport as $doneFlag => $idarchivesByDate) {
             foreach ($idarchivesByDate as $dateRange => $idarchives) {
                 foreach ($idarchives as $idarchive) {
+                    // idarchives selected can include all plugin archives, plugin specific archives and partial report
+                    // archives. only the latest data in all of these archives will be selected.
                     $this->idarchives[$doneFlag][$dateRange][] = $idarchive;
                 }
             }
@@ -785,25 +788,33 @@ class Archive implements ArchiveQuery
      */
     private function prepareArchive(array $archiveGroups, Site $site, Period $period)
     {
-        // if cron archiving is running, we will invalidate in CronArchive, not here
-        $invalidateBeforeArchiving = !SettingsServer::isArchivePhpTriggered();
+        $coreAdminHomeApi = API::getInstance();
 
-        $parameters = new ArchiveProcessor\Parameters($site, $period, $this->params->getSegment());
-        $archiveLoader = new ArchiveProcessor\Loader($parameters, $invalidateBeforeArchiving);
+        $requestedReport = null;
+        if (SettingsServer::isArchivePhpTriggered()) {
+            $requestedReport = Common::getRequestVar('requestedReport', '', 'string');
+        }
 
         $periodString = $period->getRangeString();
+        $periodDateStr = $period->getLabel() == 'range' ? $periodString : $period->getDateStart()->toString();
 
         $idSites = array($site->getId());
-        
+
         // process for each plugin as well
         foreach ($archiveGroups as $plugin) {
             $doneFlag = $this->getDoneStringForPlugin($plugin, $idSites);
             $this->initializeArchiveIdCache($doneFlag);
 
-            $idArchive = $archiveLoader->prepareArchive($plugin);
+            $prepareResult = $coreAdminHomeApi->archiveReports(
+                $site->getId(), $period->getLabel(), $periodDateStr, $this->params->getSegment()->getString(),
+                $plugin, $requestedReport);
 
-            if ($idArchive) {
-                $this->idarchives[$doneFlag][$periodString][] = $idArchive;
+            if (!empty($prepareResult)
+                && !empty($prepareResult['idarchives'])
+            ) {
+                foreach ($prepareResult['idarchives'] as $idArchive) {
+                    $this->idarchives[$doneFlag][$periodString][] = $idArchive;
+                }
             }
         }
     }
diff --git a/core/Archive/ArchiveInvalidator.php b/core/Archive/ArchiveInvalidator.php
index 392aae29cf..709a79e60f 100644
--- a/core/Archive/ArchiveInvalidator.php
+++ b/core/Archive/ArchiveInvalidator.php
@@ -12,6 +12,9 @@ namespace Piwik\Archive;
 use Piwik\Archive\ArchiveInvalidator\InvalidationResult;
 use Piwik\ArchiveProcessor\ArchivingStatus;
 use Piwik\ArchiveProcessor\Loader;
+use Piwik\Config;
+use Piwik\Container\StaticContainer;
+use Piwik\CronArchive\SegmentArchiving;
 use Piwik\DataAccess\ArchiveTableCreator;
 use Piwik\DataAccess\Model;
 use Piwik\Date;
@@ -19,6 +22,7 @@ use Piwik\Db;
 use Piwik\Option;
 use Piwik\Common;
 use Piwik\Piwik;
+use Piwik\Plugin\Manager;
 use Piwik\Plugins\CoreAdminHome\Tasks\ArchivesToPurgeDistributedList;
 use Piwik\Plugins\PrivacyManager\PrivacyManager;
 use Piwik\Period;
@@ -26,6 +30,7 @@ use Piwik\Segment;
 use Piwik\SettingsServer;
 use Piwik\Site;
 use Piwik\Tracker\Cache;
+use Piwik\Tracker\Model as TrackerModel;
 
 /**
  * Service that can be used to invalidate archives or add archive references to a list so they will
@@ -69,10 +74,16 @@ class ArchiveInvalidator
      */
     private $archivingStatus;
 
+    /**
+     * @var SegmentArchiving
+     */
+    private $segmentArchiving;
+
     public function __construct(Model $model, ArchivingStatus $archivingStatus)
     {
         $this->model = $model;
         $this->archivingStatus = $archivingStatus;
+        $this->segmentArchiving = null;
     }
 
     public function getAllRememberToInvalidateArchivedReportsLater()
@@ -228,16 +239,36 @@ class ArchiveInvalidator
 
     /**
      * @param $idSites int[]
-     * @param $dates Date[]
+     * @param $dates Date[]|string[]
      * @param $period string
      * @param $segment Segment
      * @param bool $cascadeDown
+     * @param bool $forceInvalidateNonexistantRanges set true to force inserting rows for ranges in archive_invalidations
+     * @param string $name null to make sure every plugin is archived when this invalidation is processed by core:archive,
+     *                     or a plugin name to only archive the specific plugin.
      * @return InvalidationResult
      * @throws \Exception
      */
     public function markArchivesAsInvalidated(array $idSites, array $dates, $period, Segment $segment = null, $cascadeDown = false,
-                                              $forceInvalidateNonexistantRanges = false)
+                                              $forceInvalidateNonexistantRanges = false, $name = null)
     {
+        $plugin = null;
+        if ($name && strpos($name, '.') !== false) {
+            list($plugin) = explode('.', $name);
+        }
+
+        // remove sites w/ no visits
+        $trackerModel = new TrackerModel();
+        $idSites = array_filter($idSites, function ($idSite) use ($trackerModel) {
+            return !$trackerModel->isSiteEmpty($idSite);
+        });
+
+        if ($plugin
+            && !Manager::getInstance()->isPluginActivated($plugin)
+        ) {
+            throw new \Exception("Plugin is not activated: '$plugin'");
+        }
+
         $invalidationInfo = new InvalidationResult();
 
         // quick fix for #15086, if we're only invalidating today's date for a site, don't add the site to the list of sites
@@ -247,9 +278,9 @@ class ArchiveInvalidator
             $hasMoreThanJustToday[$idSite] = true;
             $tz = Site::getTimezoneFor($idSite);
 
-            if (($period === 'day' || $period === false)
-                && count($dates) === 1
-                && $dates[0]->toString() == Date::factoryInTimezone('today', $tz)
+            if (($period == 'day' || $period === false)
+                && count($dates) == 1
+                && ((string)$dates[0]) == ((string)Date::factoryInTimezone('today', $tz))
             ) {
                 $hasMoreThanJustToday[$idSite] = false;
             }
@@ -282,7 +313,7 @@ class ArchiveInvalidator
 
         $allPeriodsToInvalidate = $this->getAllPeriodsByYearMonth($period, $datesToInvalidate, $cascadeDown);
 
-        $this->markArchivesInvalidated($idSites, $allPeriodsToInvalidate, $segment, $period != 'range', $forceInvalidateNonexistantRanges);
+        $this->markArchivesInvalidated($idSites, $allPeriodsToInvalidate, $segment, $period != 'range', $forceInvalidateNonexistantRanges, $name);
 
         foreach ($idSites as $idSite) {
             Loader::invalidateMinVisitTimeCache($idSite);
@@ -414,11 +445,73 @@ class ArchiveInvalidator
     }
 
     /**
+     * Schedule rearchiving of reports for a single plugin or single report for N months in the past. The next time
+     * core:archive is run, they will be processed.
+     *
+     * @param int[] $idSite
+     * @param Date $date1
+     * @param Date $date2
+     * @param string $plugin
+     * @param string|null $report
+     * @throws \Exception
+     * @api
+     */
+    public function reArchiveReport(array $idSites, string $plugin, string $report = null, int $lastNMonthsToInvalidate = null)
+    {
+        $lastNMonthsToInvalidate = $lastNMonthsToInvalidate ?: Config::getInstance()->General['rearchive_reports_in_past_last_n_months'];
+        if (empty($lastNMonthsToInvalidate)) {
+            return;
+        }
+
+        $lastNMonthsToInvalidate = (int) substr($lastNMonthsToInvalidate, 4);
+        if (empty($lastNMonthsToInvalidate)) {
+            return;
+        }
+
+        $date2 = Date::yesterday();
+        $date1 = $date2->subMonth($lastNMonthsToInvalidate)->setDay(1);
+
+        $dates = [];
+        $date = $date1;
+        while ($date->isEarlier($date2)) {
+            $dates[] = $date;
+            $date = $date->addDay(1);
+        }
+
+        $name = $plugin;
+        if (!empty($report)) {
+            $name .= '.' . $report;
+        }
+
+        $this->markArchivesAsInvalidated($idSites, $dates, 'day', null, $cascadeDown = false, $forceInvalidateRanges = false, $name);
+
+        foreach ($idSites as $idSite) {
+            $segmentDatesToInvalidate = $this->getSegmentArchiving()->getSegmentArchivesToInvalidate($idSite);
+            foreach ($segmentDatesToInvalidate as $info) {
+                $latestDate = Date::factory($info['date']);
+                $latestDate = $latestDate->isEarlier($date1) ? $latestDate : $date1;
+
+                $datesToInvalidateForSegment = [];
+
+                $date = $latestDate;
+                while ($date->isEarlier($date2)) {
+                    $datesToInvalidateForSegment[] = $date;
+                    $date = $date->addDay(1);
+                }
+
+                $this->markArchivesAsInvalidated($idSites, $datesToInvalidateForSegment, 'day', new Segment($info['segment'], [$idSite]),
+                    $cascadeDown = false, $forceInvalidateRanges = false, $name);
+            }
+        }
+    }
+
+    /**
      * @param int[] $idSites
      * @param string[][][] $dates
      * @throws \Exception
      */
-    private function markArchivesInvalidated($idSites, $dates, Segment $segment = null, $removeRanges = false, $forceInvalidateNonexistantRanges = false)
+    private function markArchivesInvalidated($idSites, $dates, Segment $segment = null, $removeRanges = false,
+                                             $forceInvalidateNonexistantRanges = false, $name = null)
     {
         $idSites = array_map('intval', $idSites);
 
@@ -430,7 +523,8 @@ class ArchiveInvalidator
             $table = ArchiveTableCreator::getNumericTable($tableDateObj);
             $yearMonths[] = $tableDateObj->toString('Y_m');
 
-            $this->model->updateArchiveAsInvalidated($table, $idSites, $datesForTable, $segment, $forceInvalidateNonexistantRanges);
+            $this->model->updateArchiveAsInvalidated($table, $idSites, $datesForTable, $segment, $forceInvalidateNonexistantRanges, $name);
+
             if ($removeRanges) {
                 $this->model->updateRangeArchiveAsInvalidated($table, $idSites, $datesForTable, $segment);
             }
@@ -512,4 +606,12 @@ class ArchiveInvalidator
             return Period\Factory::build($period, $date);
         }
     }
+
+    private function getSegmentArchiving()
+    {
+        if (empty($this->segmentArchiving)) {
+            $this->segmentArchiving = new SegmentArchiving(StaticContainer::get('ini.General.process_new_segments_from'));
+        }
+        return $this->segmentArchiving;
+    }
 }
diff --git a/core/ArchiveProcessor.php b/core/ArchiveProcessor.php
index 202eb5f04d..2de3a3bc84 100644
--- a/core/ArchiveProcessor.php
+++ b/core/ArchiveProcessor.php
@@ -247,8 +247,7 @@ class ArchiveProcessor
         $metrics = $this->getAggregatedNumericMetrics($columns, $operationToApply);
 
         foreach ($metrics as $column => $value) {
-            $value = Common::forceDotAsSeparatorForDecimalPoint($value);
-            $this->archiveWriter->insertRecord($column, $value);
+            $this->insertNumericRecord($column, $value);
         }
         // if asked for only one field to sum
         if (count($metrics) === 1) {
diff --git a/core/ArchiveProcessor/Loader.php b/core/ArchiveProcessor/Loader.php
index 89babfb1ad..d7ea423246 100644
--- a/core/ArchiveProcessor/Loader.php
+++ b/core/ArchiveProcessor/Loader.php
@@ -10,6 +10,7 @@ namespace Piwik\ArchiveProcessor;
 
 use Piwik\Archive\ArchiveInvalidator;
 use Piwik\Cache;
+use Piwik\Common;
 use Piwik\Config;
 use Piwik\Container\StaticContainer;
 use Piwik\Context;
@@ -21,6 +22,7 @@ use Piwik\Date;
 use Piwik\Db;
 use Piwik\Period;
 use Piwik\Piwik;
+use Piwik\SettingsServer;
 use Piwik\Site;
 use Psr\Log\LoggerInterface;
 
@@ -99,9 +101,22 @@ class Loader
     {
         $this->params->setRequestedPlugin($pluginName);
 
-        list($idArchive, $visits, $visitsConverted, $isAnyArchiveExists) = $this->loadExistingArchiveIdFromDb();
-        if (!empty($idArchive)) { // we have a usable idarchive (it's not invalidated and it's new enough)
-            return $idArchive;
+        if (SettingsServer::isArchivePhpTriggered()) {
+            $requestedReport = Common::getRequestVar('requestedReport', '', 'string');
+            if (!empty($requestedReport)) {
+                $this->params->setArchiveOnlyReport($requestedReport);
+            }
+        }
+
+        // NOTE: $idArchives will contain the latest DONE_OK/DONE_INVALIDATED archive as well as any partial archives
+        // with a ts_archived >= the DONE_OK/DONE_INVALIDATED date.
+        list($idArchives, $visits, $visitsConverted, $isAnyArchiveExists) = $this->loadExistingArchiveIdFromDb();
+        if (!empty($idArchives)
+            && !$this->params->getArchiveOnlyReport()
+        ) {
+            // we have a usable idarchive (it's not invalidated and it's new enough), and we are not archiving
+            // a single report
+            return [$idArchives, $visits];
         }
 
         // NOTE: this optimization helps when archiving large periods. eg, if archiving a year w/ a segment where
@@ -111,7 +126,7 @@ class Loader
         // we don't create an archive in this case, because the archive may be in progress in some way, so a 0
         // visits archive can be inaccurate in the long run.
         if ($this->canSkipThisArchive()) {
-            return false;
+            return [false, 0];
         }
 
         // if there is an archive, but we can't use it for some reason, invalidate existing archives before
@@ -136,10 +151,10 @@ class Loader
         }
 
         if ($this->isThereSomeVisits($visits) || PluginsArchiver::doesAnyPluginArchiveWithoutVisits()) {
-            return $idArchive;
+            return [[$idArchive], $visits];
         }
 
-        return false;
+        return [false, false];
     }
 
     /**
@@ -155,14 +170,17 @@ class Loader
 
         if ($createSeparateArchiveForCoreMetrics) {
             $requestedPlugin = $this->params->getRequestedPlugin();
+            $requestedReport = $this->params->getArchiveOnlyReport();
 
             $this->params->setRequestedPlugin('VisitsSummary');
+            $this->params->setArchiveOnlyReport(null);
 
             $pluginsArchiver = new PluginsArchiver($this->params);
             $metrics = $pluginsArchiver->callAggregateCoreMetrics();
             $pluginsArchiver->finalizeArchive();
 
             $this->params->setRequestedPlugin($requestedPlugin);
+            $this->params->setArchiveOnlyReport($requestedReport);
 
             $visits = $metrics['nb_visits'];
             $visitsConverted = $metrics['nb_visits_converted'];
@@ -342,10 +360,12 @@ class Loader
         $idSite = $params->getSite()->getId();
 
         $isWebsiteUsingTracker = $this->isWebsiteUsingTheTracker($idSite);
+        $isArchivingForcedWhenNoVisits = $this->shouldArchiveForSiteEvenWhenNoVisits();
         $hasSiteVisitsBetweenTimeframe = $this->hasSiteVisitsBetweenTimeframe($idSite, $params->getPeriod());
         $hasChildArchivesInPeriod = $this->dataAccessModel->hasChildArchivesInPeriod($idSite, $params->getPeriod());
 
         return $isWebsiteUsingTracker
+            && !$isArchivingForcedWhenNoVisits
             && !$hasSiteVisitsBetweenTimeframe
             && !$hasChildArchivesInPeriod;
     }
@@ -396,7 +416,6 @@ class Loader
 
         $timezone = Site::getTimezoneFor($idSite);
         list($date1, $date2) = $period->getBoundsInTimezone($timezone);
-
         if ($date2->isEarlier($minVisitTimesPerSite)) {
             return false;
         }
@@ -412,7 +431,9 @@ class Loader
         $value = $cache->fetch($cacheKey);
         if ($value === false) {
             $value = $this->rawLogDao->getMinimumVisitTimeForSite($idSite);
-            $cache->save($cacheKey, $value, $ttl = self::MIN_VISIT_TIME_TTL);
+            if (!empty($value)) {
+                $cache->save($cacheKey, $value, $ttl = self::MIN_VISIT_TIME_TTL);
+            }
         }
 
         if (!empty($value)) {
diff --git a/core/ArchiveProcessor/Parameters.php b/core/ArchiveProcessor/Parameters.php
index 52ec8bb12b..d3a23b1e8f 100644
--- a/core/ArchiveProcessor/Parameters.php
+++ b/core/ArchiveProcessor/Parameters.php
@@ -54,6 +54,16 @@ class Parameters
     private $isRootArchiveRequest = true;
 
     /**
+     * @var string
+     */
+    private $archiveOnlyReport = null;
+
+    /**
+     * @var bool
+     */
+    private $isArchiveOnlyReportHandled;
+
+    /**
      * Constructor.
      *
      * @ignore
@@ -66,6 +76,29 @@ class Parameters
     }
 
     /**
+     * If we want to archive only a single report, we can request that via this method.
+     * It is up to each plugin's archiver to respect the setting.
+     *
+     * @param string $archiveOnlyReport
+     * @api
+     */
+    public function setArchiveOnlyReport($archiveOnlyReport)
+    {
+        $this->archiveOnlyReport = $archiveOnlyReport;
+    }
+
+    /**
+     * Gets the report we want to archive specifically, or null if none was specified.
+     *
+     * @return string|null
+     * @api
+     */
+    public function getArchiveOnlyReport()
+    {
+        return $this->archiveOnlyReport;
+    }
+
+    /**
      * @ignore
      */
     public function setRequestedPlugin($plugin)
@@ -266,4 +299,30 @@ class Parameters
     {
         return "[idSite = {$this->getSite()->getId()}, period = {$this->getPeriod()->getLabel()} {$this->getPeriod()->getRangeString()}, segment = {$this->getSegment()->getString()}]";
     }
+
+    /**
+     * Returns whether the setArchiveOnlyReport() was handled by an Archiver.
+     *
+     * @return bool
+     */
+    public function isPartialArchive()
+    {
+        if (!$this->getRequestedPlugin()) { // sanity check, partial archives are only for
+            return false;
+        }
+
+        return $this->isArchiveOnlyReportHandled;
+    }
+
+    /**
+     * If a plugin's archiver handles the setArchiveOnlyReport() setting, it should call this method
+     * so it is known that the archive only contains the requested report. This should be called
+     * in an Archiver's __construct method.
+     *
+     * @param bool $isArchiveOnlyReportHandled
+     */
+    public function setIsPartialArchive($isArchiveOnlyReportHandled)
+    {
+        $this->isArchiveOnlyReportHandled = $isArchiveOnlyReportHandled;
+    }
 }
diff --git a/core/ArchiveProcessor/PluginsArchiver.php b/core/ArchiveProcessor/PluginsArchiver.php
index 60d95d5a2d..4fd517a1df 100644
--- a/core/ArchiveProcessor/PluginsArchiver.php
+++ b/core/ArchiveProcessor/PluginsArchiver.php
@@ -72,6 +72,7 @@ class PluginsArchiver
 
         $this->archiveProcessor = new ArchiveProcessor($this->params, $this->archiveWriter, $this->logAggregator);
 
+
         $shouldAggregateFromRawData = $this->params->isSingleSiteDayArchive();
 
         /**
@@ -149,7 +150,6 @@ class PluginsArchiver
             }
 
             if ($this->shouldProcessReportsForPlugin($pluginName)) {
-
                 $this->logAggregator->setQueryOriginHint($pluginName);
 
                 try {
@@ -266,13 +266,17 @@ class PluginsArchiver
         if (Rules::shouldProcessReportsAllPlugins(
             array($this->params->getSite()->getId()),
             $this->params->getSegment(),
-            $this->params->getPeriod()->getLabel())) {
+            $this->params->getPeriod()->getLabel())
+        ) {
             return true;
         }
 
-        if (!\Piwik\Plugin\Manager::getInstance()->isPluginLoaded($this->params->getRequestedPlugin())) {
-            return true;
+        if ($this->params->getRequestedPlugin() &&
+            !\Piwik\Plugin\Manager::getInstance()->isPluginLoaded($this->params->getRequestedPlugin())
+        ) {
+            return false;
         }
+
         return false;
     }
 
diff --git a/core/ArchiveProcessor/Rules.php b/core/ArchiveProcessor/Rules.php
index ae2a560cd6..24acf8a434 100644
--- a/core/ArchiveProcessor/Rules.php
+++ b/core/ArchiveProcessor/Rules.php
@@ -9,12 +9,14 @@
 namespace Piwik\ArchiveProcessor;
 
 use Exception;
+use Piwik\Common;
 use Piwik\Config;
 use Piwik\DataAccess\ArchiveWriter;
 use Piwik\Date;
 use Piwik\Log;
 use Piwik\Option;
 use Piwik\Piwik;
+use Piwik\Plugin\Manager;
 use Piwik\Plugins\CoreAdminHome\Controller;
 use Piwik\Segment;
 use Piwik\SettingsPiwik;
@@ -57,6 +59,10 @@ class Rules
 
     public static function shouldProcessReportsAllPlugins(array $idSites, Segment $segment, $periodLabel)
     {
+        if (self::isForceArchivingSinglePlugin()) {
+            return false;
+        }
+
         if ($segment->isEmpty() && ($periodLabel != 'range' || SettingsServer::isArchivePhpTriggered())) {
             return true;
         }
@@ -305,17 +311,27 @@ class Rules
      *
      * @return string[]
      */
-    public static function getSelectableDoneFlagValues($includeInvalidated = true, Parameters $params = null)
+    public static function getSelectableDoneFlagValues($includeInvalidated = true, Parameters $params = null, $checkAuthorizedToArchive = true)
     {
         $possibleValues = array(ArchiveWriter::DONE_OK, ArchiveWriter::DONE_OK_TEMPORARY);
 
-        if (!Rules::isRequestAuthorizedToArchive($params)
-            && $includeInvalidated
-        ) {
-            //If request is not authorized to archive then fetch also invalidated archives
-            $possibleValues[] = ArchiveWriter::DONE_INVALIDATED;
+        if ($includeInvalidated) {
+            if (!$checkAuthorizedToArchive || !Rules::isRequestAuthorizedToArchive($params)) {
+                //If request is not authorized to archive then fetch also invalidated archives
+                $possibleValues[] = ArchiveWriter::DONE_INVALIDATED;
+                $possibleValues[] = ArchiveWriter::DONE_PARTIAL;
+            }
         }
 
         return $possibleValues;
     }
+
+    public static function isForceArchivingSinglePlugin()
+    {
+        if (!SettingsServer::isArchivePhpTriggered()) {
+            return false;
+        }
+
+        return !empty($_GET['pluginOnly']) || !empty($_POST['pluginOnly']);
+    }
 }
diff --git a/core/CliMulti.php b/core/CliMulti.php
index c604ab6702..3b0909d347 100644
--- a/core/CliMulti.php
+++ b/core/CliMulti.php
@@ -12,6 +12,8 @@ use Piwik\CliMulti\CliPhp;
 use Piwik\CliMulti\Output;
 use Piwik\CliMulti\Process;
 use Piwik\Container\StaticContainer;
+use Psr\Log\LoggerInterface;
+use Psr\Log\NullLogger;
 
 /**
  * Class CliMulti.
@@ -71,9 +73,15 @@ class CliMulti
 
     protected $isTimingRequests = false;
 
-    public function __construct()
+    /**
+     * @var LoggerInterface
+     */
+    private $logger;
+
+    public function __construct(LoggerInterface $logger = null)
     {
         $this->supportsAsync = $this->supportsAsync();
+        $this->logger = $logger ?: new NullLogger();
     }
 
     /**
@@ -333,7 +341,7 @@ class CliMulti
         $hostname = Url::getHost($checkIfTrusted = false);
         $command = $this->buildCommand($hostname, $query, $output->getPathToFile());
 
-        Log::debug($command);
+        $this->logger->debug("Running command: {command}", ['command' => $command]);
         shell_exec($command);
     }
 
@@ -349,6 +357,7 @@ class CliMulti
             $url = str_replace("http://", "https://", $url);
         }
 
+        $requestBody = null;
         if ($this->runAsSuperUser) {
             $tokenAuth = self::getSuperUserTokenAuth();
 
@@ -358,12 +367,12 @@ class CliMulti
                 $url .= '&';
             }
 
-            $url .= 'token_auth=' . $tokenAuth;
+            $requestBody = 'token_auth=' . $tokenAuth;
         }
 
         try {
-            Log::debug("Execute HTTP API request: "  . $url);
-            $response = Http::sendHttpRequestBy('curl', $url, $timeout = 0, $userAgent = null, $destinationPath = null, $file = null, $followDepth = 0, $acceptLanguage = false, $this->acceptInvalidSSLCertificate);
+            $this->logger->debug("Execute HTTP API request: "  . $url);
+            $response = Http::sendHttpRequestBy('curl', $url, $timeout = 0, $userAgent = null, $destinationPath = null, $file = null, $followDepth = 0, $acceptLanguage = false, $this->acceptInvalidSSLCertificate, false, false, 'POST', null, null, $requestBody, [], $forcePost = true);
             $output->write($response);
         } catch (\Exception $e) {
             $message = "Got invalid response from API request: $url. ";
@@ -376,7 +385,7 @@ class CliMulti
 
             $output->write($message);
 
-            Log::debug($e);
+            $this->logger->debug($message, ['exception' => $e]);
         }
     }
 
diff --git a/core/CronArchive.php b/core/CronArchive.php
index b52fa24ab8..1dd1265070 100644
--- a/core/CronArchive.php
+++ b/core/CronArchive.php
@@ -19,12 +19,14 @@ use Piwik\CronArchive\FixedSiteIds;
 use Piwik\CronArchive\Performance\Logger;
 use Piwik\Archive\ArchiveInvalidator;
 use Piwik\CliMulti\RequestParser;
+use Piwik\CronArchive\QueueConsumer;
 use Piwik\CronArchive\SharedSiteIds;
 use Piwik\DataAccess\ArchiveSelector;
 use Piwik\DataAccess\ArchiveTableCreator;
 use Piwik\DataAccess\Model;
 use Piwik\DataAccess\RawLogDao;
 use Piwik\Metrics\Formatter;
+use Piwik\Period\Factory;
 use Piwik\Period\Factory as PeriodFactory;
 use Piwik\CronArchive\SegmentArchiving;
 use Piwik\Period\Range;
@@ -139,7 +141,7 @@ class CronArchive
      *
      * @var int|false
      */
-    public $dateLastForced = SegmentArchiving::DEFAULT_BEGINNIN_OF_TIME_LAST_N_YEARS;
+    public $dateLastForced = SegmentArchiving::DEFAULT_BEGINNING_OF_TIME_LAST_N_YEARS;
 
     /**
      * The number of concurrent requests to issue per website. Defaults to {@link MAX_CONCURRENT_API_REQUESTS}.
@@ -208,11 +210,6 @@ class CronArchive
     private $archiveFilter;
 
     /**
-     * @var array
-     */
-    private $invalidationsToExclude = [];
-
-    /**
      * @var RequestParser
      */
     private $cliMultiRequestParser;
@@ -329,8 +326,6 @@ class CronArchive
         $pid = Common::getProcessId();
 
         $timer = new Timer;
-        $siteTimer = null;
-        $siteRequests = 0;
 
         $this->logSection("START");
         $this->logger->info("Starting Matomo reports archiving...");
@@ -344,14 +339,12 @@ class CronArchive
 
         $countOfProcesses = $this->getMaxConcurrentApiRequests();
 
+        $queueConsumer = new QueueConsumer($this->logger, $this->websiteIdArchiveList, $countOfProcesses, $pid,
+            $this->model, $this->segmentArchiving, $this, $this->cliMultiRequestParser, $this->archiveFilter);
+
         // invalidate once at the start no matter when the last invalidation occurred
         $this->invalidateArchivedReportsForSitesThatNeedToBeArchivedAgain();
 
-        // if we skip or can't process an idarchive, we want to ignore it the next time we look for an invalidated
-        // archive. these IDs are stored here (using a list like this serves to keep our SQL simple).
-        $this->invalidationsToExclude = [];
-
-        $idSite = null;
         while (true) {
             if ($this->isMaintenanceModeEnabled()) {
                 $this->logger->info("Archiving will stop now because maintenance mode is enabled");
@@ -363,144 +356,15 @@ class CronArchive
                 flush();
             }
 
-            if (empty($idSite)) {
-                $idSite = $this->getNextIdSiteToArchive();
-                if (empty($idSite)) { // no sites left to archive, stop
-                    $this->logger->debug("No more sites left to archive, stopping.");
-                    return;
-                }
-
-                /**
-                 * This event is triggered before the cron archiving process starts archiving data for a single
-                 * site.
-                 *
-                 * Note: multiple archiving processes can post this event.
-                 *
-                 * @param int $idSite The ID of the site we're archiving data for.
-                 * @param string $pid The PID of the process processing archives for this site.
-                 */
-                Piwik::postEvent('CronArchive.archiveSingleSite.start', array($idSite, $pid));
-
-                $this->logger->info("Start processing archives for site {idSite}.", ['idSite' => $idSite]);
-
-                $siteTimer = new Timer();
-                $siteRequests = 0;
-            }
-
-            // we don't want to invalidate different periods together or segment archives w/ no-segment archives
-            // together, but it's possible to end up querying these archives. if we find one, we keep track of it
-            // in this array to exclude, but after we run the current batch, we reset the array so we'll still
-            // process them eventually.
-            $invalidationsToExcludeInBatch = [];
-
-            // get archives to process simultaneously
-            $archivesToProcess = [];
-            while (count($archivesToProcess) < $countOfProcesses) {
-                $invalidatedArchive = $this->getNextInvalidatedArchive($idSite, array_keys($invalidationsToExcludeInBatch));
-                if (empty($invalidatedArchive)) {
-                    $this->logger->debug("No next invalidated archive.");
-                    break;
-                }
-
-                if ($this->hasDifferentPeriod($archivesToProcess, $invalidatedArchive['period'])) {
-                    $this->logger->debug("Found archive with different period than others in concurrent batch, skipping until next batch: {$invalidatedArchive['period']}");
-
-                    $idinvalidation = $invalidatedArchive['idinvalidation'];
-                    $invalidationsToExcludeInBatch[$idinvalidation] = true;
-                    continue;
-                }
-
-                if ($this->hasDifferentDoneFlagType($archivesToProcess, $invalidatedArchive['name'])) {
-                    $this->logger->debug("Found archive with different done flag type (segment vs. no segment) in concurrent batch, skipping until next batch: {$invalidatedArchive['name']}");
-
-                    $idinvalidation = $invalidatedArchive['idinvalidation'];
-                    $invalidationsToExcludeInBatch[$idinvalidation] = true;
-
-                    continue;
-                }
-
-                if ($invalidatedArchive['segment'] === null) {
-                    $this->logger->debug("Found archive for segment that is not auto archived, ignoring.");
-                    $this->addInvalidationToExclude($invalidatedArchive);
-                    continue;
-                }
-
-                if ($this->isDoneFlagForPlugin($invalidatedArchive['name'])) {
-                    $this->logger->debug("Found plugin specific invalidated archive, ignoring.");
-                    $this->addInvalidationToExclude($invalidatedArchive);
-                    continue;
-                }
-
-                if ($this->archiveArrayContainsArchive($archivesToProcess, $invalidatedArchive)) {
-                    $this->logger->debug("Found duplicate invalidated archive {$invalidatedArchive['idarchive']}, ignoring.");
-                    $this->addInvalidationToExclude($invalidatedArchive);
-                    $this->model->deleteInvalidations([$invalidatedArchive]);
-                    continue;
-                }
-
-                $reason = $this->shouldSkipArchive($invalidatedArchive);
-                if ($reason) {
-                    $this->logger->debug("Skipping invalidated archive {$invalidatedArchive['idarchive']}: $reason");
-                    $this->addInvalidationToExclude($invalidatedArchive);
-                    continue;
-                }
-
-                if ($this->canSkipArchiveBecauseNoPoint($invalidatedArchive)) {
-                    $this->logger->debug("Found invalidated archive we can skip (no visits or latest archive is not invalidated). "
-                        . "[idSite = {$invalidatedArchive['idsite']}, dates = {$invalidatedArchive['date1']} - {$invalidatedArchive['date2']}, segment = {$invalidatedArchive['segment']}]");
-                    $this->addInvalidationToExclude($invalidatedArchive);
-                    $this->model->deleteInvalidations([$invalidatedArchive]);
-                    continue;
-                }
-
-                // TODO: should use descriptive string instead of just invalidation ID
-                $reason = $this->shouldSkipArchiveBecauseLowerPeriodOrSegmentIsInProgress($invalidatedArchive);
-                if ($reason) {
-                    $this->logger->debug("Skipping invalidated archive {$invalidatedArchive['idarchive']}: $reason");
-                    $invalidationsToExcludeInBatch[$invalidatedArchive['idinvalidation']] = true;
-                    $this->addInvalidationToExclude($invalidatedArchive);
-                    continue;
-                }
-
-                $started = $this->model->startArchive($invalidatedArchive);
-                if (!$started) { // another process started on this archive, pull another one
-                    $this->logger->debug("Archive invalidation {$invalidatedArchive['idinvalidation']} is being handled by another process.");
-                    $this->addInvalidationToExclude($invalidatedArchive);
-                    continue;
-                }
-
-                $this->addInvalidationToExclude($invalidatedArchive);
-
-                $archivesToProcess[] = $invalidatedArchive;
+            $archivesToProcess = $queueConsumer->getNextArchivesToProcess();
+            if ($archivesToProcess === null) {
+                break;
             }
 
-            if (empty($archivesToProcess)) { // no invalidated archive left
-                /**
-                 * This event is triggered immediately after the cron archiving process starts archiving data for a single
-                 * site.
-                 *
-                 * Note: multiple archiving processes can post this event.
-                 *
-                 * @param int $idSite The ID of the site we're archiving data for.
-                 * @param string $pid The PID of the process processing archives for this site.
-                 */
-                Piwik::postEvent('CronArchive.archiveSingleSite.finish', array($idSite, $pid));
-
-                $this->logger->info("Finished archiving for site {idSite}, {requests} API requests, {timer} [{processed} / {totalNum} done]", [
-                    'idSite' => $idSite,
-                    'processed' => $this->websiteIdArchiveList->getNumProcessedWebsites(),
-                    'totalNum' => $this->websiteIdArchiveList->getNumSites(),
-                    'timer' => $siteTimer,
-                    'requests' => $siteRequests,
-                ]);
-
-                $idSite = null;
-
+            if (empty($archivesToProcess)) {
                 continue;
             }
 
-            $siteRequests += count($archivesToProcess);
-
             $successCount = $this->launchArchivingFor($archivesToProcess);
             $numArchivesFinished += $successCount;
         }
@@ -521,64 +385,12 @@ class CronArchive
         $this->logger->info($timer->__toString());
     }
 
-    private function isDoneFlagForPlugin($doneFlag)
-    {
-        return strpos($doneFlag, '.') !== false;
-    }
-
-    private function archiveArrayContainsArchive($archiveArray, $archive)
-    {
-        foreach ($archiveArray as $entry) {
-            if ($entry['idsite'] == $archive['idsite']
-                && $entry['period'] == $archive['period']
-                && $entry['date1'] == $archive['date1']
-                && $entry['date2'] == $archive['date2']
-                && $entry['name'] == $archive['name']
-            ) {
-                return true;
-            }
-        }
-        return false;
-    }
-
-    // TODO: need to also delete rows from archive_invalidations via scheduled task, eg, if ts_invalidated is older than 3 days or something.
-    private function getNextInvalidatedArchive($idSite, $extraInvalidationsToIgnore)
-    {
-        $lastInvalidationTime = self::getLastInvalidationTime();
-        if (empty($lastInvalidationTime)
-            || (time() - $lastInvalidationTime) >= 3600
-        ) {
-            $this->invalidateArchivedReportsForSitesThatNeedToBeArchivedAgain();
-        }
-
-        $iterations = 0;
-        while ($iterations < 100) {
-            $invalidationsToExclude = array_merge($this->invalidationsToExclude, $extraInvalidationsToIgnore);
-
-            $nextArchive = $this->model->getNextInvalidatedArchive($idSite, $invalidationsToExclude);
-            if (empty($nextArchive)) {
-                break;
-            }
-
-            $isCronArchivingEnabled = $this->findSegmentForArchive($nextArchive);
-            if ($isCronArchivingEnabled) {
-                return $nextArchive;
-            }
-
-            $this->invalidationsToExclude[] = $nextArchive['idinvalidation'];
-
-            ++$iterations;
-        }
-
-        return null;
-    }
-
     private function launchArchivingFor($archives)
     {
         $urls = [];
         $archivesBeingQueried = [];
         foreach ($archives as $index => $archive) {
-            list($url, $segment) = $this->generateUrlToArchiveFromArchiveInfo($archive);
+            list($url, $segment, $plugin) = $this->generateUrlToArchiveFromArchiveInfo($archive);
             if (empty($url)) {
                 // can happen if, for example, a segment was deleted after an archive was invalidated
                 // in this case, we can just delete the archive entirely.
@@ -591,6 +403,11 @@ class CronArchive
             $period = PeriodFactory::build($this->periodIdsToLabels[$archive['period']], $dateStr);
             $params = new Parameters(new Site($idSite), $period, new Segment($segment, [$idSite], $period->getDateStart(), $period->getDateEnd()));
 
+            if (!empty($plugin)) {
+                $params->setRequestedPlugin($plugin);
+                $params->onlyArchiveRequestedPlugin();
+            }
+
             $loader = new Loader($params);
             if ($loader->canSkipThisArchive()) {
                 $this->logger->info("Found no visits for site ID = {idSite}, {period} ({date1},{date2}), site is using the tracker so skipping archiving...", [
@@ -620,7 +437,6 @@ class CronArchive
 
         $responses = $cliMulti->request($urls);
         $timers = $cliMulti->getTimers();
-
         $successCount = 0;
 
         foreach ($urls as $index => $url) {
@@ -629,13 +445,15 @@ class CronArchive
 
             $stats = json_decode($content, $assoc = true);
             if (!is_array($stats)) {
-                $this->logError("Error unserializing the following response from $url: " . $content);
+                $this->logger->info(var_export($content, true));
+
+                $this->logError("Error unserializing the following response from $url: '" . $content . "'");
                 continue;
             }
 
             $visitsForPeriod = $this->getVisitsFromApiResponse($stats);
 
-            $this->logArchiveJobFinished($url, $timers[$index], $visitsForPeriod);
+            $this->logArchiveJobFinished($url, $timers[$index], $visitsForPeriod, $archivesBeingQueried[$index]['plugin'], $archivesBeingQueried[$index]['report']);
 
             // TODO: do in ArchiveWriter
             $this->deleteInvalidatedArchives($archivesBeingQueried[$index]);
@@ -661,6 +479,8 @@ class CronArchive
 
     private function generateUrlToArchiveFromArchiveInfo($archive)
     {
+        $plugin = $archive['plugin'];
+        $report = $archive['report'];
         $period = $this->periodIdsToLabels[$archive['period']];
 
         if ($period == 'range') {
@@ -673,7 +493,7 @@ class CronArchive
 
         $segment = isset($archive['segment']) ? $archive['segment'] : '';
 
-        $url = $this->getVisitsRequestUrl($idSite, $period, $date, $segment);
+        $url = $this->getVisitsRequestUrl($idSite, $period, $date, $segment, $plugin);
         $url = $this->makeRequestUrl($url);
 
         if (!empty($segment)) {
@@ -685,35 +505,25 @@ class CronArchive
             }
         }
 
-        return [$url, $segment];
-    }
-
-    private function findSegmentForArchive(&$archive)
-    {
-        $flag = explode('.', $archive['name'])[0];
-        if ($flag == 'done') {
-            $archive['segment'] = '';
-            return true;
+        if (!empty($plugin)) {
+            $url .= "&pluginOnly=1";
         }
 
-        $hash = substr($flag, 4);
-        $storedSegment = $this->segmentArchiving->findSegmentForHash($hash, $archive['idsite']);
-        if (!isset($storedSegment['definition'])) {
-            $archive['segment'] = null;
-            return false;
+        if (!empty($report)) {
+            $url .= "&requestedReport=" . urlencode($report);
         }
 
-        $archive['segment'] = $storedSegment['definition'];
-        return $this->segmentArchiving->isAutoArchivingEnabledFor($storedSegment);
+        return [$url, $segment, $plugin];
     }
 
-    private function logArchiveJobFinished($url, $timer, $visits)
+    private function logArchiveJobFinished($url, $timer, $visits, $plugin = null, $report = null)
     {
         $params = UrlHelper::getArrayFromQueryString($url);
         $visits = (int) $visits;
 
         $this->logger->info("Archived website id {$params['idSite']}, period = {$params['period']}, date = "
-            . "{$params['date']}, segment = '" . (isset($params['segment']) ? $params['segment'] : '') . "', $visits visits found. $timer");
+            . "{$params['date']}, segment = '" . (isset($params['segment']) ? $params['segment'] : '') . "', "
+            . ($plugin ? "plugin = $plugin, " : "") . ($report ? "report = $report, " : "") . "$visits visits found. $timer");
     }
 
     public function getErrors()
@@ -787,12 +597,15 @@ class CronArchive
      * @param bool|false $segment
      * @return string
      */
-    private function getVisitsRequestUrl($idSite, $period, $date, $segment = false)
+    private function getVisitsRequestUrl($idSite, $period, $date, $segment = false, $plugin = null)
     {
-        $request = "?module=API&method=API.get&idSite=$idSite&period=$period&date=" . $date . "&format=json";
+        $request = "?module=API&method=CoreAdminHome.archiveReports&idSite=$idSite&period=$period&date=" . $date . "&format=json";
         if ($segment) {
             $request .= '&segment=' . urlencode($segment);
         }
+        if (!empty($plugin)) {
+            $request .= "&plugin=" . $plugin;
+        }
         return $request;
     }
 
@@ -912,11 +725,26 @@ class CronArchive
 
         foreach ($sitesPerDays as $date => $siteIds) {
             //Concurrent transaction logic will end up with duplicates set.  Adding array_unique to the siteIds.
-            $listSiteIds = implode(',', array_unique($siteIds));
+            $siteIds = array_unique($siteIds);
+
+            $period = Factory::build('day', $date);
+
+            $siteIdsToInvalidate = [];
+            foreach ($siteIds as $idSite) {
+                $params = new Parameters(new Site($idSite), $period, new Segment('', [$idSite], $period->getDateStart(), $period->getDateEnd()));
+                if ($this->isThereExistingValidPeriod($params)) {
+                    $this->logger->info('  Found usable archive for date range {date} for site {idSite}, skipping invalidation for now.', ['date' => $date, 'idSite' => $idSite]);
+                    continue;
+                }
+
+                $siteIdsToInvalidate[] = $idSite;
+            }
+
+            $listSiteIds = implode(',', $siteIdsToInvalidate);
 
             try {
                 $this->logger->info('  Will invalidate archived reports for ' . $date . ' for following websites ids: ' . $listSiteIds);
-                $this->getApiToInvalidateArchivedReport()->invalidateArchivedReports($siteIds, $date);
+                $this->getApiToInvalidateArchivedReport()->invalidateArchivedReports($siteIdsToInvalidate, $date);
             } catch (Exception $e) {
                 $this->logger->info('  Failed to invalidate archived reports: ' . $e->getMessage());
             }
@@ -975,8 +803,6 @@ class CronArchive
             }
         }
 
-        Db::fetchAll("SELECT idinvalidation, idarchive, idsite, date1, date2, period, name, status FROM " . Common::prefixTable('archive_invalidations'));
-
         $this->setInvalidationTime();
 
         $this->logger->info("Done invalidating");
@@ -997,7 +823,7 @@ class CronArchive
 
             $loader = new Loader($params);
             if ($loader->canSkipThisArchive()) {
-                $this->logger->debug("  " . ucfirst($dateStr) . " archive can be skipped due to no visits, skipping invalidation...");
+                $this->logger->debug("  " . ucfirst($dateStr) . " archive can be skipped due to no visits for idSite = $idSite, skipping invalidation...");
                 continue;
             }
 
@@ -1010,15 +836,15 @@ class CronArchive
         }
     }
 
-    private function isThereExistingValidPeriod(Parameters $params, $isYesterday = false)
+    public function isThereExistingValidPeriod(Parameters $params, $isYesterday = false)
     {
-        $today = Date::factory('today');
+        $today = Date::factoryInTimezone('today', Site::getTimezoneFor($params->getSite()->getId()));
 
         $isPeriodIncludesToday = $params->getPeriod()->isDateInPeriod($today);
         $minArchiveProcessedTime = $isPeriodIncludesToday ? Date::now()->subSeconds(Rules::getPeriodArchiveTimeToLiveDefault($params->getPeriod()->getLabel())) : null;
 
         // empty plugins param since we only check for an 'all' archive
-        list($idArchive, $visits, $visitsConverted, $ignore, $tsArchived) = ArchiveSelector::getArchiveIdAndVisits($params, $minArchiveProcessedTime, $includeInvalidated = false);
+        list($idArchive, $visits, $visitsConverted, $ignore, $tsArchived) = ArchiveSelector::getArchiveIdAndVisits($params, $minArchiveProcessedTime, $includeInvalidated = $isPeriodIncludesToday);
 
         // day has changed since the archive was created, we need to reprocess it
         if ($isYesterday
@@ -1096,7 +922,7 @@ class CronArchive
             $this->logger->info("  See the doc at: https://matomo.org/docs/setup-auto-archiving/");
         }
 
-        $cliMulti = new CliMulti();
+        $cliMulti = new CliMulti($this->logger);
         $supportsAsync = $cliMulti->supportsAsync();
         $this->logger->info("- " . ($supportsAsync ? 'Async process archiving supported, using CliMulti.' : 'Async process archiving not supported, using curl requests.'));
 
@@ -1255,7 +1081,7 @@ class CronArchive
     private function makeCliMulti()
     {
         /** @var CliMulti $cliMulti */
-        $cliMulti = StaticContainer::getContainer()->make('Piwik\CliMulti');
+        $cliMulti = new CliMulti($this->logger);
         $cliMulti->setUrlToPiwik($this->urlToPiwik);
         $cliMulti->setPhpCliConfigurationOptions($this->phpCliConfigurationOptions);
         $cliMulti->setAcceptInvalidSSLCertificate($this->acceptInvalidSSLCertificate);
@@ -1321,15 +1147,6 @@ class CronArchive
         return false;
     }
 
-    private function shouldSkipArchive($archive)
-    {
-        if ($this->archiveFilter) {
-            return $this->archiveFilter->filterArchive($archive);
-        }
-
-        return false;
-    }
-
     protected function wasSegmentChangedRecently($definition, $allSegments)
     {
         foreach ($allSegments as $segment) {
@@ -1354,19 +1171,6 @@ class CronArchive
         $this->archiveFilter = $archiveFilter;
     }
 
-    private function addInvalidationToExclude(array $invalidatedArchive)
-    {
-        $id = $invalidatedArchive['idinvalidation'];
-        if (empty($this->invalidationsToExclude[$id])) {
-            $this->invalidationsToExclude[$id] = $id;
-        }
-    }
-
-    private function getNextIdSiteToArchive()
-    {
-        return $this->websiteIdArchiveList->getNextSiteId();
-    }
-
     private function makeWebsiteIdArchiveList(array $websitesIds)
     {
         if ($this->shouldArchiveAllSites) {
@@ -1381,117 +1185,4 @@ class CronArchive
 
         return new SharedSiteIds($websitesIds, SharedSiteIds::OPTION_ALL_WEBSITES);
     }
-
-    private function hasDifferentPeriod(array $archivesToProcess, $period)
-    {
-        if (empty($archivesToProcess)) {
-            return false;
-        }
-
-        return $archivesToProcess[0]['period'] != $period;
-    }
-
-    private function hasDifferentDoneFlagType(array $archivesToProcess, $name)
-    {
-        if (empty($archivesToProcess)) {
-            return false;
-        }
-
-        $existingDoneFlagType = $this->getDoneFlagType($archivesToProcess[0]['name']);
-        $newArchiveDoneFlagType = $this->getDoneFlagType($name);
-
-        return $existingDoneFlagType != $newArchiveDoneFlagType;
-    }
-
-    private function getDoneFlagType($name)
-    {
-        if ($name == 'done') {
-            return 'all';
-        } else {
-            return 'segment';
-        }
-    }
-
-    private function canSkipArchiveBecauseNoPoint(array $invalidatedArchive)
-    {
-        $site = new Site($invalidatedArchive['idsite']);
-
-        $periodLabel = $this->periodIdsToLabels[$invalidatedArchive['period']];
-        $dateStr = $periodLabel == 'range' ? ($invalidatedArchive['date1'] . ',' . $invalidatedArchive['date2']) : $invalidatedArchive['date1'];
-        $period = PeriodFactory::build($periodLabel, $dateStr);
-
-        $segment = new Segment($invalidatedArchive['segment'], [$invalidatedArchive['idsite']], $period->getDateStart(), $period->getDateEnd());
-
-        $params = new Parameters($site, $period, $segment);
-
-        $loader = new Loader($params);
-        if ($loader->canSkipThisArchive()) { // if no point in archiving, skip
-            return true;
-        }
-
-        // if valid archive already exists, do not re-archive
-        $minDateTimeProcessedUTC = Date::now()->subSeconds(Rules::getPeriodArchiveTimeToLiveDefault($periodLabel));
-        $archiveIdAndVisits = ArchiveSelector::getArchiveIdAndVisits($params, $minDateTimeProcessedUTC, $includeInvalidated = false);
-
-        $idArchive = $archiveIdAndVisits[0];
-        return !empty($idArchive);
-    }
-
-    private function shouldSkipArchiveBecauseLowerPeriodOrSegmentIsInProgress(array $archiveToProcess)
-    {
-        $inProgressArchives = $this->cliMultiRequestParser->getInProgressArchivingCommands();
-
-        $periodLabel = $this->periodIdsToLabels[$archiveToProcess['period']];
-        $archiveToProcess['periodObj'] = PeriodFactory::build($periodLabel, $archiveToProcess['date1']);
-
-        foreach ($inProgressArchives as $archiveBeingProcessed) {
-            if (empty($archiveBeingProcessed['period'])
-                || empty($archiveBeingProcessed['date'])
-            ) {
-                continue;
-            }
-
-            $archiveBeingProcessed['periodObj'] = PeriodFactory::build($archiveBeingProcessed['period'], $archiveBeingProcessed['date']);
-
-            if ($this->isArchiveOfLowerPeriod($archiveToProcess, $archiveBeingProcessed)) {
-                return "lower period in progress (period = {$archiveBeingProcessed['period']}, date = {$archiveBeingProcessed['date']})";
-            }
-
-            if ($this->isArchiveNonSegmentAndInProgressArchiveSegment($archiveToProcess, $archiveBeingProcessed)) {
-                return "segment archive in progress for same site/period ({$archiveBeingProcessed['segment']})";
-            }
-        }
-
-        return false;
-    }
-
-    private function isArchiveOfLowerPeriod(array $archiveToProcess, $archiveBeingProcessed)
-    {
-        /** @var Period $archiveToProcessPeriodObj */
-        $archiveToProcessPeriodObj = $archiveToProcess['periodObj'];
-        /** @var Period $archivePeriodObj */
-        $archivePeriodObj = $archiveBeingProcessed['periodObj'];
-
-        if ($archiveToProcessPeriodObj->getId() >= $archivePeriodObj->getId()
-            && $archiveToProcessPeriodObj->isPeriodIntersectingWith($archivePeriodObj)
-        ) {
-            return true;
-        }
-
-        return false;
-    }
-
-    private function isArchiveNonSegmentAndInProgressArchiveSegment(array $archiveToProcess, array $archiveBeingProcessed)
-    {
-        // archive is for different site/period
-        if (empty($archiveBeingProcessed['idSite'])
-            || $archiveToProcess['idsite'] != $archiveBeingProcessed['idSite']
-            || $archiveToProcess['periodObj']->getId() != $archiveBeingProcessed['periodObj']->getId()
-            || $archiveToProcess['periodObj']->getDateStart()->toString() != $archiveBeingProcessed['periodObj']->getDateStart()->toString()
-        ) {
-            return false;
-        }
-
-        return empty($archiveToProcess['segment']) && !empty($archiveBeingProcessed['segment']);
-    }
-}
\ No newline at end of file
+}
diff --git a/core/CronArchive/QueueConsumer.php b/core/CronArchive/QueueConsumer.php
new file mode 100644
index 0000000000..2d64701ef1
--- /dev/null
+++ b/core/CronArchive/QueueConsumer.php
@@ -0,0 +1,544 @@
+<?php
+/**
+ * Matomo - free/libre analytics platform
+ *
+ * @link https://matomo.org
+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
+ *
+ */
+
+namespace Piwik\CronArchive;
+
+
+use Piwik\ArchiveProcessor\Loader;
+use Piwik\ArchiveProcessor\Parameters;
+use Piwik\ArchiveProcessor\Rules;
+use Piwik\CliMulti\RequestParser;
+use Piwik\CronArchive;
+use Piwik\DataAccess\ArchiveSelector;
+use Piwik\DataAccess\Model;
+use Piwik\Date;
+use Piwik\Period;
+use Piwik\Period\Factory as PeriodFactory;
+use Piwik\Piwik;
+use Piwik\Plugin\Manager;
+use Piwik\Segment;
+use Piwik\Site;
+use Piwik\Timer;
+use Psr\Log\LoggerInterface;
+
+class QueueConsumer
+{
+    /**
+     * @var LoggerInterface
+     */
+    private $logger;
+
+    /**
+     * @var FixedSiteIds|SharedSiteIds
+     */
+    private $websiteIdArchiveList;
+
+    /**
+     * @var int
+     */
+    private $countOfProcesses;
+
+    /**
+     * @var int
+     */
+    private $pid;
+
+    /**
+     * @var Model
+     */
+    private $model;
+
+    /**
+     * @var ArchiveFilter
+     */
+    private $archiveFilter;
+
+    /**
+     * @var SegmentArchiving
+     */
+    private $segmentArchiving;
+
+    /**
+     * @var CronArchive
+     */
+    private $cronArchive;
+
+    /**
+     * @var array
+     */
+    private $invalidationsToExclude;
+
+    /**
+     * @var string[]
+     */
+    private $periodIdsToLabels;
+
+    /**
+     * @var RequestParser
+     */
+    private $cliMultiRequestParser;
+
+    /**
+     * @var int
+     */
+    private $idSite;
+
+    /**
+     * @var int
+     */
+    private $siteRequests;
+
+    /**
+     * @var Timer
+     */
+    private $siteTimer;
+
+    public function __construct(LoggerInterface $logger, $websiteIdArchiveList, $countOfProcesses, $pid, Model $model,
+                                SegmentArchiving $segmentArchiving, CronArchive $cronArchive, RequestParser $cliMultiRequestParser,
+                                ArchiveFilter $archiveFilter = null)
+    {
+        $this->logger = $logger;
+        $this->websiteIdArchiveList = $websiteIdArchiveList;
+        $this->countOfProcesses = $countOfProcesses;
+        $this->pid = $pid;
+        $this->model = $model;
+        $this->segmentArchiving = $segmentArchiving;
+        $this->cronArchive = $cronArchive;
+        $this->cliMultiRequestParser = $cliMultiRequestParser;
+        $this->archiveFilter = $archiveFilter;
+
+        // if we skip or can't process an idarchive, we want to ignore it the next time we look for an invalidated
+        // archive. these IDs are stored here (using a list like this serves to keep our SQL simple).
+        $this->invalidationsToExclude = [];
+
+        $this->periodIdsToLabels = array_flip(Piwik::$idPeriods);
+    }
+
+    public function getNextArchivesToProcess()
+    {
+        if (empty($this->idSite)) {
+            $this->idSite = $this->getNextIdSiteToArchive();
+            if (empty($this->idSite)) { // no sites left to archive, stop
+                $this->logger->debug("No more sites left to archive, stopping.");
+                return null;
+            }
+
+            /**
+             * This event is triggered before the cron archiving process starts archiving data for a single
+             * site.
+             *
+             * Note: multiple archiving processes can post this event.
+             *
+             * @param int $idSite The ID of the site we're archiving data for.
+             * @param string $pid The PID of the process processing archives for this site.
+             */
+            Piwik::postEvent('CronArchive.archiveSingleSite.start', array($this->idSite, $this->pid));
+
+            $this->logger->info("Start processing archives for site {idSite}.", ['idSite' => $this->idSite]);
+
+            $this->siteTimer = new Timer();
+            $this->siteRequests = 0;
+        }
+
+        // we don't want to invalidate different periods together or segment archives w/ no-segment archives
+        // together, but it's possible to end up querying these archives. if we find one, we keep track of it
+        // in this array to exclude, but after we run the current batch, we reset the array so we'll still
+        // process them eventually.
+        $invalidationsToExcludeInBatch = [];
+
+        $siteCreationTime = Date::factory(Site::getCreationDateFor($this->idSite));
+
+        // get archives to process simultaneously
+        $archivesToProcess = [];
+        while (count($archivesToProcess) < $this->countOfProcesses) {
+            $invalidatedArchive = $this->getNextInvalidatedArchive($this->idSite, array_keys($invalidationsToExcludeInBatch));
+            if (empty($invalidatedArchive)) {
+                $this->logger->debug("No next invalidated archive.");
+                break;
+            }
+
+            $invalidationDesc = $this->getInvalidationDescription($invalidatedArchive);
+
+            if ($invalidatedArchive['periodObj']->getDateEnd()->isEarlier($siteCreationTime)) {
+                $this->logger->debug("Invalidation is for period that is older than the site's creation time, ignoring: $invalidationDesc");
+                $this->model->deleteInvalidations([$invalidatedArchive]);
+                continue;
+            }
+
+            if (!empty($invalidatedArchive['plugin'])
+                && !Manager::getInstance()->isPluginActivated($invalidatedArchive['plugin'])
+            ) {
+                $this->logger->debug("Plugin specific archive {$invalidatedArchive['idarchive']}'s plugin is deactivated, ignoring $invalidationDesc.");
+                $this->model->deleteInvalidations([$invalidatedArchive]);
+                continue;
+            }
+
+            if ($this->hasDifferentDoneFlagType($archivesToProcess, $invalidatedArchive['name'])) {
+                $this->logger->debug("Found archive with different done flag type (segment vs. no segment) in concurrent batch, skipping until next batch: $invalidationDesc");
+
+                $idinvalidation = $invalidatedArchive['idinvalidation'];
+                $invalidationsToExcludeInBatch[$idinvalidation] = true;
+
+                continue;
+            }
+
+            if ($invalidatedArchive['segment'] === null) {
+                $this->logger->debug("Found archive for segment that is not auto archived, ignoring: $invalidationDesc");
+                $this->addInvalidationToExclude($invalidatedArchive);
+                continue;
+            }
+
+            if ($this->archiveArrayContainsArchive($archivesToProcess, $invalidatedArchive)) {
+                $this->logger->debug("Found duplicate invalidated archive {$invalidatedArchive['idarchive']}, ignoring: $invalidationDesc");
+                $this->addInvalidationToExclude($invalidatedArchive);
+                $this->model->deleteInvalidations([$invalidatedArchive]);
+                continue;
+            }
+
+            if ($this->hasIntersectingPeriod($archivesToProcess, $invalidatedArchive)) {
+                $this->logger->debug("Found archive with intersecting period with others in concurrent batch, skipping until next batch: $invalidationDesc");
+
+                $idinvalidation = $invalidatedArchive['idinvalidation'];
+                $invalidationsToExcludeInBatch[$idinvalidation] = true;
+                continue;
+            }
+
+            $reason = $this->shouldSkipArchive($invalidatedArchive);
+            if ($reason) {
+                $this->logger->debug("Skipping invalidated archive {$invalidatedArchive['idinvalidation']}, $reason: $invalidationDesc");
+                $this->addInvalidationToExclude($invalidatedArchive);
+                continue;
+            }
+
+            if ($this->usableArchiveExists($invalidatedArchive)) {
+                $this->logger->debug("Found invalidation with usable archive (not yet outdated) skipping until archive is out of date: $invalidationDesc");
+                $this->addInvalidationToExclude($invalidatedArchive);
+                continue;
+            }
+
+            if ($this->canSkipArchiveBecauseNoPoint($invalidatedArchive)) {
+                $this->logger->debug("Found invalidated archive we can skip (no visits): $invalidationDesc");
+                $this->addInvalidationToExclude($invalidatedArchive);
+                $this->model->deleteInvalidations([$invalidatedArchive]);
+                continue;
+            }
+
+            // TODO: should use descriptive string instead of just invalidation ID
+            $reason = $this->shouldSkipArchiveBecauseLowerPeriodOrSegmentIsInProgress($invalidatedArchive);
+            if ($reason) {
+                $this->logger->debug("Skipping invalidated archive, $reason: $invalidationDesc");
+                $invalidationsToExcludeInBatch[$invalidatedArchive['idinvalidation']] = true;
+                $this->addInvalidationToExclude($invalidatedArchive);
+                continue;
+            }
+
+            $started = $this->model->startArchive($invalidatedArchive);
+            if (!$started) { // another process started on this archive, pull another one
+                $this->logger->debug("Archive invalidation is being handled by another process: $invalidationDesc");
+                $this->addInvalidationToExclude($invalidatedArchive);
+                continue;
+            }
+
+            $this->addInvalidationToExclude($invalidatedArchive);
+
+            $this->logger->debug("Processing invalidation: $invalidationDesc.");
+
+            $archivesToProcess[] = $invalidatedArchive;
+        }
+
+        if (empty($archivesToProcess)
+            && empty($invalidationsToExcludeInBatch)
+        ) { // no invalidated archive left
+            /**
+             * This event is triggered immediately after the cron archiving process starts archiving data for a single
+             * site.
+             *
+             * Note: multiple archiving processes can post this event.
+             *
+             * @param int $idSite The ID of the site we're archiving data for.
+             * @param string $pid The PID of the process processing archives for this site.
+             */
+            Piwik::postEvent('CronArchive.archiveSingleSite.finish', array($this->idSite, $this->pid));
+
+            $this->logger->info("Finished archiving for site {idSite}, {requests} API requests, {timer} [{processed} / {totalNum} done]", [
+                'idSite' => $this->idSite,
+                'processed' => $this->websiteIdArchiveList->getNumProcessedWebsites(),
+                'totalNum' => $this->websiteIdArchiveList->getNumSites(),
+                'timer' => $this->siteTimer,
+                'requests' => $this->siteRequests,
+            ]);
+
+            $this->idSite = null;
+        }
+
+        $this->siteRequests += count($archivesToProcess);
+
+        return $archivesToProcess;
+    }
+
+    private function archiveArrayContainsArchive($archiveArray, $archive)
+    {
+        foreach ($archiveArray as $entry) {
+            if ($entry['idsite'] == $archive['idsite']
+                && $entry['period'] == $archive['period']
+                && $entry['date1'] == $archive['date1']
+                && $entry['date2'] == $archive['date2']
+                && $entry['name'] == $archive['name']
+                && $entry['plugin'] == $archive['plugin']
+                && $entry['report'] == $archive['report']
+            ) {
+                return true;
+            }
+        }
+        return false;
+    }
+
+    private function getNextInvalidatedArchive($idSite, $extraInvalidationsToIgnore)
+    {
+        $lastInvalidationTime = CronArchive::getLastInvalidationTime();
+        if (empty($lastInvalidationTime)
+            || (time() - $lastInvalidationTime) >= 3600
+        ) {
+            $this->cronArchive->invalidateArchivedReportsForSitesThatNeedToBeArchivedAgain();
+        }
+
+        $iterations = 0;
+        while ($iterations < 100) {
+            $invalidationsToExclude = array_merge($this->invalidationsToExclude, $extraInvalidationsToIgnore);
+
+            $nextArchive = $this->model->getNextInvalidatedArchive($idSite, $invalidationsToExclude);
+            if (empty($nextArchive)) {
+                break;
+            }
+
+            $this->detectPluginForArchive($nextArchive);
+
+            $periodLabel = $this->periodIdsToLabels[$nextArchive['period']];
+            $periodDate = $periodLabel == 'range' ? $nextArchive['date1'] . ',' . $nextArchive['date2'] : $nextArchive['date1'];
+            $nextArchive['periodObj'] = PeriodFactory::build($periodLabel, $periodDate);
+
+            $isCronArchivingEnabled = $this->findSegmentForArchive($nextArchive);
+            if ($isCronArchivingEnabled) {
+                return $nextArchive;
+            }
+
+            $this->logger->debug("Found invalidation for segment that does not have auto archiving enabled, skipping: {$nextArchive['idinvalidation']}");
+            $this->invalidationsToExclude[] = $nextArchive['idinvalidation'];
+
+            ++$iterations;
+        }
+
+        return null;
+    }
+
+    private function shouldSkipArchive($archive)
+    {
+        if ($this->archiveFilter) {
+            return $this->archiveFilter->filterArchive($archive);
+        }
+
+        return false;
+    }
+
+    // public for tests
+    public function canSkipArchiveBecauseNoPoint(array $invalidatedArchive)
+    {
+        $site = new Site($invalidatedArchive['idsite']);
+
+        $periodLabel = $this->periodIdsToLabels[$invalidatedArchive['period']];
+        $dateStr = $periodLabel == 'range' ? ($invalidatedArchive['date1'] . ',' . $invalidatedArchive['date2']) : $invalidatedArchive['date1'];
+        $period = PeriodFactory::build($periodLabel, $dateStr);
+
+        $segment = new Segment($invalidatedArchive['segment'], [$invalidatedArchive['idsite']]);
+
+        $params = new Parameters($site, $period, $segment);
+
+        $loader = new Loader($params);
+        return $loader->canSkipThisArchive(); // if no point in archiving, skip
+    }
+
+    private function shouldSkipArchiveBecauseLowerPeriodOrSegmentIsInProgress(array $archiveToProcess)
+    {
+        $inProgressArchives = $this->cliMultiRequestParser->getInProgressArchivingCommands();
+
+        foreach ($inProgressArchives as $archiveBeingProcessed) {
+            if (empty($archiveBeingProcessed['period'])
+                || empty($archiveBeingProcessed['date'])
+            ) {
+                continue;
+            }
+
+            $archiveBeingProcessed['periodObj'] = PeriodFactory::build($archiveBeingProcessed['period'], $archiveBeingProcessed['date']);
+
+            if ($this->isArchiveOfLowerPeriod($archiveToProcess, $archiveBeingProcessed)) {
+                return "lower period in progress (period = {$archiveBeingProcessed['period']}, date = {$archiveBeingProcessed['date']})";
+            }
+
+            if ($this->isArchiveNonSegmentAndInProgressArchiveSegment($archiveToProcess, $archiveBeingProcessed)) {
+                return "segment archive in progress for same site/period ({$archiveBeingProcessed['segment']})";
+            }
+        }
+
+        return false;
+    }
+
+    private function isArchiveOfLowerPeriod(array $archiveToProcess, $archiveBeingProcessed)
+    {
+        /** @var Period $archiveToProcessPeriodObj */
+        $archiveToProcessPeriodObj = $archiveToProcess['periodObj'];
+        /** @var Period $archivePeriodObj */
+        $archivePeriodObj = $archiveBeingProcessed['periodObj'];
+
+        if ($archiveToProcessPeriodObj->getId() >= $archivePeriodObj->getId()
+            && $archiveToProcessPeriodObj->isPeriodIntersectingWith($archivePeriodObj)
+        ) {
+            return true;
+        }
+
+        return false;
+    }
+
+    private function isArchiveNonSegmentAndInProgressArchiveSegment(array $archiveToProcess, array $archiveBeingProcessed)
+    {
+        // archive is for different site/period
+        if (empty($archiveBeingProcessed['idSite'])
+            || $archiveToProcess['idsite'] != $archiveBeingProcessed['idSite']
+            || $archiveToProcess['periodObj']->getId() != $archiveBeingProcessed['periodObj']->getId()
+            || $archiveToProcess['periodObj']->getDateStart()->toString() != $archiveBeingProcessed['periodObj']->getDateStart()->toString()
+        ) {
+            return false;
+        }
+
+        return empty($archiveToProcess['segment']) && !empty($archiveBeingProcessed['segment']);
+    }
+
+    private function detectPluginForArchive(&$archive)
+    {
+        $archive['plugin'] = $this->getPluginNameForArchiveIfAny($archive);
+    }
+
+    private function hasIntersectingPeriod(array $archivesToProcess, $invalidatedArchive)
+    {
+        if (empty($archivesToProcess)) {
+            return false;
+        }
+
+        foreach ($archivesToProcess as $archive) {
+            if ($archive['periodObj']->isPeriodIntersectingWith($invalidatedArchive['periodObj'])) {
+                return true;
+            }
+        }
+
+        return false;
+    }
+
+    private function findSegmentForArchive(&$archive)
+    {
+        $flag = explode('.', $archive['name'])[0];
+        if ($flag == 'done') {
+            $archive['segment'] = '';
+            return true;
+        }
+
+        $hash = substr($flag, 4);
+        $storedSegment = $this->segmentArchiving->findSegmentForHash($hash, $archive['idsite']);
+        if (!isset($storedSegment['definition'])) {
+            $archive['segment'] = null;
+            return false;
+        }
+
+        $archive['segment'] = $storedSegment['definition'];
+        return $this->segmentArchiving->isAutoArchivingEnabledFor($storedSegment);
+    }
+
+    private function hasDifferentDoneFlagType(array $archivesToProcess, $name)
+    {
+        if (empty($archivesToProcess)) {
+            return false;
+        }
+
+        $existingDoneFlagType = $this->getDoneFlagType($archivesToProcess[0]['name']);
+        $newArchiveDoneFlagType = $this->getDoneFlagType($name);
+
+        return $existingDoneFlagType != $newArchiveDoneFlagType;
+    }
+
+    private function getPluginNameForArchiveIfAny($archive)
+    {
+        $name = $archive['name'];
+        if (strpos($name, '.') === false) {
+            return null;
+        }
+
+        $parts = explode('.', $name);
+        return $parts[1];
+    }
+
+    private function getDoneFlagType($name)
+    {
+        if ($name == 'done') {
+            return 'all';
+        } else {
+            return 'segment';
+        }
+    }
+
+    private function addInvalidationToExclude(array $invalidatedArchive)
+    {
+        $id = $invalidatedArchive['idinvalidation'];
+        if (empty($this->invalidationsToExclude[$id])) {
+            $this->invalidationsToExclude[$id] = $id;
+        }
+    }
+
+    private function getNextIdSiteToArchive()
+    {
+        return $this->websiteIdArchiveList->getNextSiteId();
+    }
+
+    private function getInvalidationDescription(array $invalidatedArchive)
+    {
+        return sprintf("[idinvalidation = %s, idsite = %s, period = %s(%s - %s), name = %s]",
+            $invalidatedArchive['idinvalidation'],
+            $invalidatedArchive['idsite'],
+            $this->periodIdsToLabels[$invalidatedArchive['period']],
+            $invalidatedArchive['date1'],
+            $invalidatedArchive['date2'],
+            $invalidatedArchive['name']
+        );
+    }
+
+    // public for test
+    public function usableArchiveExists(array $invalidatedArchive)
+    {
+        $site = new Site($invalidatedArchive['idsite']);
+
+        $periodLabel = $this->periodIdsToLabels[$invalidatedArchive['period']];
+        $dateStr = $periodLabel == 'range' ? ($invalidatedArchive['date1'] . ',' . $invalidatedArchive['date2']) : $invalidatedArchive['date1'];
+        $period = PeriodFactory::build($periodLabel, $dateStr);
+
+        $segment = new Segment($invalidatedArchive['segment'], [$invalidatedArchive['idsite']]);
+
+        $params = new Parameters($site, $period, $segment);
+
+        // if latest archive includes today and is usable (DONE_OK or DONE_INVALIDATED and recent enough), skip
+        $today = Date::factoryInTimezone('today', Site::getTimezoneFor($site->getId()))->subSeconds(1);
+        $isArchiveIncludesToday = $period->isDateInPeriod($today);
+        if (!$isArchiveIncludesToday) {
+            return false;
+        }
+
+        // if valid archive already exists, do not re-archive
+        $minDateTimeProcessedUTC = Date::now()->subSeconds(Rules::getPeriodArchiveTimeToLiveDefault($periodLabel));
+        $archiveIdAndVisits = ArchiveSelector::getArchiveIdAndVisits($params, $minDateTimeProcessedUTC, $includeInvalidated = true);
+
+        $idArchive = $archiveIdAndVisits[0];
+        return !empty($idArchive);
+    }
+}
\ No newline at end of file
diff --git a/core/CronArchive/SegmentArchiving.php b/core/CronArchive/SegmentArchiving.php
index 61418cd9bf..ba2c2a4790 100644
--- a/core/CronArchive/SegmentArchiving.php
+++ b/core/CronArchive/SegmentArchiving.php
@@ -31,7 +31,7 @@ class SegmentArchiving
     const BEGINNING_OF_TIME = 'beginning_of_time';
     const CREATION_TIME = 'segment_creation_time';
     const LAST_EDIT_TIME = 'segment_last_edit_time';
-    const DEFAULT_BEGINNIN_OF_TIME_LAST_N_YEARS = 7;
+    const DEFAULT_BEGINNING_OF_TIME_LAST_N_YEARS = 7;
 
     /**
      * @var Model
@@ -65,7 +65,7 @@ class SegmentArchiving
      */
     private $forceArchiveAllSegments;
 
-    public function __construct($processNewSegmentsFrom, $beginningOfTimeLastNInYears = self::DEFAULT_BEGINNIN_OF_TIME_LAST_N_YEARS,
+    public function __construct($processNewSegmentsFrom, $beginningOfTimeLastNInYears = self::DEFAULT_BEGINNING_OF_TIME_LAST_N_YEARS,
                                 Model $segmentEditorModel = null, Cache $segmentListCache = null, Date $now = null,
                                 LoggerInterface $logger = null)
     {
@@ -80,6 +80,11 @@ class SegmentArchiving
 
     public function getSegmentArchivesToInvalidateForNewSegments($idSite)
     {
+        return $this->getSegmentArchivesToInvalidate($idSite, true);
+    }
+
+    public function getSegmentArchivesToInvalidate($idSite, $checkOnlyForNewSegments = false)
+    {
         $result = [];
 
         $segmentsForSite = $this->getAllSegments();
@@ -88,7 +93,7 @@ class SegmentArchiving
                 continue;
             }
 
-            $oldestDateToProcessForNewSegment = $this->getOldestDateToProcessForNewSegment($idSite, $storedSegment);
+            $oldestDateToProcessForNewSegment = $this->getOldestDateToProcessForNewSegment($idSite, $storedSegment, $checkOnlyForNewSegments);
             if (empty($oldestDateToProcessForNewSegment)) {
                 continue;
             }
@@ -128,7 +133,7 @@ class SegmentArchiving
         return null;
     }
 
-    private function getOldestDateToProcessForNewSegment($idSite, $storedSegment)
+    private function getOldestDateToProcessForNewSegment($idSite, $storedSegment, $checkOnlyForNewSegments)
     {
         /**
          * @var Date $segmentCreatedTime
@@ -145,11 +150,13 @@ class SegmentArchiving
         }
 
         $segmentTimeToUse = $segmentLastEditedTime ?: $segmentCreatedTime;
-        if (!empty($lastInvalidationTime)
-            && !empty($segmentTimeToUse)
-            && $segmentTimeToUse->isEarlier($lastInvalidationTime)
-        ) {
-            return null; // has already have been invalidated, ignore
+        if ($checkOnlyForNewSegments) {
+            if (!empty($lastInvalidationTime)
+                && !empty($segmentTimeToUse)
+                && $segmentTimeToUse->isEarlier($lastInvalidationTime)
+            ) {
+                return null; // has already have been invalidated, ignore
+            }
         }
 
         if ($this->processNewSegmentsFrom == self::CREATION_TIME) {
diff --git a/core/DataAccess/ArchiveSelector.php b/core/DataAccess/ArchiveSelector.php
index cb33bad34f..493e208fe6 100644
--- a/core/DataAccess/ArchiveSelector.php
+++ b/core/DataAccess/ArchiveSelector.php
@@ -57,7 +57,7 @@ class ArchiveSelector
      *               - the ts_archived for the latest usable archive
      * @throws Exception
      */
-    public static function getArchiveIdAndVisits(ArchiveProcessor\Parameters $params, $minDatetimeArchiveProcessedUTC = false, $includeInvalidated = true)
+    public static function getArchiveIdAndVisits(ArchiveProcessor\Parameters $params, $minDatetimeArchiveProcessedUTC = false, $includeInvalidated = null)
     {
         $idSite       = $params->getSite()->getId();
         $period       = $params->getPeriod()->getId();
@@ -73,7 +73,7 @@ class ArchiveSelector
 
         $doneFlags      = Rules::getDoneFlags($plugins, $segment);
         $requestedPluginDoneFlags = Rules::getDoneFlags([$requestedPlugin], $segment);
-        $doneFlagValues = Rules::getSelectableDoneFlagValues($includeInvalidated, $params);
+        $doneFlagValues = Rules::getSelectableDoneFlagValues($includeInvalidated === null ? true : $includeInvalidated, $params, $includeInvalidated === null);
 
         $results = self::getModel()->getArchiveIdAndVisits($numericTable, $idSite, $period, $dateStartIso, $dateEndIso, null, $doneFlags);
         if (empty($results)) { // no archive found
@@ -86,6 +86,11 @@ class ArchiveSelector
         $visits = isset($result['nb_visits']) ? $result['nb_visits'] : false;
         $visitsConverted = isset($result['nb_visits_converted']) ? $result['nb_visits_converted'] : false;
 
+        $result['idarchive'] = empty($result['idarchive']) ? [] : [$result['idarchive']];
+        if (isset($result['partial'])) {
+            $result['idarchive'] = array_merge($result['idarchive'], $result['partial']);
+        }
+
         if (isset($result['value'])
             && !in_array($result['value'], $doneFlagValues)
         ) { // the archive cannot be considered valid for this request (has wrong done flag value)
@@ -96,25 +101,17 @@ class ArchiveSelector
             $minDatetimeArchiveProcessedUTC = Date::factory($minDatetimeArchiveProcessedUTC);
         }
 
-        if (!empty($minDatetimeArchiveProcessedUTC) && !is_object($minDatetimeArchiveProcessedUTC)) {
-            $minDatetimeArchiveProcessedUTC = Date::factory($minDatetimeArchiveProcessedUTC);
-        }
-
-        if (!empty($minDatetimeArchiveProcessedUTC) && !is_object($minDatetimeArchiveProcessedUTC)) {
-            $minDatetimeArchiveProcessedUTC = Date::factory($minDatetimeArchiveProcessedUTC);
-        }
-
         // the archive is too old
         if ($minDatetimeArchiveProcessedUTC
-            && isset($result['idarchive'])
+            && !empty($result['idarchive'])
             && Date::factory($tsArchived)->isEarlier($minDatetimeArchiveProcessedUTC)
         ) {
             return [false, $visits, $visitsConverted, true, $tsArchived];
         }
 
-        $idArchive = isset($result['idarchive']) ? $result['idarchive'] : false;
+        $idArchives = !empty($result['idarchive']) ? $result['idarchive'] : false;
 
-        return [$idArchive, $visits, $visitsConverted, true, $tsArchived];
+        return [$idArchives, $visits, $visitsConverted, true, $tsArchived];
     }
 
     /**
@@ -264,7 +261,8 @@ class ArchiveSelector
         $getValuesSql = "SELECT value, name, idsite, date1, date2, ts_archived
                                 FROM %s
                                 WHERE idarchive IN (%s)
-                                  AND " . $whereNameIs;
+                                  AND " . $whereNameIs . "
+                             ORDER BY ts_archived ASC"; // ascending order so we use the latest data found
 
         // get data from every table we're querying
         $rows = array();
@@ -381,10 +379,12 @@ class ArchiveSelector
     {
         // find latest idarchive for each done flag
         $idArchives = [];
+        $tsArchiveds = [];
         foreach ($results as $row) {
             $doneFlag = $row['name'];
-            if (!isset($idArchives[$doneFlag])) {
+            if (!isset($idArchives[$doneFlag]) && $row['value'] != ArchiveWriter::DONE_PARTIAL) {
                 $idArchives[$doneFlag] = $row['idarchive'];
+                $tsArchiveds[$doneFlag] = $row['ts_archived'];
             }
         }
 
@@ -393,7 +393,7 @@ class ArchiveSelector
             self::NB_VISITS_CONVERTED_RECORD_LOOKED_UP => false,
         ];
 
-        foreach ($results as &$result) {
+        foreach ($results as $result) {
             if (in_array($result['name'], $requestedPluginDoneFlags)
                 && in_array($result['idarchive'], $idArchives)
             ) {
@@ -422,6 +422,22 @@ class ArchiveSelector
             }
         }
 
+        // add partial archives
+        foreach ($results as $row) {
+            if (!isset($idArchives[$row['name']])) {
+                continue;
+            }
+
+            $mainTsArchived = $tsArchiveds[$row['name']];
+            $thisTsArchived = $row['ts_archived'];
+
+            if ($row['value'] === ArchiveWriter::DONE_PARTIAL
+                && ($mainTsArchived == $thisTsArchived || Date::factory($mainTsArchived)->isEarlier($thisTsArchived))
+            ) {
+                $idArchives['partial'][] = $row['idarchive'];
+            }
+        }
+
         return $archiveData;
     }
 }
diff --git a/core/DataAccess/ArchiveWriter.php b/core/DataAccess/ArchiveWriter.php
index a8446db9b4..49d8a52104 100644
--- a/core/DataAccess/ArchiveWriter.php
+++ b/core/DataAccess/ArchiveWriter.php
@@ -12,6 +12,7 @@ use Exception;
 use Piwik\Archive\Chunk;
 use Piwik\ArchiveProcessor\Rules;
 use Piwik\ArchiveProcessor;
+use Piwik\Date;
 use Piwik\Db;
 use Piwik\Db\BatchInsert;
 
@@ -56,12 +57,11 @@ class ArchiveWriter
     const DONE_INVALIDATED = 4;
 
     /**
-     * Flag indicating that the archive is currently being archived. If the archiving process is aborted or killed, the
-     * archive may remain w/ this flag.
+     * Flag indicating that the archive is
      *
      * @var int
      */
-    const DONE_IN_PROGRESS = 5;
+    const DONE_PARTIAL = 5;
 
     protected $fields = array('idarchive',
         'idsite',
@@ -80,6 +80,16 @@ class ArchiveWriter
     const MAX_SPOOL_SIZE = 50;
 
     /**
+     * @var ArchiveProcessor\Parameters
+     */
+    private $parameters;
+
+    /**
+     * @var string
+     */
+    private $earliestNow;
+
+    /**
      * ArchiveWriter constructor.
      * @param ArchiveProcessor\Parameters $params
      * @param bool $isArchiveTemporary Deprecated. Has no effect.
@@ -91,6 +101,7 @@ class ArchiveWriter
         $this->idSite    = $params->getSite()->getId();
         $this->segment   = $params->getSegment();
         $this->period    = $params->getPeriod();
+        $this->parameters = $params;
 
         $idSites = array($this->idSite);
         $this->doneFlag = Rules::getDoneStringFlagFor($idSites, $this->segment, $this->period->getLabel(), $params->getRequestedPlugin());
@@ -141,8 +152,9 @@ class ArchiveWriter
 
     public function initNewArchive()
     {
-        $this->allocateNewArchiveId();
+        $idArchive = $this->allocateNewArchiveId();
         $this->logArchiveStatusAsIncomplete();
+        return $idArchive;
     }
 
     public function finalizeArchive()
@@ -152,7 +164,15 @@ class ArchiveWriter
         $numericTable = $this->getTableNumeric();
         $idArchive    = $this->getIdArchive();
 
-        $this->getModel()->updateArchiveStatus($numericTable, $idArchive, $this->doneFlag, self::DONE_OK);
+        $doneValue = $this->parameters->isPartialArchive() ? self::DONE_PARTIAL : self::DONE_OK;
+        $this->getModel()->updateArchiveStatus($numericTable, $idArchive, $this->doneFlag, $doneValue);
+
+        if (!$this->parameters->isPartialArchive()
+            // sanity check, just in case nothing was inserted (the archive status should always be inserted)
+            && !empty($this->earliestNow)
+        ) {
+            $this->getModel()->deleteOlderArchives($this->parameters, $this->doneFlag, $this->earliestNow, $this->idArchive);
+        }
     }
 
     protected function compress($data)
@@ -273,12 +293,16 @@ class ArchiveWriter
 
     protected function getInsertRecordBind()
     {
+        $now = Date::now()->getDatetime();
+        if (empty($this->earliestNow)) {
+            $this->earliestNow = $now;
+        }
         return array($this->getIdArchive(),
             $this->idSite,
             $this->dateStart->toString('Y-m-d'),
             $this->period->getDateEnd()->toString('Y-m-d'),
             $this->period->getId(),
-            date("Y-m-d H:i:s"));
+            $now);
     }
 
     protected function getTableNameToInsert($value)
diff --git a/core/DataAccess/LogAggregator.php b/core/DataAccess/LogAggregator.php
index 860ab7bbc8..7697f6b9e4 100644
--- a/core/DataAccess/LogAggregator.php
+++ b/core/DataAccess/LogAggregator.php
@@ -166,6 +166,11 @@ class LogAggregator
     private $allowUsageSegmentCache = false;
 
     /**
+     * @var Parameters
+     */
+    private $params;
+
+    /**
      * Constructor.
      *
      * @param \Piwik\ArchiveProcessor\Parameters $params
@@ -178,6 +183,7 @@ class LogAggregator
         $this->sites = $params->getIdSites();
         $this->isRootArchiveRequest = $params->isRootArchiveRequest();
         $this->logger = $logger ?: StaticContainer::get('Psr\Log\LoggerInterface');
+        $this->params = $params;
     }
 
     public function setSites($sites)
@@ -328,7 +334,7 @@ class LogAggregator
         if (!$this->segment->isEmpty() && $this->isSegmentCacheEnabled()) {
             // here we create the TMP table and apply the segment including the datetime and the requested idsite
             // at the end we generated query will no longer need to apply the datetime/idsite and segment
-            $segment = new Segment('', $this->sites);
+            $segment = new Segment('', $this->sites, $this->params->getPeriod()->getDateTimeStart(), $this->params->getPeriod()->getDateTimeEnd());
 
             $segmentTable = $this->getSegmentTmpTableName();
 
diff --git a/core/DataAccess/Model.php b/core/DataAccess/Model.php
index 7a9a322092..6faaeb05ed 100644
--- a/core/DataAccess/Model.php
+++ b/core/DataAccess/Model.php
@@ -21,6 +21,7 @@ use Piwik\DbHelper;
 use Piwik\Period;
 use Piwik\Segment;
 use Piwik\Sequence;
+use Piwik\Site;
 use Psr\Log\LoggerInterface;
 
 /**
@@ -75,12 +76,26 @@ class Model
         $rows = Db::fetchAll($sql);
         foreach ($rows as $row) {
             $duplicateArchives = explode(',', $row['archives']);
-            $countOfArchives = count($duplicateArchives);
 
-            // if there is more than one archive, the older invalidated ones can be deleted
-            if ($countOfArchives > 1) {
-                array_shift($duplicateArchives); // we don't want to delete the latest archive if it is usable
+            // do not consider purging partial archives, if they are the latest archive,
+            // and we don't want to delete the latest archive if it is usable
+            while (!empty($duplicateArchives)) {
+                $pair = $duplicateArchives[0];
+                if (strpos($pair, '.') === false) {
+                    continue; // see below
+                }
+
+                list($idarchive, $value) = explode('.', $pair);
 
+                array_shift($duplicateArchives);
+
+                if ($value != ArchiveWriter::DONE_PARTIAL) {
+                    break;
+                }
+            }
+
+            // if there is more than one archive, the older invalidated ones can be deleted
+            if (!empty($duplicateArchives)) {
                 foreach ($duplicateArchives as $pair) {
                     if (strpos($pair, '.') === false) {
                         $this->logger->info("GROUP_CONCAT cut off the query result, you may have to purge archives again.");
@@ -104,8 +119,13 @@ class Model
         return $result;
     }
 
-    public function updateArchiveAsInvalidated($archiveTable, $idSites, $allPeriodsToInvalidate, Segment $segment = null, $forceInvalidateNonexistantRanges = false)
+    public function updateArchiveAsInvalidated($archiveTable, $idSites, $allPeriodsToInvalidate, Segment $segment = null,
+                                               $forceInvalidateNonexistantRanges = false, $name = null)
     {
+        if (empty($idSites)) {
+            return 0;
+        }
+
         // select all idarchive/name pairs we want to invalidate
         $sql = "SELECT idarchive, idsite, period, date1, date2, `name`, `value`
                   FROM `$archiveTable`
@@ -136,12 +156,23 @@ class Model
             $sql .= ")";
         }
 
-        if ($segment) {
-            $nameCondition = "name LIKE '" . Rules::getDoneFlagArchiveContainsAllPlugins($segment) . "%'";
+        if (!empty($name)) {
+            if (strpos($name, '.') !== false) {
+                list($plugin, $name) = explode('.', $name, 2);
+            } else {
+                $plugin = $name;
+                $name = null;
+            }
+        }
+
+        if (empty($plugin)) {
+            $doneFlag = Rules::getDoneFlagArchiveContainsAllPlugins($segment ?: new Segment('', []));
         } else {
-            $nameCondition = "name LIKE 'done%'";
+            $doneFlag = Rules::getDoneFlagArchiveContainsOnePlugin($segment ?: new Segment('', []), $plugin);
         }
 
+        $nameCondition = "name LIKE '$doneFlag%'";
+
         $sql .= " AND $nameCondition";
 
         $archivesToInvalidate = Db::fetchAll($sql);
@@ -157,8 +188,6 @@ class Model
             Db::query($sql);
         }
 
-        $doneFlag = Rules::getDoneFlagArchiveContainsAllPlugins($segment ?: new Segment('', []));
-
         // we add every archive we need to invalidate + the archives that do not already exist to archive_invalidations.
         // except for archives that are DONE_IN_PROGRESS.
         $archivesToCreateInvalidationRowsFor = [];
@@ -174,6 +203,7 @@ class Model
 
         $dummyArchives = [];
         foreach ($idSites as $idSite) {
+            $siteCreationTime = Date::factory(Site::getCreationDateFor($idSite));
             foreach ($allPeriodsToInvalidate as $period) {
                 if ($period->getLabel() == 'range'
                     && !$forceInvalidateNonexistantRanges
@@ -181,6 +211,10 @@ class Model
                     continue; // range
                 }
 
+                if ($period->getDateEnd()->isEarlier($siteCreationTime)) {
+                    continue; // don't add entries if it is before the time the site was created
+                }
+
                 $date1 = $period->getDateStart()->toString();
                 $date2 = $period->getDateEnd()->toString();
                 $idArchive = $archivesToCreateInvalidationRowsFor[$idSite][$period->getId()][$date1][$date2] ?? null;
@@ -188,6 +222,7 @@ class Model
                 $dummyArchives[] = [
                     'idarchive' => $idArchive,
                     'name' => $doneFlag,
+                    'report' => $name,
                     'idsite' => $idSite,
                     'date1' => $period->getDateStart()->getDatetime(),
                     'date2' => $period->getDateEnd()->getDatetime(),
@@ -197,7 +232,8 @@ class Model
             }
         }
 
-        $fields = ['idarchive', 'name', 'idsite', 'date1', 'date2', 'period', 'ts_invalidated'];
+        $fields = ['idarchive', 'name', 'report', 'idsite', 'date1', 'date2', 'period', 'ts_invalidated'];
+
         Db\BatchInsert::tableInsertBatch(Common::prefixTable('archive_invalidations'), $fields, $dummyArchives);
 
         return count($idArchives);
@@ -213,6 +249,10 @@ class Model
      */
     public function updateRangeArchiveAsInvalidated($archiveTable, $idSites, $allPeriodsToInvalidate, Segment $segment = null)
     {
+        if (empty($idSites)) {
+            return;
+        }
+
         $bind = array();
 
         $periodConditions = array();
@@ -321,6 +361,25 @@ class Model
         return $deletedRows;
     }
 
+    public function deleteOlderArchives(Parameters $params, $name, $tsArchived, $idArchive)
+    {
+        $dateStart = $params->getPeriod()->getDateStart();
+        $dateEnd = $params->getPeriod()->getDateEnd();
+
+        $numericTable = ArchiveTableCreator::getNumericTable($dateStart);
+        $blobTable = ArchiveTableCreator::getBlobTable($dateEnd);
+
+        $sql = "SELECT idarchive FROM `$numericTable` WHERE idsite = ? AND date1 = ? AND date2 = ? AND period = ? AND name = ? AND ts_archived < ? AND idarchive < ?";
+
+        $idArchives = Db::fetchAll($sql, [$params->getSite()->getId(), $dateStart, $dateEnd, $params->getPeriod()->getId(), $name, $tsArchived, $idArchive]);
+        $idArchives = array_column($idArchives, 'idarchive');
+        if (empty($idArchives)) {
+            return;
+        }
+
+        $this->deleteArchiveIds($numericTable, $blobTable, $idArchives);
+    }
+
     public function getArchiveIdAndVisits($numericTable, $idSite, $period, $dateStartIso, $dateEndIso, $minDatetimeIsoArchiveProcessedUTC,
                                           $doneFlags, $doneFlagValues = null)
     {
@@ -351,6 +410,7 @@ class Model
                          $timeStampWhere
                          AND arc1.ts_archived IS NOT NULL
                      ORDER BY arc1.ts_archived DESC, arc1.idarchive DESC";
+
         $results = Db::fetchAll($sqlQuery, $bindSQL);
 
         return $results;
@@ -599,11 +659,12 @@ class Model
     public function getNextInvalidatedArchive($idSite, $idInvalidationsToExclude = null, $useLimit = true)
     {
         $table = Common::prefixTable('archive_invalidations');
-        $sql = "SELECT idinvalidation, idarchive, idsite, date1, date2, period, `name`
+        $sql = "SELECT idinvalidation, idarchive, idsite, date1, date2, period, `name`, report
                   FROM `$table`
-                 WHERE idsite = ?";
+                 WHERE idsite = ? AND status != ?";
         $bind = [
             $idSite,
+            ArchiveInvalidator::INVALIDATION_STATUS_IN_PROGRESS,
         ];
 
         if (!empty($idInvalidationsToExclude)) {
diff --git a/core/DataTable/Map.php b/core/DataTable/Map.php
index 93921562c7..9169a36128 100644
--- a/core/DataTable/Map.php
+++ b/core/DataTable/Map.php
@@ -12,6 +12,7 @@ use Closure;
 use Piwik\Common;
 use Piwik\DataTable;
 use Piwik\DataTable\Renderer\Console;
+use Piwik\DataTable\Renderer\Html;
 
 /**
  * Stores an array of {@link DataTable}s indexed by one type of {@link DataTable} metadata (such as site ID
@@ -221,7 +222,7 @@ class Map implements DataTableInterface
      */
     public function __toString()
     {
-        $renderer = new Console();
+        $renderer = new Html();
         $renderer->setTable($this);
         return (string)$renderer;
     }
diff --git a/core/Date.php b/core/Date.php
index 276cd6d0c3..4b67eeb9cb 100644
--- a/core/Date.php
+++ b/core/Date.php
@@ -556,7 +556,7 @@ class Date
      */
     public static function today()
     {
-        return new Date(strtotime(date("Y-m-d 00:00:00")));
+        return new Date(strtotime(date("Y-m-d 00:00:00", self::getNowTimestamp())));
     }
 
     /**
@@ -566,7 +566,7 @@ class Date
      */
     public static function tomorrow()
     {
-        return new Date(strtotime('tomorrow'));
+        return new Date(strtotime('tomorrow', self::getNowTimestamp()));
     }
 
     /**
@@ -576,7 +576,7 @@ class Date
      */
     public static function yesterday()
     {
-        return new Date(strtotime("yesterday"));
+        return new Date(strtotime("yesterday", self::getNowTimestamp()));
     }
 
     /**
@@ -586,7 +586,7 @@ class Date
      */
     public static function yesterdaySameTime()
     {
-        return new Date(strtotime("yesterday " . date('H:i:s')));
+        return new Date(strtotime("yesterday " . date('H:i:s'), self::getNowTimestamp()));
     }
 
     /**
diff --git a/core/Db/Schema/Mysql.php b/core/Db/Schema/Mysql.php
index 90c765ae21..5aaeb1c0f2 100644
--- a/core/Db/Schema/Mysql.php
+++ b/core/Db/Schema/Mysql.php
@@ -318,6 +318,7 @@ class Mysql implements SchemaInterface
                                             period TINYINT UNSIGNED NOT NULL,
                                             ts_invalidated DATETIME NULL,
                                             status TINYINT(1) UNSIGNED DEFAULT 0,
+                                            `report` VARCHAR(255) NULL,
                                             PRIMARY KEY(idinvalidation),
                                             INDEX index_idsite_dates_period_name(idsite, date1, period, name)
                                         ) ENGINE=$engine DEFAULT CHARSET=$charset
diff --git a/core/Http.php b/core/Http.php
index 4999cc912d..506e64380c 100644
--- a/core/Http.php
+++ b/core/Http.php
@@ -154,7 +154,8 @@ class Http
         $httpUsername = null,
         $httpPassword = null,
         $requestBody = null,
-        $additionalHeaders = array()
+        $additionalHeaders = array(),
+        $forcePost = null
     ) {
         if ($followDepth > 5) {
             throw new Exception('Too many redirects (' . $followDepth . ')');
@@ -617,6 +618,9 @@ class Http
                     CURLOPT_FOLLOWLOCATION => true,
                     CURLOPT_MAXREDIRS      => 5,
                 );
+                if ($forcePost) {
+                    $curl_options[CURLOPT_POSTREDIR] = CURL_REDIR_POST_ALL;
+                }
                 @curl_setopt_array($ch, $curl_options);
             }
 
diff --git a/core/Updater/Migration/Db/AddColumns.php b/core/Updater/Migration/Db/AddColumns.php
index 2ab269d421..dcd882dd1b 100644
--- a/core/Updater/Migration/Db/AddColumns.php
+++ b/core/Updater/Migration/Db/AddColumns.php
@@ -7,6 +7,8 @@
  */
 namespace Piwik\Updater\Migration\Db;
 
+use Piwik\DataAccess\TableMetadata;
+
 /**
  * @see Factory::addColumns()
  * @ignore
@@ -15,8 +17,19 @@ class AddColumns extends Sql
 {
     public function __construct($table, $columns, $placeColumnAfter)
     {
+        $tableMetadata = new TableMetadata();
+        try {
+            $existingColumns = $tableMetadata->getColumns($table);
+        } catch (\Exception $ex) {
+            $existingColumns = [];
+        }
+
         $changes = array();
         foreach ($columns as $columnName => $columnType) {
+            if (in_array($columnName, $existingColumns)) {
+                continue;
+            }
+
             $part = sprintf("ADD COLUMN `%s` %s", $columnName, $columnType);
 
             if (!empty($placeColumnAfter)) {
diff --git a/core/Updates/4.0.0-b2.php b/core/Updates/4.0.0-b2.php
index d71e55a7a4..2fd9c8b035 100644
--- a/core/Updates/4.0.0-b2.php
+++ b/core/Updates/4.0.0-b2.php
@@ -47,6 +47,7 @@ class Updates_4_0_0_b2 extends PiwikUpdates
             'period' => 'TINYINT UNSIGNED NOT NULL',
             'ts_invalidated' => 'DATETIME NOT NULL',
             'status' => 'TINYINT(1) UNSIGNED DEFAULT 0',
+            'report' => 'VARCHAR(255) NULL',
         ], ['idinvalidation']);
 
         $migrations[] = $this->migration->db->addIndex('archive_invalidations', ['idsite', 'date1', 'period'], 'index_idsite_dates_period_name');