allow invalidating plugin archives only and archiving past data for plugins (#15889)

* Adding initial new code for cron archive rewrite.

* first pass at removing unused CronArchive code.

* unfinished commit

* fill out archiveinvalidator code

* getting some tests to pass

* unfinished commit

* fixing part of test

* Another test fix.

* another sql change

* fix broken merge or something else that went wrong

* Couple more fixes and extra logs.

* Fixing enough issues to get core archive command to run completely.

* Fix and log change.

* Fixed more segment/test related issues for CronArchiveTest. Includes optimization for no visits for period + segment process from handling.

* another optimization and possible build fix

* no visit optimization

* test fix

* Implement archiving_custom_ranges logic w/ queue based implementation

* fixes to get archivecrontest to work

* add logic to invalidate today period

* fix optimization and some tests

* Fixing more tests.

* Fixing more tests

* debug travis failure

* more test fixes

* more test fixes, removing more unneeded code, handling some TODOs

* Handle more TODOs including creating ArchiveFilter class for some cli options.

* tests and todos

* idarchives are specific to table + start on archivefilter tests

* one test

* more TODOs and tests

* more tests and todo taken care of

* handle more todos

* fixing more tests

* fix comment

* make sure autoarchiving is enabled for segments when cron archive picks them up

* Fixing test.

* apply more pr feedback

* order by date1 asc

* quick refactor

* use batch insert instead of createDummyArchives

* apply rest of pr feedback

* add removed events, add new test, fix an issue (when deleting idarchives older than do not lump all segments together).

* re-add fixed/shared siteids

* fix tests

* incomplete commit

* Insert archive entries into archive_invalidations table.

* Use invalidations table in core:archive and get ArchiveCronTest to pass.

* fixing some tests

* debugging travis

* fix more tests & remove DONE_IN_PROGRESS which is no longer used.

* fix more tests

* Allow forcing plugin specific archive in core:archive.

* When querying from archive data use all available archives including "all" archives and plugin specific archives.

* Adding some code for invalidating specific plugin archives.

* Get archive invalidation test to pass.

* add plugin capability to invalidate command

* Handle plugin only archives in core:archive.

* Add Archive test and get ArchiveCronTest to pass.

* update some expected files

* Fix some more tests.

* incomplete commit

* allow invalidating individual reports

* adding more API for DONE_PARTIAL support

* get archivecrontest to pass

* add archive processor tests

* fix some test randomnes

* when purging keep latest partial archives if there is no newer whole archive

* add rearchivereport method + some unfinished tests

* Add archiveReports API method, fix race condition in test, when archiving single report, always ignore inserting other reports.

* require archivers to handle partial archives themselves entirely instead of trying to do it automatically and allow requested report to be any string

* couple fixes

* Use core config option for last N montsh to invalidate.

* Add test for ArchiveSelector method.

* Ignore archives w/ deactivated plugins.

* Refactor queue looping into new QueueConsumer class.

* apply more review feedback + another fix

* invalidate segments too in reArchiveReport w/ etsts

* remove DONE_IN_PROGRESS, no longer used. use new status in query and add queue consumer test.

* forgot to add file

* delete old unneeded archives when finalizing a new one.

* tweak invalidation archive description

* add plugin archiving tests and get them to pass

* fix test

* many fixes

* fix another test

* update expected test files

* fix more tests

* last test fixes hopefully

* tweak log

* In case a column already exists, do not try to add it in an AddColumns migration or the entire migration will fail and no columns will be added.

* try to fix tests again

* fix again?

* apply some review feedback + fix test

* fix test

* fix another test

* couple fixes

* Remove extra param.

* apply pr feedback

* check for usable archive before invalidating and before initiating archiving

* fixing tests

* fixing tests

* fixing tests

* fix another test issue

* fix archiveinvalidator test

* fix one test and debug another

* more debugging

* fix test

* use twig

* remove no longer needed change

* add back previous logic

* fix tracking is not working

* apply pr feedback and add tests

* fixing tests

* update submodule

* debugging random travis failure

* update test

* more debugging

* more debugging

* another attempt at debugging

* Lets try this fix

* trying to fix the build

* debug

* simpler test

* fix test

* fix test

* fix test

* fix test

* fix test failure

* update screenshots

* update screenshots

Co-authored-by: Thomas Steur <tsteur@users.noreply.github.com>

22 files changed, 451 insertions, 104 deletions

diff --git a/plugins/API/Controller.php b/plugins/API/Controller.php
index eb0b6e51f1..585cb14ba9 100644
--- a/plugins/API/Controller.php
+++ b/plugins/API/Controller.php
@@ -15,6 +15,7 @@ use Piwik\Common;
 use Piwik\Config;
 use Piwik\Piwik;
 use Piwik\Plugin\Report;
+use Piwik\Plugins\API\Renderer\Original;
 use Piwik\Url;
 use Piwik\UrlHelper;
 use Piwik\View;
@@ -27,6 +28,8 @@ class Controller extends \Piwik\Plugin\Controller
     function index()
     {
         $tokenAuth = Common::getRequestVar('token_auth', 'anonymous', 'string');
+        $format = Common::getRequestVar('format', false);
+        $serialize = Common::getRequestVar('serialize', false);
 
         $token = 'token_auth=' . $tokenAuth;
 
@@ -43,6 +46,12 @@ class Controller extends \Piwik\Plugin\Controller
         $response = $request->process();
 
         if (is_array($response)) {
+            if ($format == 'original'
+                && $serialize != 1
+            ) {
+                Original::sendPlainTextHeader();
+            }
+
             $response = var_export($response, true);
         }
 
diff --git a/plugins/API/Renderer/Original.php b/plugins/API/Renderer/Original.php
index 3bcc0631c4..a473278929 100644
--- a/plugins/API/Renderer/Original.php
+++ b/plugins/API/Renderer/Original.php
@@ -15,6 +15,11 @@ use Piwik\DataTable\DataTableInterface;
 
 class Original extends ApiRenderer
 {
+    public static function sendPlainTextHeader()
+    {
+        Common::sendHeader('Content-Type: text/plain; charset=utf-8');
+    }
+
     public function renderSuccess($message)
     {
         return true;
@@ -72,7 +77,7 @@ class Original extends ApiRenderer
     public function sendHeader()
     {
         if ($this->shouldSerialize()) {
-            Common::sendHeader('Content-Type: text/plain; charset=utf-8');
+            self::sendPlainTextHeader();
         }
     }
 
diff --git a/plugins/CoreAdminHome/API.php b/plugins/CoreAdminHome/API.php
index 95b52fbb3d..6f84b954e9 100644
--- a/plugins/CoreAdminHome/API.php
+++ b/plugins/CoreAdminHome/API.php
@@ -13,6 +13,8 @@ use Monolog\Handler\StreamHandler;
 use Monolog\Logger;
 use Piwik\Access;
 use Piwik\ArchiveProcessor\Rules;
+use Piwik\ArchiveProcessor;
+use Piwik\Common;
 use Piwik\Config;
 use Piwik\Container\StaticContainer;
 use Piwik\Archive\ArchiveInvalidator;
@@ -22,6 +24,7 @@ use Piwik\Period\Factory;
 use Piwik\Piwik;
 use Piwik\Segment;
 use Piwik\Scheduler\Scheduler;
+use Piwik\SettingsServer;
 use Piwik\Site;
 use Piwik\Tracker\Failures;
 use Piwik\Url;
@@ -241,6 +244,48 @@ class API extends \Piwik\Plugin\API
     }
 
     /**
+     * @param $idSite
+     * @param $period
+     * @param $date
+     * @param bool $segment
+     * @param bool $plugin
+     * @param bool $report
+     * @return mixed
+     * @throws \Piwik\Exception\UnexpectedWebsiteFoundException
+     * @internal
+     */
+    public function archiveReports($idSite, $period, $date, $segment = false, $plugin = false, $report = false)
+    {
+        if (\Piwik\API\Request::getRootApiRequestMethod() === 'CoreAdminHome.archiveReports') {
+            Piwik::checkUserHasSuperUserAccess();
+        } else {
+            Piwik::checkUserHasViewAccess($idSite);
+        }
+
+        // if cron archiving is running, we will invalidate in CronArchive, not here
+        $isArchivePhpTriggered = SettingsServer::isArchivePhpTriggered();
+        $invalidateBeforeArchiving = !$isArchivePhpTriggered;
+
+        $period = Factory::build($period, $date);
+        $parameters = new ArchiveProcessor\Parameters(new Site($idSite), $period, new Segment($segment, [$idSite], $period->getDateTimeStart(), $period->getDateTimeEnd()));
+        if ($report) {
+            $parameters->setArchiveOnlyReport($report);
+        }
+
+        // TODO: need to test case when there are multiple plugin archives w/ only some data each. does purging remove some that we need?
+        $archiveLoader = new ArchiveProcessor\Loader($parameters, $invalidateBeforeArchiving);
+
+        $result = $archiveLoader->prepareArchive($plugin);
+        if (!empty($result)) {
+            $result = [
+                'idarchives' => $result[0],
+                'nb_visits' => $result[1],
+            ];
+        }
+        return $result;
+    }
+
+    /**
      * Ensure the specified dates are valid.
      * Store invalid date so we can log them
      * @param array|string $dates
diff --git a/plugins/CoreAdminHome/Commands/InvalidateReportData.php b/plugins/CoreAdminHome/Commands/InvalidateReportData.php
index 3f1e7eacb0..b3bb2d115b 100644
--- a/plugins/CoreAdminHome/Commands/InvalidateReportData.php
+++ b/plugins/CoreAdminHome/Commands/InvalidateReportData.php
@@ -53,6 +53,7 @@ class InvalidateReportData extends ConsoleCommand
             . ' also invalidate all days within 2015-09-13,2015-09-19, even those outside the date range.');
         $this->addOption('dry-run', null, InputOption::VALUE_NONE, 'For tests. Runs the command w/o actually '
             . 'invalidating anything.');
+        $this->addOption('plugin', null, InputOption::VALUE_REQUIRED, 'To invalidate data for a specific plugin only.');
         $this->setHelp('Invalidate archived report data by date range, site and period. Invalidated archive data will '
             . 'be re-archived during the next core:archive run. If your log data has changed for some reason, this '
             . 'command can be used to make sure reports are generated using the new, changed log data.');
@@ -64,6 +65,7 @@ class InvalidateReportData extends ConsoleCommand
 
         $cascade = $input->getOption('cascade');
         $dryRun = $input->getOption('dry-run');
+        $plugin = $input->getOption('plugin');
 
         $sites = $this->getSitesToInvalidateFor($input);
         $periodTypes = $this->getPeriodTypesToInvalidateFor($input);
@@ -83,11 +85,16 @@ class InvalidateReportData extends ConsoleCommand
                     $dates = $this->getPeriodDates($periodType, $dateRange);
 
                     if ($dryRun) {
-                        $output->writeln("[Dry-run] invalidating archives for site = [ " . implode(', ', $sites)
+                        $message = "[Dry-run] invalidating archives for site = [ " . implode(', ', $sites)
                             . " ], dates = [ " . implode(', ', $dates) . " ], period = [ $periodType ], segment = [ "
-                            . "$segmentStr ], cascade = [ " . (int)$cascade . " ]");
+                            . "$segmentStr ], cascade = [ " . (int)$cascade . " ]";
+                        if (!empty($plugin)) {
+                            $message .= ", plugin = [ $plugin ]";
+                        }
+                        $output->writeln($message);
                     } else {
-                        $invalidationResult = $invalidator->markArchivesAsInvalidated($sites, $dates, $periodType, $segment, $cascade);
+                        $invalidationResult = $invalidator->markArchivesAsInvalidated($sites, $dates, $periodType, $segment, $cascade,
+                            false, $plugin);
 
                         if ($output->getVerbosity() > OutputInterface::VERBOSITY_NORMAL) {
                             $output->writeln($invalidationResult->makeOutputLogs());
diff --git a/plugins/CoreAdminHome/tests/Integration/Commands/InvalidateReportDataTest.php b/plugins/CoreAdminHome/tests/Integration/Commands/InvalidateReportDataTest.php
index ccca19cb2d..8a4f9e83b3 100644
--- a/plugins/CoreAdminHome/tests/Integration/Commands/InvalidateReportDataTest.php
+++ b/plugins/CoreAdminHome/tests/Integration/Commands/InvalidateReportDataTest.php
@@ -123,9 +123,9 @@ class InvalidateReportDataTest extends ConsoleCommandTestCase
     /**
      * @dataProvider getTestDataForSuccessTests
      */
-    public function test_Command_InvalidatesCorrectSitesAndDates($dates, $periods, $sites, $cascade, $segments, $expectedOutputs)
+    public function test_Command_InvalidatesCorrectSitesAndDates($dates, $periods, $sites, $cascade, $segments, $plugin, $expectedOutputs)
     {
-        $code = $this->applicationTester->run(array(
+        $options = array(
             'command' => 'core:invalidate-report-data',
             '--dates' => $dates,
             '--periods' => $periods,
@@ -134,7 +134,13 @@ class InvalidateReportDataTest extends ConsoleCommandTestCase
             '--segment' => $segments ?: array(),
             '--dry-run' => true,
             '-vvv' => true,
-        ));
+        );
+
+        if (!empty($plugin)) {
+            $options['--plugin'] = $plugin;
+        }
+
+        $code = $this->applicationTester->run($options);
 
         $this->assertEquals(0, $code, $this->getCommandDisplayOutputErrorMessage());
 
@@ -278,6 +284,7 @@ class InvalidateReportDataTest extends ConsoleCommandTestCase
                 '1',
                 false,
                 null,
+                null,
                 array(
                     '[Dry-run] invalidating archives for site = [ 1 ], dates = [ 2012-01-01 ], period = [ day ], segment = [  ]',
                 ),
@@ -289,6 +296,7 @@ class InvalidateReportDataTest extends ConsoleCommandTestCase
                 '1',
                 true,
                 null,
+                null,
                 array(
                     '[Dry-run] invalidating archives for site = [ 1 ], dates = [ 2012-01-01 ], period = [ day ], segment = [  ]',
                 ),
@@ -300,6 +308,7 @@ class InvalidateReportDataTest extends ConsoleCommandTestCase
                 '1',
                 false,
                 null,
+                null,
                 array(
                     '[Dry-run] invalidating archives for site = [ 1 ], dates = [ 2011-12-26 ], period = [ week ], segment = [  ]',
                 ),
@@ -311,6 +320,7 @@ class InvalidateReportDataTest extends ConsoleCommandTestCase
                 '1,3',
                 false,
                 null,
+                null,
                 array(
                     '[Dry-run] invalidating archives for site = [ 1, 3 ], dates = [ 2012-01-01, 2012-02-01 ], period = [ month ], segment = [  ], cascade = [ 0 ]',
                     '[Dry-run] invalidating archives for site = [ 1, 3 ], dates = [ 2012-01-01 ], period = [ month ], segment = [  ], cascade = [ 0 ]',
@@ -327,6 +337,7 @@ class InvalidateReportDataTest extends ConsoleCommandTestCase
                 '2',
                 true,
                 null,
+                null,
                 array(
                     '[Dry-run] invalidating archives for site = [ 2 ], dates = [ 2012-01-30, 2012-02-06 ], period = [ week ], segment = [  ], cascade = [ 1 ]',
                 ),
@@ -338,6 +349,7 @@ class InvalidateReportDataTest extends ConsoleCommandTestCase
                 'all',
                 true,
                 null,
+                null,
                 array(
                     '[Dry-run] invalidating archives for site = [ 1, 2, 3 ], dates = [ 2012-02-01 ], period = [ month ], segment = [  ], cascade = [ 1 ]',
                     '[Dry-run] invalidating archives for site = [ 1, 2, 3 ], dates = [ 2012-03-01 ], period = [ month ], segment = [  ], cascade = [ 1 ]',
@@ -354,10 +366,24 @@ class InvalidateReportDataTest extends ConsoleCommandTestCase
                 'all',
                 true,
                 array('browserCode==FF'),
+                null,
                 array(
                     '[Dry-run] invalidating archives for site = [ 1, 2, 3 ], dates = [ 2011-12-26, 2012-01-02, 2012-01-09 ], period = [ week ], segment = [ browserCode==FF ], cascade = [ 1 ]',
                 ),
             ),
+
+            // w/ plugin
+            [
+                ['2015-05-04'],
+                'day',
+                '1',
+                false,
+                null,
+                'ExamplePlugin',
+                [
+                    '[Dry-run] invalidating archives for site = [ 1 ], dates = [ 2015-05-04 ], period = [ day ], segment = [  ], cascade = [ 0 ], plugin = [ ExamplePlugin ]',
+                ],
+            ],
         );
     }
 }
diff --git a/plugins/CoreConsole/tests/System/ArchiveCronTest.php b/plugins/CoreConsole/tests/System/ArchiveCronTest.php
index e426778427..3b71dcd7ba 100644
--- a/plugins/CoreConsole/tests/System/ArchiveCronTest.php
+++ b/plugins/CoreConsole/tests/System/ArchiveCronTest.php
@@ -8,17 +8,21 @@
 namespace Piwik\Plugins\CoreConsole\tests\System;
 
 use Interop\Container\ContainerInterface;
+use Piwik\Archive\ArchiveInvalidator;
 use Piwik\Common;
 use Piwik\Config;
-use Piwik\CronArchive;
+use Piwik\Container\StaticContainer;
+use Piwik\DataAccess\ArchiveTableCreator;
 use Piwik\Date;
 use Piwik\Db;
 use Piwik\Option;
-use Piwik\Plugins\SitesManager\API;
+use Piwik\Segment;
+use Piwik\Sequence;
 use Piwik\Tests\Framework\TestCase\SystemTestCase;
 use Piwik\Tests\Fixtures\ManySitesImportedLogs;
 use Piwik\Tests\Framework\Fixture;
 use Exception;
+use Psr\Log\LoggerInterface;
 
 /**
  * Tests to call the cron core:archive command script and check there is no error,
@@ -36,6 +40,13 @@ class ArchiveCronTest extends SystemTestCase
      */
     public static $fixture = null; // initialized below class definition
 
+    public static function setUpBeforeClass(): void
+    {
+        parent::setUpBeforeClass();
+
+        Db::exec("UPDATE " . Common::prefixTable('site') . ' SET ts_created = \'2005-01-02 00:00:00\'');
+    }
+
     public function getApiForTesting()
     {
         $apiRequiringSegments = ['Goals.get', 'VisitFrequency.get'];
@@ -48,14 +59,26 @@ class ArchiveCronTest extends SystemTestCase
                                                           'periods'    => array('day', 'week', 'month', 'year'),
                                                           'segment'    => $info['definition'],
                                                           'testSuffix' => '_' . $segmentName));
-
-
         }
 
+        // ExamplePlugin metric
+        $results[] = ['ExamplePlugin.getExampleArchivedMetric', [
+            'idSite' => 'all',
+            'date' => '2007-04-05',
+            'periods' => ['day', 'week'],
+        ]];
+        $results[] = ['Actions.get', [
+            'idSite' => 'all',
+            'date' => '2007-04-05',
+            'periods' => ['day', 'week'],
+            'testSuffix' => '_examplePluginNoMetricsBecauseNoOtherPluginsArchived',
+        ]];
+
         // API Call Without segments
         $results[] = array('VisitsSummary.get', array('idSite'  => 'all',
                                                       'date'    => '2012-08-09',
                                                       'periods' => array('day', 'month', 'year',  'week')));
+
         $results[] = array($apiRequiringSegments, array('idSite'  => 'all',
             'date'    => '2012-08-09',
             'periods' => array('month')));
@@ -92,6 +115,18 @@ class ArchiveCronTest extends SystemTestCase
 
     public function testArchivePhpCron()
     {
+        // invalidate exampleplugin only archives in past
+        $invalidator = StaticContainer::get(ArchiveInvalidator::class);
+        $invalidator->markArchivesAsInvalidated([1], ['2007-04-05'], 'day', new Segment('', [1]), false, false, 'ExamplePlugin');
+
+        // track a visit in 2007-04-05 so it will archive (don't want to force archiving because then this test will take another 15 mins)
+        $tracker = Fixture::getTracker(1, '2007-04-05');
+        $tracker->setUrl('http://example.com/test/url');
+        Fixture::checkResponse($tracker->doTrackPageView('abcdefg'));
+
+        // empty the list so nothing is invalidated during core:archive (so we only archive ExamplePlugin and not all plugins)
+        $invalidator->forgetRememberedArchivedReportsToInvalidate(1, Date::factory('2007-04-05'));
+
         $output = $this->runArchivePhpCron();
 
         $expectedInvalidations = [];
@@ -118,6 +153,40 @@ class ArchiveCronTest extends SystemTestCase
         }
     }
 
+    /**
+     * @depends testArchivePhpCron
+     */
+    public function testArchivePhpCronWithSingleReportRearchive()
+    {
+        // invalidate a report so we get a partial archive (using the metric that gets incremented each time it is archived)
+        // (do it after the last run so we don't end up just re-using the ExamplePlugin archive)
+        $invalidator = StaticContainer::get(ArchiveInvalidator::class);
+        $invalidator->markArchivesAsInvalidated([1], ['2007-04-05'], 'day', new Segment('', [1]), false, false, 'ExamplePlugin.ExamplePlugin_example_metric2');
+
+        $sequence = new Sequence('ExamplePlugin_archiveCount');
+        $beforeCount = $sequence->getCurrentId();
+
+        $output = $this->runArchivePhpCron(['-vvv' => null]);
+
+        $afterCount = $sequence->getCurrentId();
+
+        $this->assertNotEquals($beforeCount, $afterCount, 'example plugin archiving was not triggered');
+
+        $this->runApiTests('ExamplePlugin.getExampleArchivedMetric', [
+            'idSite' => 'all',
+            'date' => '2007-04-05',
+            'periods' => ['day', 'week'],
+            'testSuffix' => '_singleMetric',
+        ]);
+
+        // test that latest archives for ExamplePlugin are partial
+        $archiveValues = Db::fetchAll("SELECT value FROM " . ArchiveTableCreator::getNumericTable(Date::factory('2007-04-05'))
+            . " WHERE `name` = 'done.ExamplePlugin' ORDER BY ts_archived DESC LIMIT 8");
+        $archiveValues = array_column($archiveValues, 'value');
+        $archiveValues = array_unique($archiveValues);
+        $this->assertEquals([5], $archiveValues);
+    }
+
     public function testArchivePhpCronArchivesFullRanges()
     {
         self::$fixture->getTestEnvironment()->overrideConfig('General', 'enable_browser_archiving_triggering', 0);
@@ -133,9 +202,11 @@ class ArchiveCronTest extends SystemTestCase
 
         $expectedInvalidations = [];
         $invalidationEntries = $this->getInvalidatedArchiveTableEntries();
+
         $invalidationEntries = array_filter($invalidationEntries, function ($entry) {
             return $entry['period'] == 5;
         });
+
         $this->assertEquals($expectedInvalidations, $invalidationEntries);
 
         $this->runApiTests(array(
@@ -143,8 +214,7 @@ class ArchiveCronTest extends SystemTestCase
             array('idSite'     => '1',
                 'date'       => '2012-08-09,2012-08-13',
                 'periods'    => array('range'),
-                'testSuffix' => '_range_archive'
-            )
+                'testSuffix' => '_range_archive')
         );
     }
 
diff --git a/plugins/CoreConsole/tests/System/expected/test_ArchiveCronTest__ExamplePlugin.getExampleArchivedMetric_day.xml b/plugins/CoreConsole/tests/System/expected/test_ArchiveCronTest__ExamplePlugin.getExampleArchivedMetric_day.xml
new file mode 100644
index 0000000000..2e1a657b1f
--- /dev/null
+++ b/plugins/CoreConsole/tests/System/expected/test_ArchiveCronTest__ExamplePlugin.getExampleArchivedMetric_day.xml
@@ -0,0 +1,9 @@
+<?xml version="1.0" encoding="utf-8" ?>
+<results>
+	<result idSite="1">
+		<ExamplePlugin_example_metric>3382</ExamplePlugin_example_metric>
+		<ExamplePlugin_example_metric2>51</ExamplePlugin_example_metric2>
+	</result>
+	<result idSite="2" />
+	<result idSite="3" />
+</results>
\ No newline at end of file
diff --git a/plugins/CoreConsole/tests/System/expected/test_ArchiveCronTest_archive_php_cron_output.txt b/plugins/CoreConsole/tests/System/expected/test_ArchiveCronTest_archive_php_cron_output.txt
index e8b09e0083..81032bb1a7 100644
--- a/plugins/CoreConsole/tests/System/expected/test_ArchiveCronTest_archive_php_cron_output.txt
+++ b/plugins/CoreConsole/tests/System/expected/test_ArchiveCronTest_archive_php_cron_output.txt
@@ -10,90 +10,86 @@ try 'php archive.php --url=http://your.piwik/path'
 -------------------------------------------------------
 
 
-INFO [2020-06-25 09:52:47] 10351  ---------------------------
-INFO [2020-06-25 09:52:47] 10351  INIT
-INFO [2020-06-25 09:52:47] 10351  Running Matomo 4.0.0-b2 as Super User
-INFO [2020-06-25 09:52:47] 10351  ---------------------------
-INFO [2020-06-25 09:52:47] 10351  NOTES
-INFO [2020-06-25 09:52:47] 10351  - If you execute this script at least once per hour (or more often) in a crontab, you may disable 'Browser trigger archiving' in Matomo UI > Settings > General Settings.
-INFO [2020-06-25 09:52:47] 10351    See the doc at: https://matomo.org/docs/setup-auto-archiving/
-INFO [2020-06-25 09:52:47] 10351  - Async process archiving supported, using CliMulti.
-INFO [2020-06-25 09:52:47] 10351  - Reports for today will be processed at most every 900 seconds. You can change this value in Matomo UI > Settings > General Settings.
-INFO [2020-06-25 09:52:47] 10351  ---------------------------
-INFO [2020-06-25 09:52:47] 10351  START
-INFO [2020-06-25 09:52:47] 10351  Starting Matomo reports archiving...
-INFO [2020-06-25 09:52:47] 10351  Checking for queued invalidations...
-INFO [2020-06-25 09:52:47] 10351    Will invalidate archived reports for 2014-03-13 for following websites ids: 3,1
-INFO [2020-06-25 09:52:47] 10351    Will invalidate archived reports for 2014-03-12 for following websites ids: 1,3
-INFO [2020-06-25 09:52:47] 10351    Will invalidate archived reports for 2012-09-30 for following websites ids: 1
-INFO [2020-06-25 09:52:47] 10351    Will invalidate archived reports for 2012-08-19 for following websites ids: 1
-INFO [2020-06-25 09:52:47] 10351    Will invalidate archived reports for 2012-08-15 for following websites ids: 1,3,2
-INFO [2020-06-25 09:52:47] 10351    Will invalidate archived reports for 2012-08-11 for following websites ids: 1
-INFO [2020-06-25 09:52:47] 10351    Will invalidate archived reports for 2012-08-10 for following websites ids: 1
-INFO [2020-06-25 09:52:47] 10351    Will invalidate archived reports for 2012-08-09 for following websites ids: 1
-INFO [2020-06-25 09:52:47] 10351    Segment "browserCode==IE" was created or changed recently and will therefore archive today (for site ID = 1)
-INFO [2020-06-25 09:52:47] 10351    Segment "visitCount<=5;visitorType!=non-existing-type;daysSinceFirstVisit<=50" was created or changed recently and will therefore archive today (for site ID = 1)
-INFO [2020-06-25 09:52:47] 10351    Segment "visitCount<=5;visitorType!=re%2C%3Btest%20is%20encoded;daysSinceFirstVisit<=50" was created or changed recently and will therefore archive today (for site ID = 1)
-INFO [2020-06-25 09:52:47] 10351  Done invalidating
-INFO [2020-06-25 09:52:47] 10351  Start processing archives for site 1.
-INFO [2020-06-25 09:52:50] 10351  Archived website id 1, period = day, date = 2014-03-13, segment = '', 11 visits found. Time elapsed: 1.477s
-INFO [2020-06-25 09:52:50] 10351  Archived website id 1, period = day, date = 2014-03-12, segment = '', 1 visits found. Time elapsed: 1.477s
-INFO [2020-06-25 09:52:50] 10351  Archived website id 1, period = day, date = 2012-09-30, segment = '', 1 visits found. Time elapsed: 1.477s
-INFO [2020-06-25 09:52:51] 10351  Archived website id 1, period = week, date = 2014-03-10, segment = '', 12 visits found. Time elapsed: 1.116s
-INFO [2020-06-25 09:52:51] 10351  Archived website id 1, period = week, date = 2012-09-24, segment = '', 1 visits found. Time elapsed: 1.151s
-INFO [2020-06-25 09:52:51] 10351  Archived website id 1, period = week, date = 2012-08-13, segment = '', 3 visits found. Time elapsed: 1.425s
-INFO [2020-06-25 09:52:54] 10351  Archived website id 1, period = month, date = 2014-03-01, segment = '', 12 visits found. Time elapsed: 1.115s
-INFO [2020-06-25 09:52:54] 10351  Archived website id 1, period = month, date = 2012-09-01, segment = '', 1 visits found. Time elapsed: 1.115s
-INFO [2020-06-25 09:52:54] 10351  Archived website id 1, period = month, date = 2012-08-01, segment = '', 33 visits found. Time elapsed: 1.936s
-INFO [2020-06-25 09:52:55] 10351  Archived website id 1, period = year, date = 2014-01-01, segment = '', 12 visits found. Time elapsed: 0.919s
-INFO [2020-06-25 09:52:55] 10351  Archived website id 1, period = year, date = 2012-01-01, segment = '', 34 visits found. Time elapsed: 0.919s
-INFO [2020-06-25 09:52:59] 10351  Archived website id 1, period = year, date = 2014-01-01, segment = 'browserCode%3D%3DIE', 1 visits found. Time elapsed: 3.088s
-INFO [2020-06-25 09:52:59] 10351  Archived website id 1, period = year, date = 2014-01-01, segment = 'visitCount%3C%3D5%3BvisitorType%21%3Dnon-existing-type%3BdaysSinceFirstVisit%3C%3D50', 12 visits found. Time elapsed: 3.967s
-INFO [2020-06-25 09:52:59] 10351  Archived website id 1, period = year, date = 2014-01-01, segment = 'visitCount%3C%3D5%3BvisitorType%21%3Dre%252C%253Btest%2520is%2520encoded%3BdaysSinceFirstVisit%3C%3D50', 12 visits found. Time elapsed: 3.967s
-INFO [2020-06-25 09:53:01] 10351  Archived website id 1, period = month, date = 2012-09-01, segment = 'browserCode%3D%3DIE', 0 visits found. Time elapsed: 1.113s
-INFO [2020-06-25 09:53:01] 10351  Archived website id 1, period = month, date = 2012-09-01, segment = 'visitCount%3C%3D5%3BvisitorType%21%3Dnon-existing-type%3BdaysSinceFirstVisit%3C%3D50', 1 visits found. Time elapsed: 1.996s
-INFO [2020-06-25 09:53:01] 10351  Archived website id 1, period = month, date = 2012-09-01, segment = 'visitCount%3C%3D5%3BvisitorType%21%3Dre%252C%253Btest%2520is%2520encoded%3BdaysSinceFirstVisit%3C%3D50', 1 visits found. Time elapsed: 1.996s
-INFO [2020-06-25 09:53:04] 10351  Archived website id 1, period = week, date = 2012-08-13, segment = 'browserCode%3D%3DIE', 1 visits found. Time elapsed: 1.807s
-INFO [2020-06-25 09:53:04] 10351  Archived website id 1, period = week, date = 2012-08-13, segment = 'visitCount%3C%3D5%3BvisitorType%21%3Dnon-existing-type%3BdaysSinceFirstVisit%3C%3D50', 3 visits found. Time elapsed: 1.859s
-INFO [2020-06-25 09:53:04] 10351  Archived website id 1, period = week, date = 2012-08-13, segment = 'visitCount%3C%3D5%3BvisitorType%21%3Dre%252C%253Btest%2520is%2520encoded%3BdaysSinceFirstVisit%3C%3D50', 3 visits found. Time elapsed: 2.129s
-INFO [2020-06-25 09:53:05] 10351  Archived website id 1, period = day, date = 2012-08-11, segment = 'browserCode%3D%3DIE', 1 visits found. Time elapsed: 1.130s
-INFO [2020-06-25 09:53:05] 10351  Archived website id 1, period = day, date = 2012-08-11, segment = 'visitCount%3C%3D5%3BvisitorType%21%3Dnon-existing-type%3BdaysSinceFirstVisit%3C%3D50', 11 visits found. Time elapsed: 1.130s
-INFO [2020-06-25 09:53:05] 10351  Archived website id 1, period = day, date = 2012-08-11, segment = 'visitCount%3C%3D5%3BvisitorType%21%3Dre%252C%253Btest%2520is%2520encoded%3BdaysSinceFirstVisit%3C%3D50', 11 visits found. Time elapsed: 1.403s
-INFO [2020-06-25 09:53:06] 10351  Archived website id 1, period = day, date = 2012-08-10, segment = 'browserCode%3D%3DIE', 3 visits found. Time elapsed: 1.114s
-INFO [2020-06-25 09:53:06] 10351  Archived website id 1, period = day, date = 2012-08-10, segment = 'visitCount%3C%3D5%3BvisitorType%21%3Dnon-existing-type%3BdaysSinceFirstVisit%3C%3D50', 8 visits found. Time elapsed: 1.426s
-INFO [2020-06-25 09:53:06] 10351  Archived website id 1, period = day, date = 2012-08-10, segment = 'visitCount%3C%3D5%3BvisitorType%21%3Dre%252C%253Btest%2520is%2520encoded%3BdaysSinceFirstVisit%3C%3D50', 8 visits found. Time elapsed: 1.426s
-INFO [2020-06-25 09:53:08] 10351  Archived website id 1, period = day, date = 2012-08-09, segment = 'browserCode%3D%3DIE', 1 visits found. Time elapsed: 1.099s
-INFO [2020-06-25 09:53:08] 10351  Archived website id 1, period = day, date = 2012-08-09, segment = 'visitCount%3C%3D5%3BvisitorType%21%3Dnon-existing-type%3BdaysSinceFirstVisit%3C%3D50', 11 visits found. Time elapsed: 1.417s
-INFO [2020-06-25 09:53:08] 10351  Archived website id 1, period = day, date = 2012-08-09, segment = 'visitCount%3C%3D5%3BvisitorType%21%3Dre%252C%253Btest%2520is%2520encoded%3BdaysSinceFirstVisit%3C%3D50', 11 visits found. Time elapsed: 1.417s
-INFO [2020-06-25 09:53:10] 10351  Archived website id 1, period = week, date = 2012-08-06, segment = 'browserCode%3D%3DIE', 5 visits found. Time elapsed: 1.442s
-INFO [2020-06-25 09:53:10] 10351  Archived website id 1, period = week, date = 2012-08-06, segment = 'visitCount%3C%3D5%3BvisitorType%21%3Dnon-existing-type%3BdaysSinceFirstVisit%3C%3D50', 30 visits found. Time elapsed: 1.442s
-INFO [2020-06-25 09:53:10] 10351  Archived website id 1, period = week, date = 2012-08-06, segment = 'visitCount%3C%3D5%3BvisitorType%21%3Dre%252C%253Btest%2520is%2520encoded%3BdaysSinceFirstVisit%3C%3D50', 30 visits found. Time elapsed: 1.714s
-INFO [2020-06-25 09:53:11] 10351  Archived website id 1, period = month, date = 2012-08-01, segment = 'browserCode%3D%3DIE', 6 visits found. Time elapsed: 1.403s
-INFO [2020-06-25 09:53:11] 10351  Archived website id 1, period = month, date = 2012-08-01, segment = 'visitCount%3C%3D5%3BvisitorType%21%3Dnon-existing-type%3BdaysSinceFirstVisit%3C%3D50', 33 visits found. Time elapsed: 1.403s
-INFO [2020-06-25 09:53:11] 10351  Archived website id 1, period = month, date = 2012-08-01, segment = 'visitCount%3C%3D5%3BvisitorType%21%3Dre%252C%253Btest%2520is%2520encoded%3BdaysSinceFirstVisit%3C%3D50', 33 visits found. Time elapsed: 1.676s
-INFO [2020-06-25 09:53:13] 10351  Archived website id 1, period = year, date = 2012-01-01, segment = 'browserCode%3D%3DIE', 6 visits found. Time elapsed: 1.421s
-INFO [2020-06-25 09:53:13] 10351  Archived website id 1, period = year, date = 2012-01-01, segment = 'visitCount%3C%3D5%3BvisitorType%21%3Dnon-existing-type%3BdaysSinceFirstVisit%3C%3D50', 34 visits found. Time elapsed: 1.728s
-INFO [2020-06-25 09:53:13] 10351  Archived website id 1, period = year, date = 2012-01-01, segment = 'visitCount%3C%3D5%3BvisitorType%21%3Dre%252C%253Btest%2520is%2520encoded%3BdaysSinceFirstVisit%3C%3D50', 34 visits found. Time elapsed: 1.728s
-INFO [2020-06-25 09:53:13] 10351  Finished archiving for site 1, 38 API requests, Time elapsed: 25.964s [1 / 3 done]
-INFO [2020-06-25 09:53:13] 10351  Start processing archives for site 2.
-INFO [2020-06-25 09:53:14] 10351  Archived website id 2, period = day, date = 2012-08-15, segment = '', 1 visits found. Time elapsed: 0.556s
-INFO [2020-06-25 09:53:14] 10351  Archived website id 2, period = week, date = 2012-08-13, segment = '', 1 visits found. Time elapsed: 0.555s
-INFO [2020-06-25 09:53:15] 10351  Archived website id 2, period = month, date = 2012-08-01, segment = '', 1 visits found. Time elapsed: 0.555s
-INFO [2020-06-25 09:53:16] 10351  Archived website id 2, period = year, date = 2012-01-01, segment = '', 1 visits found. Time elapsed: 0.554s
-INFO [2020-06-25 09:53:16] 10351  Finished archiving for site 2, 4 API requests, Time elapsed: 2.374s [2 / 3 done]
-INFO [2020-06-25 09:53:16] 10351  Start processing archives for site 3.
-INFO [2020-06-25 09:53:17] 10351  Archived website id 3, period = day, date = 2014-03-13, segment = '', 11 visits found. Time elapsed: 1.116s
-INFO [2020-06-25 09:53:17] 10351  Archived website id 3, period = day, date = 2014-03-12, segment = '', 1 visits found. Time elapsed: 1.116s
-INFO [2020-06-25 09:53:17] 10351  Archived website id 3, period = day, date = 2012-08-15, segment = '', 3 visits found. Time elapsed: 1.116s
-INFO [2020-06-25 09:53:18] 10351  Archived website id 3, period = week, date = 2014-03-10, segment = '', 12 visits found. Time elapsed: 0.930s
-INFO [2020-06-25 09:53:18] 10351  Archived website id 3, period = week, date = 2012-08-13, segment = '', 3 visits found. Time elapsed: 0.930s
-INFO [2020-06-25 09:53:19] 10351  Archived website id 3, period = month, date = 2014-03-01, segment = '', 12 visits found. Time elapsed: 0.932s
-INFO [2020-06-25 09:53:19] 10351  Archived website id 3, period = month, date = 2012-08-01, segment = '', 3 visits found. Time elapsed: 0.932s
-INFO [2020-06-25 09:53:20] 10351  Archived website id 3, period = year, date = 2014-01-01, segment = '', 12 visits found. Time elapsed: 0.923s
-INFO [2020-06-25 09:53:20] 10351  Archived website id 3, period = year, date = 2012-01-01, segment = '', 3 visits found. Time elapsed: 0.923s
-INFO [2020-06-25 09:53:20] 10351  Finished archiving for site 3, 9 API requests, Time elapsed: 4.088s [3 / 3 done]
-INFO [2020-06-25 09:53:20] 10351  ---------------------------
-INFO [2020-06-25 09:53:20] 10351  SCHEDULED TASKS
-INFO [2020-06-25 09:53:20] 10351  Starting Scheduled tasks...
-INFO [2020-06-25 09:53:20] 10351  done
-INFO [2020-06-25 09:53:20] 10351  ---------------------------
\ No newline at end of file
+INFO [2020-07-06 21:59:34] 10307  ---------------------------
+INFO [2020-07-06 21:59:34] 10307  INIT
+INFO [2020-07-06 21:59:34] 10307  Running Matomo 4.0.0-b2 as Super User
+INFO [2020-07-06 21:59:34] 10307  ---------------------------
+INFO [2020-07-06 21:59:34] 10307  NOTES
+INFO [2020-07-06 21:59:34] 10307  - If you execute this script at least once per hour (or more often) in a crontab, you may disable 'Browser trigger archiving' in Matomo UI > Settings > General Settings.
+INFO [2020-07-06 21:59:34] 10307    See the doc at: https://matomo.org/docs/setup-auto-archiving/
+INFO [2020-07-06 21:59:34] 10307  - Async process archiving supported, using CliMulti.
+INFO [2020-07-06 21:59:34] 10307  - Reports for today will be processed at most every 900 seconds. You can change this value in Matomo UI > Settings > General Settings.
+INFO [2020-07-06 21:59:34] 10307  ---------------------------
+INFO [2020-07-06 21:59:34] 10307  START
+INFO [2020-07-06 21:59:34] 10307  Starting Matomo reports archiving...
+INFO [2020-07-06 21:59:34] 10307  Checking for queued invalidations...
+INFO [2020-07-06 21:59:34] 10307    Will invalidate archived reports for 2014-03-13 for following websites ids: 1,3
+INFO [2020-07-06 21:59:34] 10307    Will invalidate archived reports for 2014-03-12 for following websites ids: 3,1
+INFO [2020-07-06 21:59:34] 10307    Will invalidate archived reports for 2012-09-30 for following websites ids: 1
+INFO [2020-07-06 21:59:34] 10307    Will invalidate archived reports for 2012-08-19 for following websites ids: 1
+INFO [2020-07-06 21:59:34] 10307    Will invalidate archived reports for 2012-08-15 for following websites ids: 2,1,3
+INFO [2020-07-06 21:59:34] 10307    Will invalidate archived reports for 2012-08-11 for following websites ids: 1
+INFO [2020-07-06 21:59:34] 10307    Will invalidate archived reports for 2012-08-10 for following websites ids: 1
+INFO [2020-07-06 21:59:34] 10307    Will invalidate archived reports for 2012-08-09 for following websites ids: 1
+INFO [2020-07-06 21:59:34] 10307    Segment "browserCode==IE" was created or changed recently and will therefore archive today (for site ID = 1)
+INFO [2020-07-06 21:59:34] 10307    Segment "visitCount<=5;visitorType!=non-existing-type;daysSinceFirstVisit<=50" was created or changed recently and will therefore archive today (for site ID = 1)
+INFO [2020-07-06 21:59:34] 10307    Segment "visitCount<=5;visitorType!=re%2C%3Btest%20is%20encoded;daysSinceFirstVisit<=50" was created or changed recently and will therefore archive today (for site ID = 1)
+INFO [2020-07-06 21:59:34] 10307  Done invalidating
+INFO [2020-07-06 21:59:34] 10307  Start processing archives for site 1.
+INFO [2020-07-06 21:59:37] 10307  Archived website id 1, period = day, date = 2014-03-13, segment = '', 11 visits found. Time elapsed: 1.124s
+INFO [2020-07-06 21:59:37] 10307  Archived website id 1, period = day, date = 2014-03-12, segment = '', 1 visits found. Time elapsed: 1.124s
+INFO [2020-07-06 21:59:37] 10307  Archived website id 1, period = day, date = 2012-09-30, segment = '', 1 visits found. Time elapsed: 1.124s
+INFO [2020-07-06 21:59:38] 10307  Archived website id 1, period = week, date = 2014-03-10, segment = '', 12 visits found. Time elapsed: 1.761s
+INFO [2020-07-06 21:59:38] 10307  Archived website id 1, period = week, date = 2012-09-24, segment = '', 1 visits found. Time elapsed: 1.761s
+INFO [2020-07-06 21:59:38] 10307  Archived website id 1, period = day, date = 2012-08-19, segment = '', 1 visits found. Time elapsed: 1.761s
+INFO [2020-07-06 21:59:40] 10307  Archived website id 1, period = month, date = 2014-03-01, segment = '', 12 visits found. Time elapsed: 1.743s
+INFO [2020-07-06 21:59:40] 10307  Archived website id 1, period = month, date = 2012-09-01, segment = '', 1 visits found. Time elapsed: 1.743s
+INFO [2020-07-06 21:59:40] 10307  Archived website id 1, period = day, date = 2012-08-15, segment = '', 2 visits found. Time elapsed: 1.743s
+INFO [2020-07-06 21:59:43] 10307  Archived website id 1, period = year, date = 2014-01-01, segment = '', 12 visits found. Time elapsed: 2.510s
+INFO [2020-07-06 21:59:43] 10307  Archived website id 1, period = week, date = 2012-08-13, segment = '', 3 visits found. Time elapsed: 2.510s
+INFO [2020-07-06 21:59:43] 10307  Archived website id 1, period = day, date = 2012-08-11, segment = '', 11 visits found. Time elapsed: 2.510s
+INFO [2020-07-06 21:59:47] 10307  Archived website id 1, period = year, date = 2014-01-01, segment = 'browserCode%3D%3DIE', 1 visits found. Time elapsed: 3.015s
+INFO [2020-07-06 21:59:47] 10307  Archived website id 1, period = year, date = 2012-01-01, segment = 'browserCode%3D%3DIE', 6 visits found. Time elapsed: 4.118s
+INFO [2020-07-06 21:59:47] 10307  Archived website id 1, period = day, date = 2007-04-05, segment = '', plugin = ExamplePlugin, 1 visits found. Time elapsed: 4.118s
+INFO [2020-07-06 21:59:53] 10307  Archived website id 1, period = year, date = 2014-01-01, segment = 'visitCount%3C%3D5%3BvisitorType%21%3Dnon-existing-type%3BdaysSinceFirstVisit%3C%3D50', 12 visits found. Time elapsed: 4.601s
+INFO [2020-07-06 21:59:53] 10307  Archived website id 1, period = year, date = 2012-01-01, segment = 'visitCount%3C%3D5%3BvisitorType%21%3Dnon-existing-type%3BdaysSinceFirstVisit%3C%3D50', 34 visits found. Time elapsed: 5.707s
+INFO [2020-07-06 21:59:53] 10307  Archived website id 1, period = week, date = 2007-04-02, segment = '', plugin = ExamplePlugin, 1 visits found. Time elapsed: 5.707s
+INFO [2020-07-06 21:59:59] 10307  Archived website id 1, period = year, date = 2014-01-01, segment = 'visitCount%3C%3D5%3BvisitorType%21%3Dre%252C%253Btest%2520is%2520encoded%3BdaysSinceFirstVisit%3C%3D50', 12 visits found. Time elapsed: 3.018s
+INFO [2020-07-06 21:59:59] 10307  Archived website id 1, period = year, date = 2012-01-01, segment = 'visitCount%3C%3D5%3BvisitorType%21%3Dre%252C%253Btest%2520is%2520encoded%3BdaysSinceFirstVisit%3C%3D50', 34 visits found. Time elapsed: 5.246s
+INFO [2020-07-06 21:59:59] 10307  Archived website id 1, period = month, date = 2007-04-01, segment = '', plugin = ExamplePlugin, 1 visits found. Time elapsed: 5.246s
+INFO [2020-07-06 22:00:00] 10307  Archived website id 1, period = day, date = 2012-08-10, segment = '', 8 visits found. Time elapsed: 0.632s
+INFO [2020-07-06 22:00:00] 10307  Archived website id 1, period = day, date = 2012-08-09, segment = '', 11 visits found. Time elapsed: 0.916s
+INFO [2020-07-06 22:00:01] 10307  Archived website id 1, period = week, date = 2012-08-06, segment = '', 30 visits found. Time elapsed: 0.827s
+INFO [2020-07-06 22:00:02] 10307  Archived website id 1, period = month, date = 2012-08-01, segment = '', 33 visits found. Time elapsed: 1.104s
+INFO [2020-07-06 22:00:03] 10307  Archived website id 1, period = year, date = 2012-01-01, segment = '', 34 visits found. Time elapsed: 1.102s
+INFO [2020-07-06 22:00:03] 10307  Archived website id 1, period = year, date = 2007-01-01, segment = '', plugin = ExamplePlugin, 1 visits found. Time elapsed: 0.564s
+INFO [2020-07-06 22:00:03] 10307  Finished archiving for site 1, 27 API requests, Time elapsed: 28.981s [1 / 3 done]
+INFO [2020-07-06 22:00:04] 10307  Start processing archives for site 2.
+INFO [2020-07-06 22:00:04] 10307  Archived website id 2, period = day, date = 2012-08-15, segment = '', 1 visits found. Time elapsed: 0.549s
+INFO [2020-07-06 22:00:05] 10307  Archived website id 2, period = week, date = 2012-08-13, segment = '', 1 visits found. Time elapsed: 0.826s
+INFO [2020-07-06 22:00:06] 10307  Archived website id 2, period = month, date = 2012-08-01, segment = '', 1 visits found. Time elapsed: 0.825s
+INFO [2020-07-06 22:00:07] 10307  Archived website id 2, period = year, date = 2012-01-01, segment = '', 1 visits found. Time elapsed: 1.110s
+INFO [2020-07-06 22:00:07] 10307  Finished archiving for site 2, 4 API requests, Time elapsed: 3.469s [2 / 3 done]
+INFO [2020-07-06 22:00:07] 10307  Start processing archives for site 3.
+INFO [2020-07-06 22:00:08] 10307  Archived website id 3, period = day, date = 2014-03-13, segment = '', 11 visits found. Time elapsed: 0.834s
+INFO [2020-07-06 22:00:08] 10307  Archived website id 3, period = day, date = 2014-03-12, segment = '', 1 visits found. Time elapsed: 0.834s
+INFO [2020-07-06 22:00:08] 10307  Archived website id 3, period = day, date = 2012-08-15, segment = '', 3 visits found. Time elapsed: 1.115s
+INFO [2020-07-06 22:00:09] 10307  Archived website id 3, period = week, date = 2014-03-10, segment = '', 12 visits found. Time elapsed: 1.257s
+INFO [2020-07-06 22:00:09] 10307  Archived website id 3, period = week, date = 2012-08-13, segment = '', 3 visits found. Time elapsed: 1.257s
+INFO [2020-07-06 22:00:11] 10307  Archived website id 3, period = month, date = 2014-03-01, segment = '', 12 visits found. Time elapsed: 1.266s
+INFO [2020-07-06 22:00:11] 10307  Archived website id 3, period = month, date = 2012-08-01, segment = '', 3 visits found. Time elapsed: 1.818s
+INFO [2020-07-06 22:00:13] 10307  Archived website id 3, period = year, date = 2014-01-01, segment = '', 12 visits found. Time elapsed: 1.899s
+INFO [2020-07-06 22:00:13] 10307  Archived website id 3, period = year, date = 2012-01-01, segment = '', 3 visits found. Time elapsed: 1.920s
+INFO [2020-07-06 22:00:13] 10307  Finished archiving for site 3, 9 API requests, Time elapsed: 6.303s [3 / 3 done]
+INFO [2020-07-06 22:00:13] 10307  Done archiving!
+INFO [2020-07-06 22:00:13] 10307  ---------------------------
+INFO [2020-07-06 22:00:13] 10307  SUMMARY
+INFO [2020-07-06 22:00:13] 10307  Processed 40 archives.
+INFO [2020-07-06 22:00:13] 10307  Total API requests: 40
+INFO [2020-07-06 22:00:13] 10307  done: 40 req, 39477 ms, no error
+INFO [2020-07-06 22:00:13] 10307  Time elapsed: 39.477s
+INFO [2020-07-06 22:00:13] 10307  ---------------------------
+INFO [2020-07-06 22:00:13] 10307  SCHEDULED TASKS
+INFO [2020-07-06 22:00:13] 10307  Starting Scheduled tasks...
+INFO [2020-07-06 22:00:13] 10307  done
+INFO [2020-07-06 22:00:13] 10307  ---------------------------
\ No newline at end of file
diff --git a/plugins/CoreConsole/tests/System/expected/test_ArchiveCronTest_examplePluginNoMetricsBecauseNoOtherPluginsArchived_noOptions__Actions.get_day.xml b/plugins/CoreConsole/tests/System/expected/test_ArchiveCronTest_examplePluginNoMetricsBecauseNoOtherPluginsArchived_noOptions__Actions.get_day.xml
new file mode 100644
index 0000000000..dd52dc4ff4
--- /dev/null
+++ b/plugins/CoreConsole/tests/System/expected/test_ArchiveCronTest_examplePluginNoMetricsBecauseNoOtherPluginsArchived_noOptions__Actions.get_day.xml
@@ -0,0 +1,6 @@
+<?xml version="1.0" encoding="utf-8" ?>
+<results>
+	<result idSite="1" />
+	<result idSite="2" />
+	<result idSite="3" />
+</results>
\ No newline at end of file
diff --git a/plugins/CoreConsole/tests/System/expected/test_ArchiveCronTest_examplePluginNoMetricsBecauseNoOtherPluginsArchived_noOptions__Actions.get_week.xml b/plugins/CoreConsole/tests/System/expected/test_ArchiveCronTest_examplePluginNoMetricsBecauseNoOtherPluginsArchived_noOptions__Actions.get_week.xml
new file mode 100644
index 0000000000..dd52dc4ff4
--- /dev/null
+++ b/plugins/CoreConsole/tests/System/expected/test_ArchiveCronTest_examplePluginNoMetricsBecauseNoOtherPluginsArchived_noOptions__Actions.get_week.xml
@@ -0,0 +1,6 @@
+<?xml version="1.0" encoding="utf-8" ?>
+<results>
+	<result idSite="1" />
+	<result idSite="2" />
+	<result idSite="3" />
+</results>
\ No newline at end of file
diff --git a/plugins/CoreConsole/tests/System/expected/test_ArchiveCronTest_noOptions__ExamplePlugin.getExampleArchivedMetric_day.xml b/plugins/CoreConsole/tests/System/expected/test_ArchiveCronTest_noOptions__ExamplePlugin.getExampleArchivedMetric_day.xml
new file mode 100644
index 0000000000..bbba10abb2
--- /dev/null
+++ b/plugins/CoreConsole/tests/System/expected/test_ArchiveCronTest_noOptions__ExamplePlugin.getExampleArchivedMetric_day.xml
@@ -0,0 +1,9 @@
+<?xml version="1.0" encoding="utf-8" ?>
+<results>
+	<result idSite="1">
+		<ExamplePlugin_example_metric>3382</ExamplePlugin_example_metric>
+		<ExamplePlugin_example_metric2>60</ExamplePlugin_example_metric2>
+	</result>
+	<result idSite="2" />
+	<result idSite="3" />
+</results>
\ No newline at end of file
diff --git a/plugins/CoreConsole/tests/System/expected/test_ArchiveCronTest_noOptions__ExamplePlugin.getExampleArchivedMetric_week.xml b/plugins/CoreConsole/tests/System/expected/test_ArchiveCronTest_noOptions__ExamplePlugin.getExampleArchivedMetric_week.xml
new file mode 100644
index 0000000000..bbba10abb2
--- /dev/null
+++ b/plugins/CoreConsole/tests/System/expected/test_ArchiveCronTest_noOptions__ExamplePlugin.getExampleArchivedMetric_week.xml
@@ -0,0 +1,9 @@
+<?xml version="1.0" encoding="utf-8" ?>
+<results>
+	<result idSite="1">
+		<ExamplePlugin_example_metric>3382</ExamplePlugin_example_metric>
+		<ExamplePlugin_example_metric2>60</ExamplePlugin_example_metric2>
+	</result>
+	<result idSite="2" />
+	<result idSite="3" />
+</results>
\ No newline at end of file
diff --git a/plugins/CoreConsole/tests/System/expected/test_ArchiveCronTest_singleMetric__ExamplePlugin.getExampleArchivedMetric_day.xml b/plugins/CoreConsole/tests/System/expected/test_ArchiveCronTest_singleMetric__ExamplePlugin.getExampleArchivedMetric_day.xml
new file mode 100644
index 0000000000..4214cca45d
--- /dev/null
+++ b/plugins/CoreConsole/tests/System/expected/test_ArchiveCronTest_singleMetric__ExamplePlugin.getExampleArchivedMetric_day.xml
@@ -0,0 +1,9 @@
+<?xml version="1.0" encoding="utf-8" ?>
+<results>
+	<result idSite="1">
+		<ExamplePlugin_example_metric>3382</ExamplePlugin_example_metric>
+		<ExamplePlugin_example_metric2>90</ExamplePlugin_example_metric2>
+	</result>
+	<result idSite="2" />
+	<result idSite="3" />
+</results>
\ No newline at end of file
diff --git a/plugins/CoreConsole/tests/System/expected/test_ArchiveCronTest_singleMetric__ExamplePlugin.getExampleArchivedMetric_week.xml b/plugins/CoreConsole/tests/System/expected/test_ArchiveCronTest_singleMetric__ExamplePlugin.getExampleArchivedMetric_week.xml
new file mode 100644
index 0000000000..4214cca45d
--- /dev/null
+++ b/plugins/CoreConsole/tests/System/expected/test_ArchiveCronTest_singleMetric__ExamplePlugin.getExampleArchivedMetric_week.xml
@@ -0,0 +1,9 @@
+<?xml version="1.0" encoding="utf-8" ?>
+<results>
+	<result idSite="1">
+		<ExamplePlugin_example_metric>3382</ExamplePlugin_example_metric>
+		<ExamplePlugin_example_metric2>90</ExamplePlugin_example_metric2>
+	</result>
+	<result idSite="2" />
+	<result idSite="3" />
+</results>
\ No newline at end of file
diff --git a/plugins/CustomVariables b/plugins/CustomVariables
-Subproject 1aee5523fb807e0d8a8cc5290e81133ca7dd563
+Subproject 6f2b1419fdf11f4ce845658e4d5d2634b61dcda
diff --git a/plugins/Diagnostics/tests/Integration/Commands/AnalyzeArchiveTableTest.php b/plugins/Diagnostics/tests/Integration/Commands/AnalyzeArchiveTableTest.php
index 9e3fbd224a..b71601e41e 100644
--- a/plugins/Diagnostics/tests/Integration/Commands/AnalyzeArchiveTableTest.php
+++ b/plugins/Diagnostics/tests/Integration/Commands/AnalyzeArchiveTableTest.php
@@ -41,9 +41,9 @@ Statistics for the archive_numeric_2010_03 and archive_blob_2010_03 tables:
 +-------------------------------------------+------------+---------------+-------------+---------+-----------+----------------+-------------+-------------+
 | Group                                     | # Archives | # Invalidated | # Temporary | # Error | # Segment | # Numeric Rows | # Blob Rows | # Blob Data |
 +-------------------------------------------+------------+---------------+-------------+---------+-----------+----------------+-------------+-------------+
-| week[2010-03-01 - 2010-03-07] idSite = 1  | 7          | 0             | 0           | 0       | 6         | 73             | 97          | %d       |
-| month[2010-03-01 - 2010-03-31] idSite = 1 | 7          | 0             | 0           | 0       | 6         | 73             | 97          | %d       |
-| day[2010-03-06 - 2010-03-06] idSite = 1   | 7          | 0             | 0           | 0       | 6         | 73             | 73          | %d       |
+| week[2010-03-01 - 2010-03-07] idSite = 1  | 7          | 0             | 0           | 0       | 6         | 75             | 97          | %d       |
+| month[2010-03-01 - 2010-03-31] idSite = 1 | 7          | 0             | 0           | 0       | 6         | 75             | 97          | %d       |
+| day[2010-03-06 - 2010-03-06] idSite = 1   | 7          | 0             | 0           | 0       | 6         | 75             | 73          | %d       |
 +-------------------------------------------+------------+---------------+-------------+---------+-----------+----------------+-------------+-------------+
 
 Total # Archives: 21
diff --git a/plugins/ExamplePlugin/API.php b/plugins/ExamplePlugin/API.php
index 44d67cd80c..88f6249ecb 100644
--- a/plugins/ExamplePlugin/API.php
+++ b/plugins/ExamplePlugin/API.php
@@ -8,6 +8,7 @@
 
 namespace Piwik\Plugins\ExamplePlugin;
 
+use Piwik\Archive;
 use Piwik\DataTable;
 use Piwik\DataTable\Row;
 
@@ -54,4 +55,19 @@ class API extends \Piwik\Plugin\API
 
         return $table;
     }
+
+    /**
+     * Returns the example metric we archive in Archiver.php.
+     * @param int    $idSite
+     * @param string $period
+     * @param string $date
+     * @param bool|string $segment
+     * @return DataTable
+     */
+    public function getExampleArchivedMetric($idSite, $period, $date, $segment = false)
+    {
+        $archive = Archive::build($idSite, $period, $date, $segment);
+        $dataTable = $archive->getDataTableFromNumeric([Archiver::EXAMPLEPLUGIN_METRIC_NAME, Archiver::EXAMPLEPLUGIN_CONST_METRIC_NAME]);
+        return $dataTable;
+    }
 }
diff --git a/plugins/ExamplePlugin/Archiver.php b/plugins/ExamplePlugin/Archiver.php
index f507c6087a..13455ce6eb 100644
--- a/plugins/ExamplePlugin/Archiver.php
+++ b/plugins/ExamplePlugin/Archiver.php
@@ -8,6 +8,13 @@
 
 namespace Piwik\Plugins\ExamplePlugin;
 
+use Piwik\ArchiveProcessor;
+use Piwik\Container\StaticContainer;
+use Piwik\Date;
+use Piwik\Option;
+use Piwik\Sequence;
+use Psr\Log\LoggerInterface;
+
 /**
  * Class Archiver
  *
@@ -31,6 +38,27 @@ class Archiver extends \Piwik\Plugin\Archiver
      * This is only an example record name, so feel free to change it to suit your needs.
      */
     const EXAMPLEPLUGIN_ARCHIVE_RECORD = "ExamplePlugin_archive_record";
+    const EXAMPLEPLUGIN_METRIC_NAME = 'ExamplePlugin_example_metric';
+    const EXAMPLEPLUGIN_CONST_METRIC_NAME = 'ExamplePlugin_example_metric2';
+
+    private $daysFrom = '2016-07-08';
+
+    /**
+     * @var string
+     */
+    private $requestedReport = null;
+
+    public function __construct(ArchiveProcessor $processor)
+    {
+        parent::__construct($processor);
+
+        $this->requestedReport = $processor->getParams()->getArchiveOnlyReport();
+        if ($this->requestedReport) {
+            $processor->getParams()->setIsPartialArchive(true);
+        }
+
+        $this->createSequence();
+    }
 
     public function aggregateDayReport()
     {
@@ -45,6 +73,25 @@ class Archiver extends \Piwik\Plugin\Archiver
          * $visitorReport = $visitorMetrics->getSerialized();
          * $this->getProcessor()->insertBlobRecord(self::EXAMPLEPLUGIN_ARCHIVE_RECORD, $visitorReport);
          */
+
+        if ($this->isArchiving(self::EXAMPLEPLUGIN_METRIC_NAME)) {
+            // insert a test numeric metric that is the difference in days between the day we're archiving and
+            // $this->daysFrom.
+            $daysFrom = Date::factory($this->daysFrom);
+            $date = $this->getProcessor()->getParams()->getPeriod()->getDateStart();
+
+            $differenceInSeconds = $daysFrom->getTimestamp() - $date->getTimestamp();
+            $differenceInDays = round($differenceInSeconds / 86400);
+
+            $this->getProcessor()->insertNumericRecord(self::EXAMPLEPLUGIN_METRIC_NAME, $differenceInDays);
+        }
+
+        if ($this->isArchiving(self::EXAMPLEPLUGIN_CONST_METRIC_NAME)) {
+            $archiveCount = $this->incrementArchiveCount();
+            $archiveCount = 50 + $archiveCount;
+            $archiveCount += 5 - ($archiveCount % 5); // round up to nearest 5 multiple to avoid random test failures
+            $this->getProcessor()->insertNumericRecord(self::EXAMPLEPLUGIN_CONST_METRIC_NAME, $archiveCount);
+        }
     }
 
     public function aggregateMultipleReports()
@@ -57,5 +104,41 @@ class Archiver extends \Piwik\Plugin\Archiver
          *
          * $this->getProcessor()->aggregateDataTableRecords(self::EXAMPLEPLUGIN_ARCHIVE_RECORD);
          */
+
+        $reports = [];
+        if ($this->isArchiving(self::EXAMPLEPLUGIN_METRIC_NAME)) {
+            $reports[] = self::EXAMPLEPLUGIN_METRIC_NAME;
+        }
+        if ($this->isArchiving(self::EXAMPLEPLUGIN_CONST_METRIC_NAME)) {
+            $reports[] = self::EXAMPLEPLUGIN_CONST_METRIC_NAME;
+        }
+        $this->getProcessor()->aggregateNumericMetrics($reports);
+    }
+
+    private function incrementArchiveCount()
+    {
+        $sequence = new Sequence('ExamplePlugin_archiveCount');
+        $result = $sequence->getNextId();
+        return $result;
+    }
+
+    private function isArchiving(string $reportName)
+    {
+        return empty($this->requestedReport) || $this->requestedReport == $reportName;
+    }
+
+    private function createSequence()
+    {
+        $sequence = new Sequence('ExamplePlugin_archiveCount');
+        if (!$sequence->exists()) {
+            for ($i = 0; $i < 100; ++$i) {
+                try {
+                    $sequence->create();
+                    break;
+                } catch (\Exception $ex) {
+                    // ignore
+                }
+            }
+        }
     }
 }
diff --git a/plugins/ExamplePlugin/ExamplePlugin.php b/plugins/ExamplePlugin/ExamplePlugin.php
index f974975c42..8b737f5784 100644
--- a/plugins/ExamplePlugin/ExamplePlugin.php
+++ b/plugins/ExamplePlugin/ExamplePlugin.php
@@ -10,4 +10,18 @@ namespace Piwik\Plugins\ExamplePlugin;
 
 class ExamplePlugin extends \Piwik\Plugin
 {
+    public function registerEvents()
+    {
+        return [
+            'CronArchive.getArchivingAPIMethodForPlugin' => 'getArchivingAPIMethodForPlugin',
+        ];
+    }
+
+    // support archiving just this plugin via core:archive
+    public function getArchivingAPIMethodForPlugin(&$method, $plugin)
+    {
+        if ($plugin == 'ExamplePlugin') {
+            $method = 'ExamplePlugin.getExampleArchivedMetric';
+        }
+    }
 }
diff --git a/plugins/GeoIp2/LocationProvider/GeoIp2/ServerModule.php b/plugins/GeoIp2/LocationProvider/GeoIp2/ServerModule.php
index 5ebe6114ce..f1baf19552 100644
--- a/plugins/GeoIp2/LocationProvider/GeoIp2/ServerModule.php
+++ b/plugins/GeoIp2/LocationProvider/GeoIp2/ServerModule.php
@@ -16,7 +16,9 @@ use Piwik\Piwik;
 use Piwik\Plugins\GeoIp2\LocationProvider\GeoIp2;
 use Piwik\Plugins\UserCountry\LocationProvider;
 use Piwik\Plugins\GeoIp2\SystemSettings;
+use Piwik\SettingsServer;
 use Piwik\Url;
+use Piwik\View;
 
 /**
  * A LocationProvider that uses an GeoIP 2 module installed in an HTTP Server.
@@ -250,7 +252,12 @@ class ServerModule extends GeoIp2
         $configUrl = Url::getCurrentQueryStringWithParametersModified(array(
             'module' => 'CoreAdminHome', 'action' => 'generalSettings'
         ));
-        $extraMessage .= '<br />'.Piwik::translate('GeoIp2_GeoIPVariablesConfigurationHere', ['<a href="'.$configUrl.'">', '</a>']);
+        if (!SettingsServer::isTrackerApiRequest()) {
+            // can't render in tracking mode as there is no theme
+            $view = new View('@GeoIp2/serverModule');
+            $view->configUrl = $configUrl;
+            $extraMessage .= $view->render();
+        }
 
         return array('id'            => self::ID,
             'title'         => $title,
diff --git a/plugins/GeoIp2/templates/serverModule.twig b/plugins/GeoIp2/templates/serverModule.twig
new file mode 100644
index 0000000000..66df4331fb
--- /dev/null
+++ b/plugins/GeoIp2/templates/serverModule.twig
@@ -0,0 +1 @@
+<br />{{ 'GeoIp2_GeoIPVariablesConfigurationHere'|translate('<a href="' ~ configUrl|e('html_attr') ~ '">', '</a>')|raw }}
\ No newline at end of file
diff --git a/plugins/Widgetize/Controller.php b/plugins/Widgetize/Controller.php
index e53c27589f..6a8376b16e 100644
--- a/plugins/Widgetize/Controller.php
+++ b/plugins/Widgetize/Controller.php
@@ -8,9 +8,13 @@
  */
 namespace Piwik\Plugins\Widgetize;
 
+use Piwik\Access;
 use Piwik\Common;
+use Piwik\Container\StaticContainer;
 use Piwik\FrontController;
 use Piwik\Piwik;
+use Piwik\Session\SessionInitializer;
+use Piwik\Url;
 use Piwik\View;
 
 /**
@@ -27,6 +31,12 @@ class Controller extends \Piwik\Plugin\Controller
 
     public function iframe()
     {
+        $token_auth = Common::getRequestVar('token_auth', '', 'string');
+
+        if (!empty($token_auth) && Access::getInstance()->isUserHasSomeAdminAccess() && !defined('PIWIK_TEST_MODE')) {
+            throw new \Exception(Piwik::translate('Widgetize_ViewAccessRequired'));
+        }
+
         $this->init();
 
         $controllerName = Common::getRequestVar('moduleToWidgetize');
@@ -82,4 +92,5 @@ class Controller extends \Piwik\Plugin\Controller
 
         return $view->render();
     }
+
 }