allow invalidating plugin archives only and archiving past data for plugins (#15889)

* Adding initial new code for cron archive rewrite.

* first pass at removing unused CronArchive code.

* unfinished commit

* fill out archiveinvalidator code

* getting some tests to pass

* unfinished commit

* fixing part of test

* Another test fix.

* another sql change

* fix broken merge or something else that went wrong

* Couple more fixes and extra logs.

* Fixing enough issues to get core archive command to run completely.

* Fix and log change.

* Fixed more segment/test related issues for CronArchiveTest. Includes optimization for no visits for period + segment process from handling.

* another optimization and possible build fix

* no visit optimization

* test fix

* Implement archiving_custom_ranges logic w/ queue based implementation

* fixes to get archivecrontest to work

* add logic to invalidate today period

* fix optimization and some tests

* Fixing more tests.

* Fixing more tests

* debug travis failure

* more test fixes

* more test fixes, removing more unneeded code, handling some TODOs

* Handle more TODOs including creating ArchiveFilter class for some cli options.

* tests and todos

* idarchives are specific to table + start on archivefilter tests

* one test

* more TODOs and tests

* more tests and todo taken care of

* handle more todos

* fixing more tests

* fix comment

* make sure autoarchiving is enabled for segments when cron archive picks them up

* Fixing test.

* apply more pr feedback

* order by date1 asc

* quick refactor

* use batch insert instead of createDummyArchives

* apply rest of pr feedback

* add removed events, add new test, fix an issue (when deleting idarchives older than do not lump all segments together).

* re-add fixed/shared siteids

* fix tests

* incomplete commit

* Insert archive entries into archive_invalidations table.

* Use invalidations table in core:archive and get ArchiveCronTest to pass.

* fixing some tests

* debugging travis

* fix more tests & remove DONE_IN_PROGRESS which is no longer used.

* fix more tests

* Allow forcing plugin specific archive in core:archive.

* When querying from archive data use all available archives including "all" archives and plugin specific archives.

* Adding some code for invalidating specific plugin archives.

* Get archive invalidation test to pass.

* add plugin capability to invalidate command

* Handle plugin only archives in core:archive.

* Add Archive test and get ArchiveCronTest to pass.

* update some expected files

* Fix some more tests.

* incomplete commit

* allow invalidating individual reports

* adding more API for DONE_PARTIAL support

* get archivecrontest to pass

* add archive processor tests

* fix some test randomnes

* when purging keep latest partial archives if there is no newer whole archive

* add rearchivereport method + some unfinished tests

* Add archiveReports API method, fix race condition in test, when archiving single report, always ignore inserting other reports.

* require archivers to handle partial archives themselves entirely instead of trying to do it automatically and allow requested report to be any string

* couple fixes

* Use core config option for last N montsh to invalidate.

* Add test for ArchiveSelector method.

* Ignore archives w/ deactivated plugins.

* Refactor queue looping into new QueueConsumer class.

* apply more review feedback + another fix

* invalidate segments too in reArchiveReport w/ etsts

* remove DONE_IN_PROGRESS, no longer used. use new status in query and add queue consumer test.

* forgot to add file

* delete old unneeded archives when finalizing a new one.

* tweak invalidation archive description

* add plugin archiving tests and get them to pass

* fix test

* many fixes

* fix another test

* update expected test files

* fix more tests

* last test fixes hopefully

* tweak log

* In case a column already exists, do not try to add it in an AddColumns migration or the entire migration will fail and no columns will be added.

* try to fix tests again

* fix again?

* apply some review feedback + fix test

* fix test

* fix another test

* couple fixes

* Remove extra param.

* apply pr feedback

* check for usable archive before invalidating and before initiating archiving

* fixing tests

* fixing tests

* fixing tests

* fix another test issue

* fix archiveinvalidator test

* fix one test and debug another

* more debugging

* fix test

* use twig

* remove no longer needed change

* add back previous logic

* fix tracking is not working

* apply pr feedback and add tests

* fixing tests

* update submodule

* debugging random travis failure

* update test

* more debugging

* more debugging

* another attempt at debugging

* Lets try this fix

* trying to fix the build

* debug

* simpler test

* fix test

* fix test

* fix test

* fix test

* fix test failure

* update screenshots

* update screenshots

Co-authored-by: Thomas Steur <tsteur@users.noreply.github.com>

3 files changed, 113 insertions, 0 deletions

diff --git a/plugins/ExamplePlugin/API.php b/plugins/ExamplePlugin/API.php
index 44d67cd80c..88f6249ecb 100644
--- a/plugins/ExamplePlugin/API.php
+++ b/plugins/ExamplePlugin/API.php
@@ -8,6 +8,7 @@
 
 namespace Piwik\Plugins\ExamplePlugin;
 
+use Piwik\Archive;
 use Piwik\DataTable;
 use Piwik\DataTable\Row;
 
@@ -54,4 +55,19 @@ class API extends \Piwik\Plugin\API
 
         return $table;
     }
+
+    /**
+     * Returns the example metric we archive in Archiver.php.
+     * @param int    $idSite
+     * @param string $period
+     * @param string $date
+     * @param bool|string $segment
+     * @return DataTable
+     */
+    public function getExampleArchivedMetric($idSite, $period, $date, $segment = false)
+    {
+        $archive = Archive::build($idSite, $period, $date, $segment);
+        $dataTable = $archive->getDataTableFromNumeric([Archiver::EXAMPLEPLUGIN_METRIC_NAME, Archiver::EXAMPLEPLUGIN_CONST_METRIC_NAME]);
+        return $dataTable;
+    }
 }
diff --git a/plugins/ExamplePlugin/Archiver.php b/plugins/ExamplePlugin/Archiver.php
index f507c6087a..13455ce6eb 100644
--- a/plugins/ExamplePlugin/Archiver.php
+++ b/plugins/ExamplePlugin/Archiver.php
@@ -8,6 +8,13 @@
 
 namespace Piwik\Plugins\ExamplePlugin;
 
+use Piwik\ArchiveProcessor;
+use Piwik\Container\StaticContainer;
+use Piwik\Date;
+use Piwik\Option;
+use Piwik\Sequence;
+use Psr\Log\LoggerInterface;
+
 /**
  * Class Archiver
  *
@@ -31,6 +38,27 @@ class Archiver extends \Piwik\Plugin\Archiver
      * This is only an example record name, so feel free to change it to suit your needs.
      */
     const EXAMPLEPLUGIN_ARCHIVE_RECORD = "ExamplePlugin_archive_record";
+    const EXAMPLEPLUGIN_METRIC_NAME = 'ExamplePlugin_example_metric';
+    const EXAMPLEPLUGIN_CONST_METRIC_NAME = 'ExamplePlugin_example_metric2';
+
+    private $daysFrom = '2016-07-08';
+
+    /**
+     * @var string
+     */
+    private $requestedReport = null;
+
+    public function __construct(ArchiveProcessor $processor)
+    {
+        parent::__construct($processor);
+
+        $this->requestedReport = $processor->getParams()->getArchiveOnlyReport();
+        if ($this->requestedReport) {
+            $processor->getParams()->setIsPartialArchive(true);
+        }
+
+        $this->createSequence();
+    }
 
     public function aggregateDayReport()
     {
@@ -45,6 +73,25 @@ class Archiver extends \Piwik\Plugin\Archiver
          * $visitorReport = $visitorMetrics->getSerialized();
          * $this->getProcessor()->insertBlobRecord(self::EXAMPLEPLUGIN_ARCHIVE_RECORD, $visitorReport);
          */
+
+        if ($this->isArchiving(self::EXAMPLEPLUGIN_METRIC_NAME)) {
+            // insert a test numeric metric that is the difference in days between the day we're archiving and
+            // $this->daysFrom.
+            $daysFrom = Date::factory($this->daysFrom);
+            $date = $this->getProcessor()->getParams()->getPeriod()->getDateStart();
+
+            $differenceInSeconds = $daysFrom->getTimestamp() - $date->getTimestamp();
+            $differenceInDays = round($differenceInSeconds / 86400);
+
+            $this->getProcessor()->insertNumericRecord(self::EXAMPLEPLUGIN_METRIC_NAME, $differenceInDays);
+        }
+
+        if ($this->isArchiving(self::EXAMPLEPLUGIN_CONST_METRIC_NAME)) {
+            $archiveCount = $this->incrementArchiveCount();
+            $archiveCount = 50 + $archiveCount;
+            $archiveCount += 5 - ($archiveCount % 5); // round up to nearest 5 multiple to avoid random test failures
+            $this->getProcessor()->insertNumericRecord(self::EXAMPLEPLUGIN_CONST_METRIC_NAME, $archiveCount);
+        }
     }
 
     public function aggregateMultipleReports()
@@ -57,5 +104,41 @@ class Archiver extends \Piwik\Plugin\Archiver
          *
          * $this->getProcessor()->aggregateDataTableRecords(self::EXAMPLEPLUGIN_ARCHIVE_RECORD);
          */
+
+        $reports = [];
+        if ($this->isArchiving(self::EXAMPLEPLUGIN_METRIC_NAME)) {
+            $reports[] = self::EXAMPLEPLUGIN_METRIC_NAME;
+        }
+        if ($this->isArchiving(self::EXAMPLEPLUGIN_CONST_METRIC_NAME)) {
+            $reports[] = self::EXAMPLEPLUGIN_CONST_METRIC_NAME;
+        }
+        $this->getProcessor()->aggregateNumericMetrics($reports);
+    }
+
+    private function incrementArchiveCount()
+    {
+        $sequence = new Sequence('ExamplePlugin_archiveCount');
+        $result = $sequence->getNextId();
+        return $result;
+    }
+
+    private function isArchiving(string $reportName)
+    {
+        return empty($this->requestedReport) || $this->requestedReport == $reportName;
+    }
+
+    private function createSequence()
+    {
+        $sequence = new Sequence('ExamplePlugin_archiveCount');
+        if (!$sequence->exists()) {
+            for ($i = 0; $i < 100; ++$i) {
+                try {
+                    $sequence->create();
+                    break;
+                } catch (\Exception $ex) {
+                    // ignore
+                }
+            }
+        }
     }
 }
diff --git a/plugins/ExamplePlugin/ExamplePlugin.php b/plugins/ExamplePlugin/ExamplePlugin.php
index f974975c42..8b737f5784 100644
--- a/plugins/ExamplePlugin/ExamplePlugin.php
+++ b/plugins/ExamplePlugin/ExamplePlugin.php
@@ -10,4 +10,18 @@ namespace Piwik\Plugins\ExamplePlugin;
 
 class ExamplePlugin extends \Piwik\Plugin
 {
+    public function registerEvents()
+    {
+        return [
+            'CronArchive.getArchivingAPIMethodForPlugin' => 'getArchivingAPIMethodForPlugin',
+        ];
+    }
+
+    // support archiving just this plugin via core:archive
+    public function getArchivingAPIMethodForPlugin(&$method, $plugin)
+    {
+        if ($plugin == 'ExamplePlugin') {
+            $method = 'ExamplePlugin.getExampleArchivedMetric';
+        }
+    }
 }