Proportional evolution comparison for incomplete periods (#18099)

* Multisite evolution metrics changed to calculate proportionally to percent of the current period which is complete

* Use piwik date class, use report generated date if available, added unit test, added tooltip

* Improved tooltip detail

* Updated unit tested, added tests for evolution metric getRatio(), changes to allow row metadata to be preserved during datatable merges

* Additional API test fixes

* More test fixes

* More test fixes

* Remove ts_archived row metadata from final API output

* Test fix reversions, added deleteRowsMetadata() method to DataTableInterface

* More test fix reversions

* Fixed integration test

* Trigger Build

* Update core/DataTable/Map.php

Co-authored-by: Stefan Giehl <stefan@matomo.org>

* Update core/DataTable.php

Co-authored-by: Stefan Giehl <stefan@matomo.org>

* Update core/Archive/DataCollection.php

Co-authored-by: Stefan Giehl <stefan@matomo.org>

* Update core/DataTable.php

Co-authored-by: Stefan Giehl <stefan@matomo.org>

* Update core/DataTable.php

Co-authored-by: Stefan Giehl <stefan@matomo.org>

* Update plugins/CoreHome/Columns/Metrics/EvolutionMetric.php

Co-authored-by: Stefan Giehl <stefan@matomo.org>

* Improved tooltips for translation, use NumberFormatter for percents, moved additional constructor param to end, null checks

* Update plugins/CoreHome/Columns/Metrics/EvolutionMetric.php

Co-authored-by: Stefan Giehl <stefan@matomo.org>

* Update plugins/CoreHome/Columns/Metrics/EvolutionMetric.php

Co-authored-by: Stefan Giehl <stefan@matomo.org>

* Use localized period string, remove unnecessary tooltip percent digits

* Formatting fixes

* Fix for an issue where evolution values > 999% would be displayed incorrectly

* Added data table processor option to provide raw copy of formatted metrics

* Update plugins/MultiSites/API.php

Fix for row metadata removed too early

Co-authored-by: Stefan Giehl <stefan@matomo.org>

* Replace evolution metrics 'add raw copy' api parameter with _trend column

* ensure to use correct metric to check if lower value is better

* updates expected test files

* fix some more tests

* update test file

Co-authored-by: Stefan Giehl <stefan@matomo.org>

8 files changed, 159 insertions, 14 deletions

diff --git a/plugins/MultiSites/API.php b/plugins/MultiSites/API.php
index a0e20a470c..097725bd84 100644
--- a/plugins/MultiSites/API.php
+++ b/plugins/MultiSites/API.php
@@ -313,6 +313,12 @@ class API extends \Piwik\Plugin\API
             );
         }
 
+        // Remove <ts_archived> row metadata, it's already been used by any filters that needed it
+        $dataTable->queueFilter(function($dataTable) {
+            $dataTable->deleteRowsMetadata(DataTable::ARCHIVED_DATE_METADATA_NAME);
+            $dataTable->deleteColumn('_metadata');
+        });
+
         if ($multipleWebsitesRequested && $dataTable->getRowsCount() === 1 && $dataTable instanceof DataTable\Simple) {
             $simpleTable = $dataTable;
             $dataTable   = $simpleTable->getEmptyClone();
@@ -358,7 +364,8 @@ class API extends \Piwik\Plugin\API
                     $metricSettings[self::METRIC_RECORD_NAME_KEY],
                     $pastData,
                     $metricSettings[self::METRIC_EVOLUTION_COL_NAME_KEY],
-                    $quotientPrecision = 1
+                    $quotientPrecision = 1,
+                    $currentData
                 );
             }
             $currentData->setMetadata(DataTable::EXTRA_PROCESSED_METRICS_METADATA_NAME, $extraProcessedMetrics);
diff --git a/plugins/MultiSites/Columns/Metrics/EcommerceOnlyEvolutionMetric.php b/plugins/MultiSites/Columns/Metrics/EcommerceOnlyEvolutionMetric.php
index 58718b0101..0a1fa62742 100644
--- a/plugins/MultiSites/Columns/Metrics/EcommerceOnlyEvolutionMetric.php
+++ b/plugins/MultiSites/Columns/Metrics/EcommerceOnlyEvolutionMetric.php
@@ -22,9 +22,10 @@ class EcommerceOnlyEvolutionMetric extends EvolutionMetric
 {
     private $isRevenueEvolution;
 
-    public function __construct($wrapped, DataTable $pastData, $evolutionMetricName = false, $quotientPrecision = 0)
+    public function __construct($wrapped, DataTable $pastData, $evolutionMetricName = false, $quotientPrecision = 0,
+                                ?DataTable $currentData = null)
     {
-        parent::__construct($wrapped, $pastData, $evolutionMetricName, $quotientPrecision);
+        parent::__construct($wrapped, $pastData, $evolutionMetricName, $quotientPrecision, $currentData);
 
         $this->isRevenueEvolution = $this->getName() == 'revenue_evolution';
     }
diff --git a/plugins/MultiSites/MultiSites.php b/plugins/MultiSites/MultiSites.php
index f5aa8bb795..27bf465763 100644
--- a/plugins/MultiSites/MultiSites.php
+++ b/plugins/MultiSites/MultiSites.php
@@ -72,6 +72,12 @@ class MultiSites extends \Piwik\Plugin
         $translations[] = 'General_Faq';
         $translations[] = 'Feedback_CommunityHelp';
         $translations[] = 'Feedback_ProfessionalHelp';
+        $translations[] = 'MultiSites_EvolutionComparisonIncomplete';
+        $translations[] = 'MultiSites_EvolutionComparisonProportional';
+        $translations[] = 'MultiSites_EvolutionComparisonDay';
+        $translations[] = 'MultiSites_EvolutionComparisonWeek';
+        $translations[] = 'MultiSites_EvolutionComparisonMonth';
+        $translations[] = 'MultiSites_EvolutionComparisonYear';
     }
 
     public function getJsFiles(&$jsFiles)
diff --git a/plugins/MultiSites/angularjs/dashboard/dashboard-model.service.js b/plugins/MultiSites/angularjs/dashboard/dashboard-model.service.js
index a1f2936e97..32c2f404ca 100644
--- a/plugins/MultiSites/angularjs/dashboard/dashboard-model.service.js
+++ b/plugins/MultiSites/angularjs/dashboard/dashboard-model.service.js
@@ -63,9 +63,57 @@
 
             var allSites = report.sites;
             angular.forEach(allSites, function (site, index) {
-                site.visits_evolution    = parseInt(site.visits_evolution, 10);
-                site.pageviews_evolution = parseInt(site.pageviews_evolution, 10);
-                site.revenue_evolution   = parseInt(site.revenue_evolution, 10);
+
+                if (site.hasOwnProperty('ratio') && site.ratio != 1) {
+                    var percent = NumberFormatter.formatPercent(Math.round((site.ratio * 100)));
+                    var metricName = null;
+                    var previousTotal = 0;
+                    var currentTotal = 0;
+                    var evolution = 0;
+                    var previousTotalAdjusted = 0;
+                    if (model.sortColumn == 'nb_visits' || model.sortColumn == 'visits_evolution') {
+                        previousTotal = NumberFormatter.formatNumber(site.previous_nb_visits);
+                        currentTotal = NumberFormatter.formatNumber(site.nb_visits);
+                        evolution = NumberFormatter.formatPercent(site.visits_evolution);
+                        metricName = _pk_translate("General_ColumnNbVisits");
+                        previousTotalAdjusted = NumberFormatter.formatNumber(Math.round(site.previous_nb_visits * site.ratio));
+                    }
+                    if (model.sortColumn == 'pageviews_evolution') {
+                        previousTotal = site.previous_Actions_nb_pageviews;
+                        currentTotal = site.nb_pageviews;
+                        evolution = NumberFormatter.formatPercent(site.pageviews_evolution);
+                        metricName = _pk_translate("General_ColumnPageviews");
+                        previousTotalAdjusted = NumberFormatter.formatNumber(Math.round(site.previous_Actions_nb_pageviews * site.ratio));
+                    }
+                    if (model.sortColumn == 'revenue_evolution') {
+                        previousTotal = NumberFormatter.formatCurrency(site.previous_Goal_revenue, site.currencySymbol);
+                        currentTotal = NumberFormatter.formatCurrency(site.revenue, site.currencySymbol);
+                        evolution = NumberFormatter.formatPercent(site.revenue_evolution);
+                        metricName = _pk_translate("General_ColumnRevenue");
+                        previousTotalAdjusted = NumberFormatter.formatCurrency(Math.round(site.previous_Goal_revenue * site.ratio), site.currencySymbol);
+                    }
+
+                    if (metricName) {
+                        site.tooltip = _pk_translate("MultiSites_EvolutionComparisonIncomplete", [percent]) + "\n";
+                        site.tooltip += _pk_translate("MultiSites_EvolutionComparisonProportional", [percent, previousTotalAdjusted, metricName, previousTotal]) + "\n";
+
+                        switch (site.periodName) {
+                            case 'day':
+                                site.tooltip += _pk_translate("MultiSites_EvolutionComparisonDay", [currentTotal, metricName, previousTotalAdjusted, site.previousRange, evolution]);
+                                break;
+                            case 'week':
+                                site.tooltip += _pk_translate("MultiSites_EvolutionComparisonWeek", [currentTotal, metricName, previousTotalAdjusted, site.previousRange, evolution]);
+                                break;
+                            case 'month':
+                                site.tooltip += _pk_translate("MultiSites_EvolutionComparisonMonth", [currentTotal, metricName, previousTotalAdjusted, site.previousRange, evolution]);
+                                break;
+                            case 'year':
+                                site.tooltip += _pk_translate("MultiSites_EvolutionComparisonYear", [currentTotal, metricName, previousTotalAdjusted, site.previousRange, evolution]);
+                                break;
+                        }
+                    }
+
+                }
             });
 
             model.totalVisits   = report.totals.nb_visits;
@@ -110,7 +158,7 @@
 
             model.sortColumn = metric;
             fetchAllSites();
-        };
+        }
 
         function previousPage() {
             model.currentPage = model.currentPage - 1;
@@ -145,7 +193,7 @@
                 filter_sort_order: 'asc',
                 filter_limit: model.pageSize,
                 filter_offset: getCurrentPagingOffset(),
-                showColumns: 'label,nb_visits,nb_pageviews,visits_evolution,pageviews_evolution,revenue_evolution,nb_actions,revenue'
+                showColumns: 'label,nb_visits,nb_pageviews,visits_evolution,visits_evolution_trend,pageviews_evolution,pageviews_evolution_trend,revenue_evolution,revenue_evolution_trend,nb_actions,revenue'
             };
 
             if (model.searchTerm) {
diff --git a/plugins/MultiSites/angularjs/site/site.directive.html b/plugins/MultiSites/angularjs/site/site.directive.html
index 376d70349a..b2b0d3780a 100644
--- a/plugins/MultiSites/angularjs/site/site.directive.html
+++ b/plugins/MultiSites/angularjs/site/site.directive.html
@@ -20,11 +20,11 @@
         <span class="value">{{ website.revenue }}</span>
     </td>
 
-    <td ng-if="period != 'range'" class="multisites-evolution">
+    <td ng-if="period != 'range'" class="multisites-evolution" title="{{ website.tooltip }}">
         <div class="visits value" ng-if="!website.isGroup">
-            <span ng-show="website[evolutionMetric] > 0"><img src="plugins/MultiSites/images/arrow_up.png" alt="" /> <span style="color: green;">{{ website[evolutionMetric] }}%</span></span>
-            <span ng-show="website[evolutionMetric] == 0"><img src="plugins/MultiSites/images/stop.png" alt="" /> <span>{{ website[evolutionMetric] }}%</span></span>
-            <span ng-show="website[evolutionMetric] < 0"><img src="plugins/MultiSites/images/arrow_down.png" alt="" /> <span style="color: red;">{{ website[evolutionMetric] }}%</span></span>
+            <span ng-show="website[evolutionMetric+'_trend'] == 1"><img src="plugins/MultiSites/images/arrow_up.png" alt="" /> <span style="color: green;">{{ website[evolutionMetric] }}</span></span>
+            <span ng-show="website[evolutionMetric+'_trend'] == 0"><img src="plugins/MultiSites/images/stop.png" alt="" /> <span>{{ website[evolutionMetric] }}</span></span>
+            <span ng-show="website[evolutionMetric+'_trend'] == -1"><img src="plugins/MultiSites/images/arrow_down.png" alt="" /> <span style="color: red;">{{ website[evolutionMetric] }}</span></span>
         </div>
     </td>
 
@@ -37,3 +37,4 @@
         </div>
     </td>
 </tr>
+
diff --git a/plugins/MultiSites/lang/en.json b/plugins/MultiSites/lang/en.json
index f67438a2e4..c2a351b689 100644
--- a/plugins/MultiSites/lang/en.json
+++ b/plugins/MultiSites/lang/en.json
@@ -5,6 +5,14 @@
         "PluginDescription": "View and compare all your websites and apps in this useful 'All Websites' dashboard.",
         "TopLinkTooltip": "Compare Web Analytics stats for all of your Websites.",
         "SingleWebsitesDashboardDocumentation": "This report gives you an informational overview for a specific website, containing the most general metrics about your visitors.",
-        "AllWebsitesDashboardDocumentation": "This report gives you an informational overview for each of your websites, containing the most general metrics about your visitors."
+        "AllWebsitesDashboardDocumentation": "This report gives you an informational overview for each of your websites, containing the most general metrics about your visitors.",
+        "EvolutionComparisonIncomplete": "The currently selected time period is %1$s complete.",
+        "EvolutionComparisonProportional": "When the previous period was also %1$s complete, there would have been an estimated %2$s %3$s\n (out of a total of %4$s %3$s in the previous period).",
+        "EvolutionComparisonDay": "%1$s %2$s this day compared to %3$s %2$s in the previous period (%4$s) Evolution: %5$s",
+        "EvolutionComparisonWeek": "%1$s %2$s this week compared to %3$s %2$s in the previous period (%4$s) Evolution: %5$s",
+        "EvolutionComparisonMonth": "%1$s %2$s this month compared to %3$s %2$s in the previous period (%4$s) Evolution: %5$s",
+        "EvolutionComparisonYear": "%1$s %2$s this year compared to %3$s %2$s in the previous period (%4$s) Evolution: %5$s"
+
+
     }
 }
\ No newline at end of file
diff --git a/plugins/MultiSites/tests/Integration/ControllerTest.php b/plugins/MultiSites/tests/Integration/ControllerTest.php
index b0ac45ad8f..dd54beefad 100644
--- a/plugins/MultiSites/tests/Integration/ControllerTest.php
+++ b/plugins/MultiSites/tests/Integration/ControllerTest.php
@@ -53,6 +53,24 @@ class ControllerTest extends SystemTestCase
             'nb_conversions_evolution' => '100%',
             'orders_evolution' => '100%',
             'ecommerce_revenue_evolution' => '100%',
+            'ratio' => 1,
+            'previous_nb_visits' => 0,
+            'periodName' => 'day',
+            'previousRange' => 'Tue, Jan 22',
+            'previous_nb_actions' => 0,
+            'previous_Actions_nb_pageviews' => 0,
+            'previous_Goal_revenue' => 0,
+            'previous_Goal_nb_conversions' => 0,
+            'previous_Goal_0_nb_conversions' => 0,
+            'previous_Goal_0_revenue' => 0,
+            'visits_evolution_trend' => 1,
+            'actions_evolution_trend' => 1,
+            'pageviews_evolution_trend' => 1,
+            'revenue_evolution_trend' => 1,
+            'nb_conversions_evolution_trend' => 1,
+            'orders_evolution_trend' => 1,
+            'ecommerce_revenue_evolution_trend' => 1,
+            'currencySymbol' => '$',
         ), $sites['sites'][0]);
 
         unset($sites['sites']);
diff --git a/plugins/MultiSites/tests/Integration/DashboardTest.php b/plugins/MultiSites/tests/Integration/DashboardTest.php
index 982a6ecc37..277f52617b 100644
--- a/plugins/MultiSites/tests/Integration/DashboardTest.php
+++ b/plugins/MultiSites/tests/Integration/DashboardTest.php
@@ -80,6 +80,20 @@ class DashboardTest extends IntegrationTestCase
                 'main_url' => 'http://piwik.net',
                 'nb_conversions' => 0,
                 'nb_conversions_evolution' => '0%',
+                'ratio' => 1,
+                'previous_nb_visits' => 0,
+                'periodName' => 'day',
+                'previousRange' => 'Wed, Dec 12',
+                'previous_nb_actions' => 0,
+                'previous_Actions_nb_pageviews' => 0,
+                'previous_Goal_revenue' => 0,
+                'previous_Goal_nb_conversions' => 0,
+                'visits_evolution_trend' => 0,
+                'actions_evolution_trend' => 0,
+                'pageviews_evolution_trend' => 0,
+                'revenue_evolution_trend' => 0,
+                'nb_conversions_evolution_trend' => 0,
+                'currencySymbol' => '$',
             ),
             array (
                 'label' => 'Site 2',
@@ -96,7 +110,21 @@ class DashboardTest extends IntegrationTestCase
                 'main_url' => 'http://piwik.net',
                 'nb_conversions' => 0,
                 'nb_conversions_evolution' => '0%',
-            ),
+                'ratio' => 1,
+                'previous_nb_visits' => 0,
+                'periodName' => 'day',
+                'previousRange' => 'Wed, Dec 12',
+                'previous_nb_actions' => 0,
+                'previous_Actions_nb_pageviews' => 0,
+                'previous_Goal_revenue' => 0,
+                'previous_Goal_nb_conversions' => 0,
+                'visits_evolution_trend' => 0,
+                'actions_evolution_trend' => 0,
+                'pageviews_evolution_trend' => 0,
+                'revenue_evolution_trend' => 0,
+                'nb_conversions_evolution_trend' => 0,
+                'currencySymbol' => '$',
+    ),
             array (
                 'label' => 'Site 3',
                 'nb_visits' => '0',
@@ -112,6 +140,20 @@ class DashboardTest extends IntegrationTestCase
                 'main_url' => 'http://piwik.net',
                 'nb_conversions' => 0,
                 'nb_conversions_evolution' => '0%',
+                'ratio' => 1,
+                'previous_nb_visits' => 0,
+                'periodName' => 'day',
+                'previousRange' => 'Wed, Dec 12',
+                'previous_nb_actions' => 0,
+                'previous_Actions_nb_pageviews' => 0,
+                'previous_Goal_revenue' => 0,
+                'previous_Goal_nb_conversions' => 0,
+                'visits_evolution_trend' => 0,
+                'actions_evolution_trend' => 0,
+                'pageviews_evolution_trend' => 0,
+                'revenue_evolution_trend' => 0,
+                'nb_conversions_evolution_trend' => 0,
+                'currencySymbol' => '$',
             ),
         );
         $this->assertEquals($expectedSites, $dashboard->getSites(array(), $limit = 10));
@@ -138,6 +180,20 @@ class DashboardTest extends IntegrationTestCase
                 'main_url' => 'http://piwik.net',
                 'nb_conversions' => 0,
                 'nb_conversions_evolution' => '0%',
+                'ratio' => 1,
+                'previous_nb_visits' => 0,
+                'periodName' => 'day',
+                'previousRange' => 'Wed, Dec 12',
+                'previous_nb_actions' => 0,
+                'previous_Actions_nb_pageviews' => 0,
+                'previous_Goal_revenue' => 0,
+                'previous_Goal_nb_conversions' => 0,
+                'visits_evolution_trend' => 0,
+                'actions_evolution_trend' => 0,
+                'pageviews_evolution_trend' => 0,
+                'revenue_evolution_trend' => 0,
+                'nb_conversions_evolution_trend' => 0,
+                'currencySymbol' => '$',
             ),
         );
         $dashboard->search('site 2');