Add notification when report w/ segment has no data, but segment is unprocessed (#12823)3.6.0-b4

* When a report has no data, check if it is for an unprocessed segment and display an explanatory notification.

* Remove transient notifications on reporting page change, allow datatable views to add notifications, use to display unprocessed segment message.

* Update changelog.

* Internationalize unprocessed segment message.

* Parse notification divs in dashboardWidget.js when setting widget content.

* Tweak message.

* Change PR to use different approach: throw exception when no archives found and segment is used, then detect exception in new event.

* Update changelog + document new event.

* Unfinished review changes

* more review fixes

* Do not show notification w/ custom segments.

* apply pr review

* Apply review.

* undo escaping since it was not needed & get test to pass

* Different strategy + new test.

* Fix tests.

* Update two expected screenshots.

9 files changed, 300 insertions, 0 deletions

diff --git a/plugins/SegmentEditor/tests/System/UnprocessedSegmentsTest.php b/plugins/SegmentEditor/tests/System/UnprocessedSegmentsTest.php
new file mode 100644
index 0000000000..e4d3d45c17
--- /dev/null
+++ b/plugins/SegmentEditor/tests/System/UnprocessedSegmentsTest.php
@@ -0,0 +1,226 @@
+<?php
+/**
+ * Piwik - free/libre analytics platform
+ *
+ * @link http://piwik.org
+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
+ */
+
+namespace Piwik\Plugins\SegmentEditor\tests\System;
+
+use Piwik\ArchiveProcessor\Rules;
+use Piwik\Common;
+use Piwik\Config;
+use Piwik\Date;
+use Piwik\Db;
+use Piwik\Plugins\SegmentEditor\API;
+use Piwik\Plugins\VisitsSummary;
+use Piwik\Tests\Fixtures\OneVisitorTwoVisits;
+use Piwik\Tests\Framework\TestCase\IntegrationTestCase;
+
+/**
+ * @group SegmentEditor
+ * @group System
+ * @group UnprocessedSegmentsTest
+ */
+class UnprocessedSegmentsTest extends IntegrationTestCase
+{
+    /**
+     * @var OneVisitorTwoVisits
+     */
+    public static $fixture;
+
+    const TEST_SEGMENT = 'browserCode==ff';
+
+    public function test_apiOutput_whenCustomSegmentUsed_WithBrowserArchivingDisabled()
+    {
+        Rules::setBrowserTriggerArchiving(false);
+
+        $segments = Rules::getSegmentsToProcess([self::$fixture->idSite]);
+        $this->assertNotContains(self::TEST_SEGMENT, $segments);
+
+        $this->runAnyApiTest('VisitsSummary.get', 'customSegmentUnprocessed', [
+            'idSite' => self::$fixture->idSite,
+            'date' => Date::factory(self::$fixture->dateTime)->toString(),
+            'period' => 'week',
+            'segment' => self::TEST_SEGMENT,
+        ]);
+    }
+
+    public function test_apiOutput_whenRealTimeProcessedSegmentUsed_WithBrowserArchivingDisabled()
+    {
+        $idSegment = API::getInstance()->add('testsegment', self::TEST_SEGMENT, self::$fixture->idSite, $autoArchive = false);
+
+        $storedSegment = API::getInstance()->get($idSegment);
+        $this->assertNotEmpty($storedSegment);
+
+        Rules::setBrowserTriggerArchiving(false);
+
+        $segments = Rules::getSegmentsToProcess([self::$fixture->idSite]);
+        $this->assertNotContains(self::TEST_SEGMENT, $segments);
+
+        $this->runAnyApiTest('VisitsSummary.get', 'realTimeSegmentUnprocessed', [
+            'idSite' => self::$fixture->idSite,
+            'date' => Date::factory(self::$fixture->dateTime)->toString(),
+            'period' => 'week',
+            'segment' => self::TEST_SEGMENT,
+        ]);
+    }
+
+    public function test_apiOutput_whenUnprocessedAutoArchiveSegmentUsed_WithBrowserArchivingDisabled()
+    {
+        $idSegment = API::getInstance()->add('testsegment', self::TEST_SEGMENT, self::$fixture->idSite, $autoArchive = true);
+
+        $storedSegment = API::getInstance()->get($idSegment);
+        $this->assertNotEmpty($storedSegment);
+
+        Rules::setBrowserTriggerArchiving(false);
+
+        $segments = Rules::getSegmentsToProcess([self::$fixture->idSite]);
+        $this->assertContains(self::TEST_SEGMENT, $segments);
+
+        $this->runAnyApiTest('VisitsSummary.get', 'autoArchiveSegmentUnprocessed', [
+            'idSite' => self::$fixture->idSite,
+            'date' => Date::factory(self::$fixture->dateTime)->toString(),
+            'period' => 'week',
+            'segment' => self::TEST_SEGMENT,
+        ]);
+    }
+
+    public function test_apiOutput_whenPreprocessedSegmentUsed_WithBrowserArchivingDisabled()
+    {
+        $idSegment = API::getInstance()->add('testsegment', self::TEST_SEGMENT, self::$fixture->idSite, $autoArchive = true);
+
+        $storedSegment = API::getInstance()->get($idSegment);
+        $this->assertNotEmpty($storedSegment);
+
+        VisitsSummary\API::getInstance()->get(self::$fixture->idSite, 'week',
+            Date::factory(self::$fixture->dateTime)->toString(), self::TEST_SEGMENT); // archive
+
+        Rules::setBrowserTriggerArchiving(false);
+
+        $segments = Rules::getSegmentsToProcess([self::$fixture->idSite]);
+        $this->assertContains(self::TEST_SEGMENT, $segments);
+
+        $this->runAnyApiTest('VisitsSummary.get', 'autoArchiveSegmentPreprocessed', [
+            'idSite' => self::$fixture->idSite,
+            'date' => Date::factory(self::$fixture->dateTime)->toString(),
+            'period' => 'week',
+            'segment' => self::TEST_SEGMENT,
+        ]);
+    }
+
+    public function test_apiOutput_whenPreprocessedCustomSegmentUsed_WithBrowserArchivingDisabled()
+    {
+        VisitsSummary\API::getInstance()->get(self::$fixture->idSite, 'week',
+            Date::factory(self::$fixture->dateTime)->toString(), self::TEST_SEGMENT); // archive
+
+        Rules::setBrowserTriggerArchiving(false);
+
+        $segments = Rules::getSegmentsToProcess([self::$fixture->idSite]);
+        $this->assertNotContains(self::TEST_SEGMENT, $segments);
+
+        $this->runAnyApiTest('VisitsSummary.get', 'customSegmentPreprocessed', [
+            'idSite' => self::$fixture->idSite,
+            'date' => Date::factory(self::$fixture->dateTime)->toString(),
+            'period' => 'week',
+            'segment' => self::TEST_SEGMENT,
+        ]);
+    }
+
+    public function test_apiOutput_whenPreprocessedSegmentUsed_WithNoData_AndBrowserArchivingDisabled()
+    {
+        $this->clearLogData();
+
+        $idSegment = API::getInstance()->add('testsegment', self::TEST_SEGMENT, self::$fixture->idSite, $autoArchive = true);
+
+        $storedSegment = API::getInstance()->get($idSegment);
+        $this->assertNotEmpty($storedSegment);
+
+        VisitsSummary\API::getInstance()->get(self::$fixture->idSite, 'week',
+            Date::factory(self::$fixture->dateTime)->toString(), self::TEST_SEGMENT); // archive
+
+        Rules::setBrowserTriggerArchiving(false);
+
+        $segments = Rules::getSegmentsToProcess([self::$fixture->idSite]);
+        $this->assertContains(self::TEST_SEGMENT, $segments);
+
+        $this->runAnyApiTest('VisitsSummary.get', 'autoArchiveSegmentNoDataPreprocessed', [
+            'idSite' => self::$fixture->idSite,
+            'date' => Date::factory(self::$fixture->dateTime)->toString(),
+            'period' => 'week',
+            'segment' => self::TEST_SEGMENT,
+        ]);
+    }
+
+    public function test_apiOutput_whenNoLogDataAndUnprocessedSegmentUsed_AndBrowserArchivingDisabled()
+    {
+        $this->clearLogData();
+
+        $idSegment = API::getInstance()->add('testsegment', self::TEST_SEGMENT, self::$fixture->idSite, $autoArchive = true);
+
+        $storedSegment = API::getInstance()->get($idSegment);
+        $this->assertNotEmpty($storedSegment);
+
+        Rules::setBrowserTriggerArchiving(false);
+
+        $segments = Rules::getSegmentsToProcess([self::$fixture->idSite]);
+        $this->assertContains(self::TEST_SEGMENT, $segments);
+
+        $this->runAnyApiTest('VisitsSummary.get', 'noLogDataSegmentUnprocessed', [
+            'idSite' => self::$fixture->idSite,
+            'date' => Date::factory(self::$fixture->dateTime)->toString(),
+            'period' => 'week',
+            'segment' => self::TEST_SEGMENT,
+        ]);
+    }
+
+    public function test_apiOutput_whenMultipleSitesRequested_OneWithDataOneNot_AndBrowserArchivingDisabled()
+    {
+        $idSegment = API::getInstance()->add('testsegment', self::TEST_SEGMENT, $idSite = false, $autoArchive = true);
+
+        $storedSegment = API::getInstance()->get($idSegment);
+        $this->assertNotEmpty($storedSegment);
+
+        Rules::setBrowserTriggerArchiving(false);
+
+        $segments = Rules::getSegmentsToProcess([self::$fixture->idSite]);
+        $this->assertContains(self::TEST_SEGMENT, $segments);
+
+        $this->runAnyApiTest('VisitsSummary.get', 'noLogDataSegmentUnprocessedMultiSite', [
+            'idSite' => 'all',
+            'date' => Date::factory(self::$fixture->dateTime)->toString(),
+            'period' => 'week',
+            'segment' => self::TEST_SEGMENT,
+        ]);
+    }
+
+    public static function getOutputPrefix()
+    {
+        return '';
+    }
+
+    public static function getPathToTestDirectory()
+    {
+        return dirname(__FILE__);
+    }
+
+    public function provideContainerConfig()
+    {
+        return [
+            Config::class => \DI\decorate(function (Config $previous) {
+                $previous->General['browser_archiving_disabled_enforce'] = 1;
+                return $previous;
+            }),
+        ];
+    }
+
+    private function clearLogData()
+    {
+        Db::query('TRUNCATE ' . Common::prefixTable('log_visit'));
+        Db::query('TRUNCATE ' . Common::prefixTable('log_link_visit_action'));
+        Db::query('TRUNCATE ' . Common::prefixTable('log_conversion'));
+    }
+}
+
+UnprocessedSegmentsTest::$fixture = new OneVisitorTwoVisits();
diff --git a/plugins/SegmentEditor/tests/System/expected/test___VisitsSummary.get_autoArchiveSegmentNoDataPreprocessed.xml b/plugins/SegmentEditor/tests/System/expected/test___VisitsSummary.get_autoArchiveSegmentNoDataPreprocessed.xml
new file mode 100644
index 0000000000..32b66284be
--- /dev/null
+++ b/plugins/SegmentEditor/tests/System/expected/test___VisitsSummary.get_autoArchiveSegmentNoDataPreprocessed.xml
@@ -0,0 +1,14 @@
+<?xml version="1.0" encoding="utf-8" ?>
+<result>
+	<nb_uniq_visitors>0</nb_uniq_visitors>
+	<nb_users>0</nb_users>
+	<nb_visits>0</nb_visits>
+	<nb_actions>0</nb_actions>
+	<nb_visits_converted>0</nb_visits_converted>
+	<bounce_count>0</bounce_count>
+	<sum_visit_length>0</sum_visit_length>
+	<max_actions>0</max_actions>
+	<bounce_rate>0%</bounce_rate>
+	<nb_actions_per_visit>0</nb_actions_per_visit>
+	<avg_time_on_site>0</avg_time_on_site>
+</result>
\ No newline at end of file
diff --git a/plugins/SegmentEditor/tests/System/expected/test___VisitsSummary.get_autoArchiveSegmentPreprocessed.xml b/plugins/SegmentEditor/tests/System/expected/test___VisitsSummary.get_autoArchiveSegmentPreprocessed.xml
new file mode 100644
index 0000000000..c8a2c198e8
--- /dev/null
+++ b/plugins/SegmentEditor/tests/System/expected/test___VisitsSummary.get_autoArchiveSegmentPreprocessed.xml
@@ -0,0 +1,14 @@
+<?xml version="1.0" encoding="utf-8" ?>
+<result>
+	<nb_uniq_visitors>1</nb_uniq_visitors>
+	<nb_users>0</nb_users>
+	<nb_visits>2</nb_visits>
+	<nb_actions>8</nb_actions>
+	<nb_visits_converted>2</nb_visits_converted>
+	<bounce_count>1</bounce_count>
+	<sum_visit_length>1621</sum_visit_length>
+	<max_actions>7</max_actions>
+	<bounce_rate>50%</bounce_rate>
+	<nb_actions_per_visit>4</nb_actions_per_visit>
+	<avg_time_on_site>811</avg_time_on_site>
+</result>
\ No newline at end of file
diff --git a/plugins/SegmentEditor/tests/System/expected/test___VisitsSummary.get_autoArchiveSegmentUnprocessed.xml b/plugins/SegmentEditor/tests/System/expected/test___VisitsSummary.get_autoArchiveSegmentUnprocessed.xml
new file mode 100644
index 0000000000..57b75e01db
--- /dev/null
+++ b/plugins/SegmentEditor/tests/System/expected/test___VisitsSummary.get_autoArchiveSegmentUnprocessed.xml
@@ -0,0 +1,4 @@
+<?xml version="1.0" encoding="utf-8" ?>
+<result>
+	<error message="These reports have no data, because the Segment you requested (testsegment) has not yet been processed by the system. Data for this Segment should become available in a few hours when processing completes. (If it does not, there may be a problem.) Please note that you can test whether your segment will work without having to wait for it to be processed by using the Live.getLastVisitsDetails API. When using this API method, you will see which users and actions were matched by your &amp;segment= parameter. This can help you confirm your Segment matches the users and actions you expect it to." />
+</result>
\ No newline at end of file
diff --git a/plugins/SegmentEditor/tests/System/expected/test___VisitsSummary.get_customSegmentPreprocessed.xml b/plugins/SegmentEditor/tests/System/expected/test___VisitsSummary.get_customSegmentPreprocessed.xml
new file mode 100644
index 0000000000..c8a2c198e8
--- /dev/null
+++ b/plugins/SegmentEditor/tests/System/expected/test___VisitsSummary.get_customSegmentPreprocessed.xml
@@ -0,0 +1,14 @@
+<?xml version="1.0" encoding="utf-8" ?>
+<result>
+	<nb_uniq_visitors>1</nb_uniq_visitors>
+	<nb_users>0</nb_users>
+	<nb_visits>2</nb_visits>
+	<nb_actions>8</nb_actions>
+	<nb_visits_converted>2</nb_visits_converted>
+	<bounce_count>1</bounce_count>
+	<sum_visit_length>1621</sum_visit_length>
+	<max_actions>7</max_actions>
+	<bounce_rate>50%</bounce_rate>
+	<nb_actions_per_visit>4</nb_actions_per_visit>
+	<avg_time_on_site>811</avg_time_on_site>
+</result>
\ No newline at end of file
diff --git a/plugins/SegmentEditor/tests/System/expected/test___VisitsSummary.get_customSegmentUnprocessed.xml b/plugins/SegmentEditor/tests/System/expected/test___VisitsSummary.get_customSegmentUnprocessed.xml
new file mode 100644
index 0000000000..ed4fec1ebd
--- /dev/null
+++ b/plugins/SegmentEditor/tests/System/expected/test___VisitsSummary.get_customSegmentUnprocessed.xml
@@ -0,0 +1,4 @@
+<?xml version="1.0" encoding="utf-8" ?>
+<result>
+	<error message="The Segment you requested has not yet been created in the Segment Editor and so the report data has not been pre-processed. To see data for this segment, you must go to Matomo and create this segment manually in the Segment Editor. (Alternatively, you can create a new segment programatically using the SegmentEditor.add API method). Once created the segment in the editor (or via API), this error message will disappear and within a few hours you will see your segmented report data, after the segment data has been pre-processed. (If it does not, there may be a problem.) Please note that you can test whether your segment will work without having to wait for it to be processed by using the Live.getLastVisitsDetails API. When using this API method, you will see which users and actions were matched by your &amp;segment= parameter. This can help you confirm your Segment matches the users and actions you expect it to." />
+</result>
\ No newline at end of file
diff --git a/plugins/SegmentEditor/tests/System/expected/test___VisitsSummary.get_noLogDataSegmentUnprocessed.xml b/plugins/SegmentEditor/tests/System/expected/test___VisitsSummary.get_noLogDataSegmentUnprocessed.xml
new file mode 100644
index 0000000000..32b66284be
--- /dev/null
+++ b/plugins/SegmentEditor/tests/System/expected/test___VisitsSummary.get_noLogDataSegmentUnprocessed.xml
@@ -0,0 +1,14 @@
+<?xml version="1.0" encoding="utf-8" ?>
+<result>
+	<nb_uniq_visitors>0</nb_uniq_visitors>
+	<nb_users>0</nb_users>
+	<nb_visits>0</nb_visits>
+	<nb_actions>0</nb_actions>
+	<nb_visits_converted>0</nb_visits_converted>
+	<bounce_count>0</bounce_count>
+	<sum_visit_length>0</sum_visit_length>
+	<max_actions>0</max_actions>
+	<bounce_rate>0%</bounce_rate>
+	<nb_actions_per_visit>0</nb_actions_per_visit>
+	<avg_time_on_site>0</avg_time_on_site>
+</result>
\ No newline at end of file
diff --git a/plugins/SegmentEditor/tests/System/expected/test___VisitsSummary.get_noLogDataSegmentUnprocessedMultiSite.xml b/plugins/SegmentEditor/tests/System/expected/test___VisitsSummary.get_noLogDataSegmentUnprocessedMultiSite.xml
new file mode 100644
index 0000000000..dd52dc4ff4
--- /dev/null
+++ b/plugins/SegmentEditor/tests/System/expected/test___VisitsSummary.get_noLogDataSegmentUnprocessedMultiSite.xml
@@ -0,0 +1,6 @@
+<?xml version="1.0" encoding="utf-8" ?>
+<results>
+	<result idSite="1" />
+	<result idSite="2" />
+	<result idSite="3" />
+</results>
\ No newline at end of file
diff --git a/plugins/SegmentEditor/tests/System/expected/test___VisitsSummary.get_realTimeSegmentUnprocessed.xml b/plugins/SegmentEditor/tests/System/expected/test___VisitsSummary.get_realTimeSegmentUnprocessed.xml
new file mode 100644
index 0000000000..0dcab5ab6d
--- /dev/null
+++ b/plugins/SegmentEditor/tests/System/expected/test___VisitsSummary.get_realTimeSegmentUnprocessed.xml
@@ -0,0 +1,4 @@
+<?xml version="1.0" encoding="utf-8" ?>
+<result>
+	<error message="The Segment 'testsegment' is set to 'segmented reports are processed in real time' but Matomo is not currently configured to process segmented reports in API requests. To see data for this report in the future, you will need to edit your segment and choose the option labeled 'segmented reports are pre-processed (faster, requires cron)'. Then after a few hours your segment data should become available through the API. (If it does not, there may be a problem.)" />
+</result>
\ No newline at end of file