disable cron task when segment is deleted (#18383)

* Update Segment.php

add function

* update segment on cron when is deleted

update segment on cron when is deleted

* Update QueueConsumer.php

change to check segement

* Update QueueConsumer.php

remove used function

* Update QueueConsumer.php

update Segment availbel

* Update QueueConsumer.php

update queue

* Update QueueConsumer.php

trigger update

* update

update

* Update QueueConsumer.php

remove plugin check

* Revert "Update QueueConsumer.php"

This reverts commit e059f4b7f3eef4e0807987e3f0bb09fc12cd0bf8.

* Update QueueConsumer.php

remove the right part

* Update core/CronArchive/QueueConsumer.php

Co-authored-by: Stefan Giehl <stefan@matomo.org>

* try simple solution

try simple solution

* Adds invalid segment test case

* fix segment errors

fix segment errors

* Update CronArchiveInvalidSegmentTest.php

update tests

* Update CronArchiveInvalidSegmentTest.php

update tests

* update tests

update tests

* update tests and words

update tests and words

* update log info and tests

update log info and tests

Co-authored-by: Stefan Giehl <stefan@matomo.org>

2 files changed, 146 insertions, 4 deletions

diff --git a/core/CronArchive.php b/core/CronArchive.php
index 4a8204ef02..dcde1460fc 100644
--- a/core/CronArchive.php
+++ b/core/CronArchive.php
@@ -497,7 +497,7 @@ class CronArchive
 
         foreach ($urls as $index => $url) {
             $content = array_key_exists($index, $responses) ? $responses[$index] : null;
-            $this->checkResponse($content, $url);
+            $checkInvalid = $this->checkResponse($content, $url);
 
             $stats = json_decode($content, $assoc = true);
             if (!is_array($stats)) {
@@ -514,7 +514,10 @@ class CronArchive
 
             $visitsForPeriod = $this->getVisitsFromApiResponse($stats);
 
-            $this->logArchiveJobFinished($url, $timers[$index], $visitsForPeriod, $archivesBeingQueried[$index]['plugin'], $archivesBeingQueried[$index]['report']);
+
+            $this->logArchiveJobFinished($url, $timers[$index], $visitsForPeriod,
+              $archivesBeingQueried[$index]['plugin'], $archivesBeingQueried[$index]['report'], !$checkInvalid);
+
 
             $this->deleteInvalidatedArchives($archivesBeingQueried[$index]);
 
@@ -572,12 +575,14 @@ class CronArchive
         return [$url, $segment, $plugin];
     }
 
-    private function logArchiveJobFinished($url, $timer, $visits, $plugin = null, $report = null)
+    private function logArchiveJobFinished($url, $timer, $visits, $plugin = null, $report = null, $wasSkipped = null)
     {
         $params = UrlHelper::getArrayFromQueryString($url);
         $visits = (int) $visits;
 
-        $this->logger->info("Archived website id {$params['idSite']}, period = {$params['period']}, date = "
+        $message = $wasSkipped ? "Skipped Archiving website" : "Archived website";
+
+        $this->logger->info($message." id {$params['idSite']}, period = {$params['period']}, date = "
             . "{$params['date']}, segment = '" . (isset($params['segment']) ? urldecode(urldecode($params['segment'])) : '') . "', "
             . ($plugin ? "plugin = $plugin, " : "") . ($report ? "report = $report, " : "") . "$visits visits found. $timer");
     }
@@ -715,6 +720,12 @@ class CronArchive
 
     private function logNetworkError($url, $response)
     {
+
+        if (preg_match("/Segment (.*?) is not a supported segment/i", $response, $match)) {
+            $this->logger->info($match[0]);
+            return false;
+        }
+
         $message = "Got invalid response from API request: $url. ";
         if (empty($response)) {
             $message .= "The response was empty. This usually means a server error. A solution to this error is generally to increase the value of 'memory_limit' in your php.ini file. ";
@@ -919,6 +930,11 @@ class CronArchive
             }
 
             foreach ($this->segmentArchiving->getAllSegmentsToArchive($idSite) as $segmentDefinition) {
+
+               // check if the segment is available
+                if (!$this->isSegmentAvailable($segmentDefinition, [$idSite])) {
+                    continue;
+                }
                 $params = new Parameters(new Site($idSite), $periodObj, new Segment($segmentDefinition, [$idSite], $periodObj->getDateStart(), $periodObj->getDateEnd()));
                 if ($this->canWeSkipInvalidatingBecauseThereIsAUsablePeriod($params, $doNotIncludeTtlInExistingArchiveCheck)) {
                     $this->logger->debug('  Found usable archive for {archive}, skipping invalidation.', ['archive' => $params]);
@@ -947,6 +963,24 @@ class CronArchive
         }
     }
 
+
+    /**
+     * check if segments that contain dimensions that don't exist anymore
+     * @param $segmentDefinition
+     * @param $idSites
+     * @return bool
+     */
+    protected function isSegmentAvailable($segmentDefinition, $idSites)
+    {
+        try {
+            new Segment($segmentDefinition, $idSites);
+        } catch (\Exception $e) {
+            $this->logger->info("Segment '".$segmentDefinition."' is not a supported segment");
+            return false;
+        }
+        return true;
+    }
+
     /**
      * Returns true if there is an existing valid period we can use, or false if there isn't and the invalidation should go through.
      *
diff --git a/tests/PHPUnit/Integration/CronArchiveInvalidSegmentTest.php b/tests/PHPUnit/Integration/CronArchiveInvalidSegmentTest.php
new file mode 100644
index 0000000000..2752244c04
--- /dev/null
+++ b/tests/PHPUnit/Integration/CronArchiveInvalidSegmentTest.php
@@ -0,0 +1,108 @@
+<?php
+/**
+ * Matomo - free/libre analytics platform
+ *
+ * @link https://matomo.org
+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
+ */
+
+namespace Piwik\Tests\Integration;
+
+use Piwik\ArchiveProcessor\Rules;
+use Piwik\CronArchive;
+use Piwik\Date;
+use Piwik\Plugin\Manager;
+use Piwik\Tests\Framework\Fixture;
+use Piwik\Tests\Framework\Mock\FakeLogger;
+use Piwik\Tests\Framework\TestCase\IntegrationTestCase;
+use Piwik\Plugins\SegmentEditor\API as SegmentAPI;
+use Piwik\Version;
+
+/**
+ * @group Archiver
+ * @group CronArchive
+ */
+class CronArchiveInvalidSegmentTest extends IntegrationTestCase
+{
+    /** @var FakeLogger  */
+    private $logger;
+
+    public function setUp(): void
+    {
+        \Piwik\Tests\Framework\Mock\FakeCliMulti::$specifiedResults = array(
+          '/method=API.get/' => json_encode(array(array('nb_visits' => 1)))
+        );
+
+        Fixture::createWebsite('2014-12-12 00:01:02');
+
+        Manager::getInstance()->activatePlugin('UserLanguage');
+
+        Rules::setBrowserTriggerArchiving(false);
+        SegmentAPI::getInstance()->add('languageSegment', 'languageCode==fr', 1, true, true);
+
+        $tracker = Fixture::getTracker(1, '2019-12-12 02:03:00');
+        $tracker->setUrl('http://someurl.com');
+        Fixture::checkResponse($tracker->doTrackPageView('abcdefg'));
+
+        $tracker->setForceVisitDateTime('2019-12-11 03:04:05');
+        $tracker->setUrl('http://someurl.com/2');
+        Fixture::checkResponse($tracker->doTrackPageView('abcdefg2'));
+
+        $tracker->setForceVisitDateTime('2019-12-10 03:04:05');
+        $tracker->setUrl('http://someurl.com/3');
+        Fixture::checkResponse($tracker->doTrackPageView('abcdefg3'));
+
+        $tracker->setForceVisitDateTime('2019-12-02 03:04:05');
+        $tracker->setUrl('http://someurl.com/4');
+        Fixture::checkResponse($tracker->doTrackPageView('abcdefg4'));
+
+        $this->logger = new FakeLogger();
+
+
+    }
+
+    public function test_output_invalidSegment()
+    {
+        $archiver = new CronArchive($this->logger);
+
+        $archiver->init();
+        $archiver->run();
+
+        $this->assertStringNotContainsStringIgnoringCase('Got invalid response from API request', $this->logger->output);
+        $this->assertStringContainsString("Skipped Archiving website id 1", $this->logger->output);
+        $this->assertStringContainsString('no error', $this->logger->output);
+    }
+
+    public function test_output_invalidSegment_whenPluginIsNotActive()
+    {
+        Manager::getInstance()->deactivatePlugin('UserLanguage');
+
+        $archiver = new CronArchive($this->logger);
+
+        $archiver->init();
+        $archiver->run();
+
+        $this->assertStringNotContainsStringIgnoringCase('Got invalid response from API request', $this->logger->output);
+        $this->assertStringContainsString("Segment 'languageCode==fr' is not a supported segment", $this->logger->output);
+        $this->assertStringContainsString('no error', $this->logger->output);
+    }
+
+    public function provideContainerConfig()
+    {
+        Date::$now = strtotime('2020-02-03 04:05:06');
+
+        return array(
+            'observers.global' => \DI\add([
+                ['API.CoreAdminHome.archiveReports', \DI\value(function (&$result) {
+                    Manager::getInstance()->deactivatePlugin('UserLanguage');
+                })]
+            ]),
+        );
+    }
+
+    protected static function configureFixture($fixture)
+    {
+        parent::configureFixture($fixture);
+        $fixture->createSuperUser = true;
+    }
+}