Fixes #7223, add new INI option [General] process_new_segments_from that controls the start date of archiving for segments that have not been archived before. If a creation date cannot be found, now is used.

5 files changed, 353 insertions, 4 deletions

diff --git a/config/global.ini.php b/config/global.ini.php
index de3e80ca72..c23522ab53 100644
--- a/config/global.ini.php
+++ b/config/global.ini.php
@@ -220,6 +220,13 @@ adding_segment_requires_access = "view"
 ; on Piwik performance.
 allow_adding_segments_for_all_websites = 1
 
+; When archiving segments for the first time, this determines the oldest date that will be archived.
+; This option can be used to avoid archiving (for isntance) the lastN years for every new segment.
+; Valid option values include: beginning_of_time (start date of archiving will not be changed)
+;                              creation_time (start date of archiving will be the creation date of the segment)
+;                              lastN where N is an integer (start date of archiving will be N days before the creation date)
+process_new_segments_from = beginning_of_time
+
 ; this action name is used when the URL ends with a slash /
 ; it is useful to have an actual string to write in the UI
 action_default_name = index
diff --git a/core/CronArchive.php b/core/CronArchive.php
index 0ff2ee8242..7a4c0bf972 100644
--- a/core/CronArchive.php
+++ b/core/CronArchive.php
@@ -10,6 +10,7 @@ namespace Piwik;
 
 use Exception;
 use Piwik\ArchiveProcessor\Rules;
+use Piwik\Container\StaticContainer;
 use Piwik\CronArchive\FixedSiteIds;
 use Piwik\CronArchive\SharedSiteIds;
 use Piwik\Archive\ArchiveInvalidator;
@@ -17,6 +18,7 @@ use Piwik\Exception\UnexpectedWebsiteFoundException;
 use Piwik\Metrics\Formatter;
 use Piwik\Period\Factory as PeriodFactory;
 use Piwik\CronArchive\SitesToReprocessDistributedList;
+use Piwik\CronArchive\SegmentArchivingRequestUrlProvider;
 use Piwik\Plugins\CoreAdminHome\API as CoreAdminHomeAPI;
 use Piwik\Plugins\SitesManager\API as APISitesManager;
 use Piwik\Plugins\UsersManager\API as APIUsersManager;
@@ -199,6 +201,11 @@ class CronArchive
     private $formatter;
 
     /**
+     * @var SegmentArchivingRequestUrlProvider
+     */
+    private $segmentArchivingRequestUrlProvider;
+
+    /**
      * Returns the option name of the option that stores the time core:archive was last executed.
      *
      * @param int $idSite
@@ -217,14 +224,19 @@ class CronArchive
      *                               we determine it using the current request information.
      *
      *                               If invoked via the command line, $piwikUrl cannot be false.
+     * @param string|null $processNewSegmentsFrom When to archive new segments from. See [General] process_new_segments_from
+     *                                            for possible values.
      */
-    public function __construct($piwikUrl = false)
+    public function __construct($piwikUrl = false, $processNewSegmentsFrom = null)
     {
         $this->formatter = new Formatter();
 
         $this->initPiwikHost($piwikUrl);
         $this->initCore();
         $this->initTokenAuth();
+
+        $processNewSegmentsFrom = $processNewSegmentsFrom ?: StaticContainer::get('ini.General.process_new_segments_from');
+        $this->segmentArchivingRequestUrlProvider = new SegmentArchivingRequestUrlProvider($this->piwikUrl, $this->token_auth, $processNewSegmentsFrom);
     }
 
     /**
@@ -771,7 +783,9 @@ class CronArchive
         }
 
         foreach ($this->getSegmentsForSite($idSite) as $segment) {
-            $urlWithSegment = $url . '&segment=' . urlencode($segment);
+            $urlWithSegment = $this->segmentArchivingRequestUrlProvider->getUrlToArchiveSegment($idSite, $period, $date, $segment);
+            $urlWithSegment .= self::APPEND_TO_API_REQUEST;
+
             $urls[] = $urlWithSegment;
             $this->requests++;
         }
@@ -1531,5 +1545,4 @@ class CronArchive
 
         return $customDateRangesToProcessForSites;
     }
-
-}
+}
\ No newline at end of file
diff --git a/core/CronArchive/SegmentArchivingRequestUrlProvider.php b/core/CronArchive/SegmentArchivingRequestUrlProvider.php
new file mode 100644
index 0000000000..6b3828af06
--- /dev/null
+++ b/core/CronArchive/SegmentArchivingRequestUrlProvider.php
@@ -0,0 +1,143 @@
+<?php
+/**
+ * Piwik - free/libre analytics platform
+ *
+ * @link http://piwik.org
+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
+ */
+namespace Piwik\CronArchive;
+
+use Piwik\Cache\Cache;
+use Piwik\Cache\Transient;
+use Piwik\Date;
+use Piwik\Period\Factory as PeriodFactory;
+use Piwik\Period\Range;
+use Piwik\Plugins\SegmentEditor\Model;
+
+/**
+ * Provides URLs that initiate archiving during cron archiving for segments.
+ *
+ * Handles the `[General] process_new_segments_from` INI option.
+ */
+class SegmentArchivingRequestUrlProvider
+{
+    const BEGINNING_OF_TIME = 'beginning_of_time';
+    const CREATION_TIME = 'creation_time';
+
+    /**
+     * @var Model
+     */
+    private $segmentEditorModel;
+
+    /**
+     * @var Cache
+     */
+    private $segmentListCache;
+
+    /**
+     * @var Date
+     */
+    private $now;
+
+    private $baseUrl;
+    private $tokenAuth;
+    private $processNewSegmentsFrom;
+
+    public function __construct($baseUrl, $tokenAuth, $processNewSegmentsFrom, Model $segmentEditorModel = null, Cache $segmentListCache = null, Date $now = null)
+    {
+        $this->baseUrl = $baseUrl;
+        $this->tokenAuth = $tokenAuth;
+        $this->processNewSegmentsFrom = $processNewSegmentsFrom;
+        $this->segmentEditorModel = $segmentEditorModel ?: new Model();
+        $this->segmentListCache = $segmentListCache ?: new Transient();
+        $this->now = $now ?: Date::factory('now');
+    }
+
+    public function getUrlToArchiveSegment($idSite, $period, $date, $segment)
+    {
+        $date = $this->getModifiedSegmentArchvingDateRange($idSite, $period, $date, $segment);
+
+        return $this->baseUrl
+            . "?module=API&method=API.get&idSite=$idSite&period=$period&date=" . $date . "&format=php&token_auth=" . $this->tokenAuth
+            . '&segment=' . urlencode($segment);
+    }
+
+    private function getModifiedSegmentArchvingDateRange($idSite, $period, $date, $segment)
+    {
+        $segmentCreatedTime = $this->getCreatedTimeOfSegment($idSite, $segment);
+        if (empty($segmentCreatedTime)) {
+            return $date;
+        }
+
+        $oldestDateToProcessForNewSegment = $this->getOldestDateToProcessForNewSegment($segmentCreatedTime);
+        if (empty($oldestDateToProcessForNewSegment)) {
+            return $date;
+        }
+
+        // if the start date for the archiving request is before the minimum date allowed for processing this segment,
+        // use the minimum allowed date as the start date
+        $periodObj = PeriodFactory::build($period, $date);
+        if ($periodObj->getDateStart()->getTimestamp() < $oldestDateToProcessForNewSegment->getTimestamp()) {
+            $date = $oldestDateToProcessForNewSegment->toString().','.$periodObj->getDateEnd();
+        }
+
+        return $date;
+    }
+
+    private function getOldestDateToProcessForNewSegment(Date $segmentCreatedTime)
+    {
+        if ($this->processNewSegmentsFrom == self::CREATION_TIME) {
+            return $segmentCreatedTime;
+        } else if (preg_match("/^last([0-9]+)$/", $this->processNewSegmentsFrom, $matches)) {
+            $lastN = $matches[1];
+
+            list($lastDate, $lastPeriod) = Range::getDateXPeriodsAgo($lastN, $segmentCreatedTime, 'day');
+
+            return Date::factory($lastDate);
+        } else {
+            return null;
+        }
+    }
+
+    private function getCreatedTimeOfSegment($idSite, $segmentDefinition)
+    {
+        $segments = $this->getAllSegments();
+
+        $earliestCreatedTime = $this->now;
+        foreach ($segments as $segment) {
+            if (empty($segment['ts_created'])
+                || empty($segment['definition'])
+                || !isset($segment['enable_only_idsite'])
+            ) {
+                continue;
+            }
+
+            if ($this->isSegmentForSite($segment, $idSite)
+                && $segment['definition'] == $segmentDefinition
+            ) {
+                $createdTime = Date::factory($segment['ts_created']);
+                if ($createdTime->getTimestamp() < $earliestCreatedTime->getTimestamp()) {
+                    $earliestCreatedTime = $createdTime;
+                }
+            }
+        }
+        return $earliestCreatedTime;
+    }
+
+    private function getAllSegments()
+    {
+        if (!$this->segmentListCache->contains('all')) {
+            $segments = $this->segmentEditorModel->getAllSegmentsAndIgnoreVisibility();
+
+            $this->segmentListCache->save('all', $segments);
+        }
+
+        return $this->segmentListCache->fetch('all');
+    }
+
+    private function isSegmentForSite($segment, $idSite)
+    {
+        return $segment['enable_only_idsite'] == 0
+            || $segment['enable_only_idsite'] == $idSite;
+    }
+}
\ No newline at end of file
diff --git a/plugins/SegmentEditor/Model.php b/plugins/SegmentEditor/Model.php
index 8bfcc957ae..2192e3ca58 100644
--- a/plugins/SegmentEditor/Model.php
+++ b/plugins/SegmentEditor/Model.php
@@ -26,6 +26,21 @@ class Model
     }
 
     /**
+     * Returns all stored segments that haven't been deleted. Ignores the site the segments are enabled
+     * for and whether to auto archive or not.
+     *
+     * @return array
+     */
+    public function getAllSegmentsAndIgnoreVisibility()
+    {
+        $sql = "SELECT * FROM " . $this->table . " WHERE deleted = 0";
+
+        $segments = $this->getDb()->fetchAll($sql);
+
+        return $segments;
+    }
+
+    /**
      * Returns all stored segments.
      *
      * @param bool|int $idSite Whether to return stored segments for a specific idSite, or segments that are available
diff --git a/tests/PHPUnit/Unit/CronArchive/SegmentArchivingRequestUrlProviderTest.php b/tests/PHPUnit/Unit/CronArchive/SegmentArchivingRequestUrlProviderTest.php
new file mode 100644
index 0000000000..7d50d3f17f
--- /dev/null
+++ b/tests/PHPUnit/Unit/CronArchive/SegmentArchivingRequestUrlProviderTest.php
@@ -0,0 +1,171 @@
+<?php
+/**
+ * Piwik - free/libre analytics platform
+ *
+ * @link http://piwik.org
+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
+ */
+namespace Piwik\Tests\Unit\CronArchive;
+
+use Piwik\Date;
+use Piwik\CronArchive\SegmentArchivingRequestUrlProvider;
+
+/**
+ * @group Core
+ */
+class SegmentArchivingRequestUrlProviderTest extends \PHPUnit_Framework_TestCase
+{
+    const BASE_URL = 'http://base/';
+    const TOKEN_AUTH = 'tokenauth';
+    const TEST_NOW = '2015-03-01';
+
+    private $mockSegmentEntries;
+
+    public function setUp()
+    {
+        $this->mockSegmentEntries = array(
+            array(
+                'ts_created' => '2014-01-01',
+                'definition' => 'browserName==FF',
+                'enable_only_idsite' => 1
+            ),
+
+            array(
+                'ts_created' => '2014-01-01',
+                'definition' => 'countryCode==us',
+                'enable_only_idsite' => 1
+            ),
+
+            array(
+                'ts_created' => '2012-01-01',
+                'definition' => 'countryCode==us',
+                'enable_only_idsite' => 1
+            ),
+
+            array(
+                'ts_created' => '2014-01-01',
+                'definition' => 'countryCode==ca',
+                'enable_only_idsite' => 2
+            ),
+
+            array(
+                'ts_created' => '2012-01-01',
+                'definition' => 'countryCode==ca',
+                'enable_only_idsite' => 2
+            ),
+
+            array(
+                'ts_created' => '2011-01-01',
+                'definition' => 'countryCode==ca',
+                'enable_only_idsite' => 0
+            )
+        );
+    }
+
+    /**
+     * @dataProvider getUrlToArchiveSegmentTestData
+     */
+    public function test_getUrlToArchiveSegment_CorrectlyModifiesDateInOutputUrl($processNewSegmentsFrom, $idSite, $date, $period, $segment, $expected)
+    {
+        $urlProvider = $this->createUrlProviderToTest($processNewSegmentsFrom);
+
+        $actual = $urlProvider->getUrlToArchiveSegment($idSite, $period, $date, $segment);
+        $this->assertEquals($expected, $actual);
+    }
+
+    public function getUrlToArchiveSegmentTestData()
+    {
+        $dateRange = '2010-02-01,' . self::TEST_NOW;
+        return array(
+            array( // test beginning_of_time does not modify date
+                'beginning_of_time',
+                1,
+                $dateRange,
+                'week',
+                'browserName==FF',
+                "http://base/?module=API&method=API.get&idSite=1&period=week&date=$dateRange&format=php&token_auth=tokenauth&segment=" . urlencode('browserName==FF')
+            ),
+
+            array( // test garbage string does not modify date
+                'salkdfjsdfl',
+                1,
+                $dateRange,
+                'week',
+                'browserName==FF',
+                "http://base/?module=API&method=API.get&idSite=1&period=week&date=$dateRange&format=php&token_auth=tokenauth&segment=" . urlencode('browserName==FF')
+            ),
+
+            array( // test creation_time uses creation time of segment
+                'creation_time',
+                1,
+                $dateRange,
+                'week',
+                'browserName==FF',
+                'http://base/?module=API&method=API.get&idSite=1&period=week&date=2014-01-01,2015-03-01&format=php&token_auth=tokenauth&segment=' . urlencode('browserName==FF')
+            ),
+
+            array( // test creation_time uses earliest time of segment if multiple match (multiple for site)
+                'creation_time',
+                1,
+                $dateRange,
+                'week',
+                'countryCode==us',
+                'http://base/?module=API&method=API.get&idSite=1&period=week&date=2012-01-01,2015-03-01&format=php&token_auth=tokenauth&segment=' . urlencode('countryCode==us')
+            ),
+
+            array( // test creation_time uses earliest time of segment if multiple match (multiple for site + one for all)
+                'creation_time',
+                2,
+                $dateRange,
+                'week',
+                'countryCode==ca',
+                'http://base/?module=API&method=API.get&idSite=2&period=week&date=2011-01-01,2015-03-01&format=php&token_auth=tokenauth&segment=' . urlencode('countryCode==ca')
+            ),
+
+            array( // test 'now' is used if no site matches (testing w/o any segments)
+                'creation_time',
+                1,
+                $dateRange,
+                'week',
+                'pageTitle==abc',
+                'http://base/?module=API&method=API.get&idSite=1&period=week&date=2015-03-01,2015-03-01&format=php&token_auth=tokenauth&segment=' . urlencode('pageTitle==abc')
+            ),
+
+            array( // test 'now' is used if no site matches (testing w/ segment for another site)
+                'creation_time',
+                3,
+                $dateRange,
+                'week',
+                'countryCode==us',
+                'http://base/?module=API&method=API.get&idSite=3&period=week&date=2015-03-01,2015-03-01&format=php&token_auth=tokenauth&segment=' . urlencode('countryCode==us')
+            ),
+
+            array( // test lastN rewinds created date by N days
+                'last10',
+                1,
+                $dateRange,
+                'week',
+                'countryCode==us',
+                'http://base/?module=API&method=API.get&idSite=1&period=week&date=2011-12-22,2015-03-01&format=php&token_auth=tokenauth&segment=' . urlencode('countryCode==us')
+            ),
+
+            array( // test lastN rewinds now by N days (testing w/ no found segment)
+                'last10',
+                3,
+                $dateRange,
+                'week',
+                'countryCode==us',
+                'http://base/?module=API&method=API.get&idSite=3&period=week&date=2015-02-19,2015-03-01&format=php&token_auth=tokenauth&segment=' . urlencode('countryCode==us')
+            ),
+        );
+    }
+
+    private function createUrlProviderToTest($processNewSegmentsFrom)
+    {
+        $mockSegmentEditorModel = $this->getMock('Piwik\Plugins\SegmentEditor\Model', array('getAllSegmentsAndIgnoreVisibility'));
+        $mockSegmentEditorModel->expects($this->any())->method('getAllSegmentsAndIgnoreVisibility')->will($this->returnValue($this->mockSegmentEntries));
+
+        return new SegmentArchivingRequestUrlProvider(self::BASE_URL, self::TOKEN_AUTH, $processNewSegmentsFrom,
+            $mockSegmentEditorModel, null, Date::factory(self::TEST_NOW));
+    }
+}
\ No newline at end of file