diff options
author | diosmosis <benaka@piwik.pro> | 2015-03-17 04:44:25 +0300 |
---|---|---|
committer | diosmosis <benaka@piwik.pro> | 2015-03-18 08:30:28 +0300 |
commit | acd924de32bb40be3305027b812f83ba7310bf6f (patch) | |
tree | ddf6a1bfcd4a7a8b81d06fc37741e01fc2ec1324 | |
parent | 42807de32ab64bef2c18eb1f6a6d0a0358f8c57f (diff) |
Fixes #7223, add new INI option [General] process_new_segments_from that controls the start date of archiving for segments that have not been archived before. If a creation date cannot be found, now is used.
-rw-r--r-- | config/global.ini.php | 7 | ||||
-rw-r--r-- | core/CronArchive.php | 21 | ||||
-rw-r--r-- | core/CronArchive/SegmentArchivingRequestUrlProvider.php | 143 | ||||
-rw-r--r-- | plugins/SegmentEditor/Model.php | 15 | ||||
-rw-r--r-- | tests/PHPUnit/Unit/CronArchive/SegmentArchivingRequestUrlProviderTest.php | 171 |
5 files changed, 353 insertions, 4 deletions
diff --git a/config/global.ini.php b/config/global.ini.php index de3e80ca72..c23522ab53 100644 --- a/config/global.ini.php +++ b/config/global.ini.php @@ -220,6 +220,13 @@ adding_segment_requires_access = "view" ; on Piwik performance. allow_adding_segments_for_all_websites = 1 +; When archiving segments for the first time, this determines the oldest date that will be archived. +; This option can be used to avoid archiving (for isntance) the lastN years for every new segment. +; Valid option values include: beginning_of_time (start date of archiving will not be changed) +; creation_time (start date of archiving will be the creation date of the segment) +; lastN where N is an integer (start date of archiving will be N days before the creation date) +process_new_segments_from = beginning_of_time + ; this action name is used when the URL ends with a slash / ; it is useful to have an actual string to write in the UI action_default_name = index diff --git a/core/CronArchive.php b/core/CronArchive.php index 0ff2ee8242..7a4c0bf972 100644 --- a/core/CronArchive.php +++ b/core/CronArchive.php @@ -10,6 +10,7 @@ namespace Piwik; use Exception; use Piwik\ArchiveProcessor\Rules; +use Piwik\Container\StaticContainer; use Piwik\CronArchive\FixedSiteIds; use Piwik\CronArchive\SharedSiteIds; use Piwik\Archive\ArchiveInvalidator; @@ -17,6 +18,7 @@ use Piwik\Exception\UnexpectedWebsiteFoundException; use Piwik\Metrics\Formatter; use Piwik\Period\Factory as PeriodFactory; use Piwik\CronArchive\SitesToReprocessDistributedList; +use Piwik\CronArchive\SegmentArchivingRequestUrlProvider; use Piwik\Plugins\CoreAdminHome\API as CoreAdminHomeAPI; use Piwik\Plugins\SitesManager\API as APISitesManager; use Piwik\Plugins\UsersManager\API as APIUsersManager; @@ -199,6 +201,11 @@ class CronArchive private $formatter; /** + * @var SegmentArchivingRequestUrlProvider + */ + private $segmentArchivingRequestUrlProvider; + + /** * Returns the option name of the option that stores the time core:archive was last executed. * * @param int $idSite @@ -217,14 +224,19 @@ class CronArchive * we determine it using the current request information. * * If invoked via the command line, $piwikUrl cannot be false. + * @param string|null $processNewSegmentsFrom When to archive new segments from. See [General] process_new_segments_from + * for possible values. */ - public function __construct($piwikUrl = false) + public function __construct($piwikUrl = false, $processNewSegmentsFrom = null) { $this->formatter = new Formatter(); $this->initPiwikHost($piwikUrl); $this->initCore(); $this->initTokenAuth(); + + $processNewSegmentsFrom = $processNewSegmentsFrom ?: StaticContainer::get('ini.General.process_new_segments_from'); + $this->segmentArchivingRequestUrlProvider = new SegmentArchivingRequestUrlProvider($this->piwikUrl, $this->token_auth, $processNewSegmentsFrom); } /** @@ -771,7 +783,9 @@ class CronArchive } foreach ($this->getSegmentsForSite($idSite) as $segment) { - $urlWithSegment = $url . '&segment=' . urlencode($segment); + $urlWithSegment = $this->segmentArchivingRequestUrlProvider->getUrlToArchiveSegment($idSite, $period, $date, $segment); + $urlWithSegment .= self::APPEND_TO_API_REQUEST; + $urls[] = $urlWithSegment; $this->requests++; } @@ -1531,5 +1545,4 @@ class CronArchive return $customDateRangesToProcessForSites; } - -} +}
\ No newline at end of file diff --git a/core/CronArchive/SegmentArchivingRequestUrlProvider.php b/core/CronArchive/SegmentArchivingRequestUrlProvider.php new file mode 100644 index 0000000000..6b3828af06 --- /dev/null +++ b/core/CronArchive/SegmentArchivingRequestUrlProvider.php @@ -0,0 +1,143 @@ +<?php +/** + * Piwik - free/libre analytics platform + * + * @link http://piwik.org + * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later + */ +namespace Piwik\CronArchive; + +use Piwik\Cache\Cache; +use Piwik\Cache\Transient; +use Piwik\Date; +use Piwik\Period\Factory as PeriodFactory; +use Piwik\Period\Range; +use Piwik\Plugins\SegmentEditor\Model; + +/** + * Provides URLs that initiate archiving during cron archiving for segments. + * + * Handles the `[General] process_new_segments_from` INI option. + */ +class SegmentArchivingRequestUrlProvider +{ + const BEGINNING_OF_TIME = 'beginning_of_time'; + const CREATION_TIME = 'creation_time'; + + /** + * @var Model + */ + private $segmentEditorModel; + + /** + * @var Cache + */ + private $segmentListCache; + + /** + * @var Date + */ + private $now; + + private $baseUrl; + private $tokenAuth; + private $processNewSegmentsFrom; + + public function __construct($baseUrl, $tokenAuth, $processNewSegmentsFrom, Model $segmentEditorModel = null, Cache $segmentListCache = null, Date $now = null) + { + $this->baseUrl = $baseUrl; + $this->tokenAuth = $tokenAuth; + $this->processNewSegmentsFrom = $processNewSegmentsFrom; + $this->segmentEditorModel = $segmentEditorModel ?: new Model(); + $this->segmentListCache = $segmentListCache ?: new Transient(); + $this->now = $now ?: Date::factory('now'); + } + + public function getUrlToArchiveSegment($idSite, $period, $date, $segment) + { + $date = $this->getModifiedSegmentArchvingDateRange($idSite, $period, $date, $segment); + + return $this->baseUrl + . "?module=API&method=API.get&idSite=$idSite&period=$period&date=" . $date . "&format=php&token_auth=" . $this->tokenAuth + . '&segment=' . urlencode($segment); + } + + private function getModifiedSegmentArchvingDateRange($idSite, $period, $date, $segment) + { + $segmentCreatedTime = $this->getCreatedTimeOfSegment($idSite, $segment); + if (empty($segmentCreatedTime)) { + return $date; + } + + $oldestDateToProcessForNewSegment = $this->getOldestDateToProcessForNewSegment($segmentCreatedTime); + if (empty($oldestDateToProcessForNewSegment)) { + return $date; + } + + // if the start date for the archiving request is before the minimum date allowed for processing this segment, + // use the minimum allowed date as the start date + $periodObj = PeriodFactory::build($period, $date); + if ($periodObj->getDateStart()->getTimestamp() < $oldestDateToProcessForNewSegment->getTimestamp()) { + $date = $oldestDateToProcessForNewSegment->toString().','.$periodObj->getDateEnd(); + } + + return $date; + } + + private function getOldestDateToProcessForNewSegment(Date $segmentCreatedTime) + { + if ($this->processNewSegmentsFrom == self::CREATION_TIME) { + return $segmentCreatedTime; + } else if (preg_match("/^last([0-9]+)$/", $this->processNewSegmentsFrom, $matches)) { + $lastN = $matches[1]; + + list($lastDate, $lastPeriod) = Range::getDateXPeriodsAgo($lastN, $segmentCreatedTime, 'day'); + + return Date::factory($lastDate); + } else { + return null; + } + } + + private function getCreatedTimeOfSegment($idSite, $segmentDefinition) + { + $segments = $this->getAllSegments(); + + $earliestCreatedTime = $this->now; + foreach ($segments as $segment) { + if (empty($segment['ts_created']) + || empty($segment['definition']) + || !isset($segment['enable_only_idsite']) + ) { + continue; + } + + if ($this->isSegmentForSite($segment, $idSite) + && $segment['definition'] == $segmentDefinition + ) { + $createdTime = Date::factory($segment['ts_created']); + if ($createdTime->getTimestamp() < $earliestCreatedTime->getTimestamp()) { + $earliestCreatedTime = $createdTime; + } + } + } + return $earliestCreatedTime; + } + + private function getAllSegments() + { + if (!$this->segmentListCache->contains('all')) { + $segments = $this->segmentEditorModel->getAllSegmentsAndIgnoreVisibility(); + + $this->segmentListCache->save('all', $segments); + } + + return $this->segmentListCache->fetch('all'); + } + + private function isSegmentForSite($segment, $idSite) + { + return $segment['enable_only_idsite'] == 0 + || $segment['enable_only_idsite'] == $idSite; + } +}
\ No newline at end of file diff --git a/plugins/SegmentEditor/Model.php b/plugins/SegmentEditor/Model.php index 8bfcc957ae..2192e3ca58 100644 --- a/plugins/SegmentEditor/Model.php +++ b/plugins/SegmentEditor/Model.php @@ -26,6 +26,21 @@ class Model } /** + * Returns all stored segments that haven't been deleted. Ignores the site the segments are enabled + * for and whether to auto archive or not. + * + * @return array + */ + public function getAllSegmentsAndIgnoreVisibility() + { + $sql = "SELECT * FROM " . $this->table . " WHERE deleted = 0"; + + $segments = $this->getDb()->fetchAll($sql); + + return $segments; + } + + /** * Returns all stored segments. * * @param bool|int $idSite Whether to return stored segments for a specific idSite, or segments that are available diff --git a/tests/PHPUnit/Unit/CronArchive/SegmentArchivingRequestUrlProviderTest.php b/tests/PHPUnit/Unit/CronArchive/SegmentArchivingRequestUrlProviderTest.php new file mode 100644 index 0000000000..7d50d3f17f --- /dev/null +++ b/tests/PHPUnit/Unit/CronArchive/SegmentArchivingRequestUrlProviderTest.php @@ -0,0 +1,171 @@ +<?php +/** + * Piwik - free/libre analytics platform + * + * @link http://piwik.org + * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later + */ +namespace Piwik\Tests\Unit\CronArchive; + +use Piwik\Date; +use Piwik\CronArchive\SegmentArchivingRequestUrlProvider; + +/** + * @group Core + */ +class SegmentArchivingRequestUrlProviderTest extends \PHPUnit_Framework_TestCase +{ + const BASE_URL = 'http://base/'; + const TOKEN_AUTH = 'tokenauth'; + const TEST_NOW = '2015-03-01'; + + private $mockSegmentEntries; + + public function setUp() + { + $this->mockSegmentEntries = array( + array( + 'ts_created' => '2014-01-01', + 'definition' => 'browserName==FF', + 'enable_only_idsite' => 1 + ), + + array( + 'ts_created' => '2014-01-01', + 'definition' => 'countryCode==us', + 'enable_only_idsite' => 1 + ), + + array( + 'ts_created' => '2012-01-01', + 'definition' => 'countryCode==us', + 'enable_only_idsite' => 1 + ), + + array( + 'ts_created' => '2014-01-01', + 'definition' => 'countryCode==ca', + 'enable_only_idsite' => 2 + ), + + array( + 'ts_created' => '2012-01-01', + 'definition' => 'countryCode==ca', + 'enable_only_idsite' => 2 + ), + + array( + 'ts_created' => '2011-01-01', + 'definition' => 'countryCode==ca', + 'enable_only_idsite' => 0 + ) + ); + } + + /** + * @dataProvider getUrlToArchiveSegmentTestData + */ + public function test_getUrlToArchiveSegment_CorrectlyModifiesDateInOutputUrl($processNewSegmentsFrom, $idSite, $date, $period, $segment, $expected) + { + $urlProvider = $this->createUrlProviderToTest($processNewSegmentsFrom); + + $actual = $urlProvider->getUrlToArchiveSegment($idSite, $period, $date, $segment); + $this->assertEquals($expected, $actual); + } + + public function getUrlToArchiveSegmentTestData() + { + $dateRange = '2010-02-01,' . self::TEST_NOW; + return array( + array( // test beginning_of_time does not modify date + 'beginning_of_time', + 1, + $dateRange, + 'week', + 'browserName==FF', + "http://base/?module=API&method=API.get&idSite=1&period=week&date=$dateRange&format=php&token_auth=tokenauth&segment=" . urlencode('browserName==FF') + ), + + array( // test garbage string does not modify date + 'salkdfjsdfl', + 1, + $dateRange, + 'week', + 'browserName==FF', + "http://base/?module=API&method=API.get&idSite=1&period=week&date=$dateRange&format=php&token_auth=tokenauth&segment=" . urlencode('browserName==FF') + ), + + array( // test creation_time uses creation time of segment + 'creation_time', + 1, + $dateRange, + 'week', + 'browserName==FF', + 'http://base/?module=API&method=API.get&idSite=1&period=week&date=2014-01-01,2015-03-01&format=php&token_auth=tokenauth&segment=' . urlencode('browserName==FF') + ), + + array( // test creation_time uses earliest time of segment if multiple match (multiple for site) + 'creation_time', + 1, + $dateRange, + 'week', + 'countryCode==us', + 'http://base/?module=API&method=API.get&idSite=1&period=week&date=2012-01-01,2015-03-01&format=php&token_auth=tokenauth&segment=' . urlencode('countryCode==us') + ), + + array( // test creation_time uses earliest time of segment if multiple match (multiple for site + one for all) + 'creation_time', + 2, + $dateRange, + 'week', + 'countryCode==ca', + 'http://base/?module=API&method=API.get&idSite=2&period=week&date=2011-01-01,2015-03-01&format=php&token_auth=tokenauth&segment=' . urlencode('countryCode==ca') + ), + + array( // test 'now' is used if no site matches (testing w/o any segments) + 'creation_time', + 1, + $dateRange, + 'week', + 'pageTitle==abc', + 'http://base/?module=API&method=API.get&idSite=1&period=week&date=2015-03-01,2015-03-01&format=php&token_auth=tokenauth&segment=' . urlencode('pageTitle==abc') + ), + + array( // test 'now' is used if no site matches (testing w/ segment for another site) + 'creation_time', + 3, + $dateRange, + 'week', + 'countryCode==us', + 'http://base/?module=API&method=API.get&idSite=3&period=week&date=2015-03-01,2015-03-01&format=php&token_auth=tokenauth&segment=' . urlencode('countryCode==us') + ), + + array( // test lastN rewinds created date by N days + 'last10', + 1, + $dateRange, + 'week', + 'countryCode==us', + 'http://base/?module=API&method=API.get&idSite=1&period=week&date=2011-12-22,2015-03-01&format=php&token_auth=tokenauth&segment=' . urlencode('countryCode==us') + ), + + array( // test lastN rewinds now by N days (testing w/ no found segment) + 'last10', + 3, + $dateRange, + 'week', + 'countryCode==us', + 'http://base/?module=API&method=API.get&idSite=3&period=week&date=2015-02-19,2015-03-01&format=php&token_auth=tokenauth&segment=' . urlencode('countryCode==us') + ), + ); + } + + private function createUrlProviderToTest($processNewSegmentsFrom) + { + $mockSegmentEditorModel = $this->getMock('Piwik\Plugins\SegmentEditor\Model', array('getAllSegmentsAndIgnoreVisibility')); + $mockSegmentEditorModel->expects($this->any())->method('getAllSegmentsAndIgnoreVisibility')->will($this->returnValue($this->mockSegmentEntries)); + + return new SegmentArchivingRequestUrlProvider(self::BASE_URL, self::TOKEN_AUTH, $processNewSegmentsFrom, + $mockSegmentEditorModel, null, Date::factory(self::TEST_NOW)); + } +}
\ No newline at end of file |