diff options
| -rw-r--r-- | config/global.ini.php | 7 | ||||
| -rw-r--r-- | core/CronArchive.php | 68 | ||||
| -rw-r--r-- | core/CronArchive/SegmentArchivingRequestUrlProvider.php | 138 | ||||
| -rw-r--r-- | core/DataTable.php | 8 | ||||
| -rw-r--r-- | core/DataTable/Filter/Sort.php | 2 | ||||
| -rw-r--r-- | core/DataTable/Filter/Truncate.php | 2 | ||||
| -rw-r--r-- | core/Version.php | 2 | ||||
| -rw-r--r-- | plugins/Actions/DataTable/Filter/Actions.php | 2 | ||||
| -rw-r--r-- | plugins/Referrers/DataTable/Filter/UrlsFromWebsiteId.php | 2 | ||||
| -rw-r--r-- | plugins/SegmentEditor/Model.php | 30 | ||||
| -rw-r--r-- | tests/PHPUnit/Unit/CronArchive/SegmentArchivingRequestUrlProviderTest.php | 196 |
11 files changed, 423 insertions, 34 deletions
diff --git a/config/global.ini.php b/config/global.ini.php index de3e80ca72..b892b7d822 100644 --- a/config/global.ini.php +++ b/config/global.ini.php @@ -220,6 +220,13 @@ adding_segment_requires_access = "view" ; on Piwik performance. allow_adding_segments_for_all_websites = 1 +; When archiving segments for the first time, this determines the oldest date that will be archived. +; This option can be used to avoid archiving (for isntance) the lastN years for every new segment. +; Valid option values include: "beginning_of_time" (start date of archiving will not be changed) +; "segment_creation_time" (start date of archiving will be the creation date of the segment) +; lastN where N is an integer (eg "last10" to archive for 10 days before the segment creation date) +process_new_segments_from = "beginning_of_time" + ; this action name is used when the URL ends with a slash / ; it is useful to have an actual string to write in the UI action_default_name = index diff --git a/core/CronArchive.php b/core/CronArchive.php index 0ff2ee8242..50716ea756 100644 --- a/core/CronArchive.php +++ b/core/CronArchive.php @@ -10,6 +10,7 @@ namespace Piwik; use Exception; use Piwik\ArchiveProcessor\Rules; +use Piwik\Container\StaticContainer; use Piwik\CronArchive\FixedSiteIds; use Piwik\CronArchive\SharedSiteIds; use Piwik\Archive\ArchiveInvalidator; @@ -17,6 +18,7 @@ use Piwik\Exception\UnexpectedWebsiteFoundException; use Piwik\Metrics\Formatter; use Piwik\Period\Factory as PeriodFactory; use Piwik\CronArchive\SitesToReprocessDistributedList; +use Piwik\CronArchive\SegmentArchivingRequestUrlProvider; use Piwik\Plugins\CoreAdminHome\API as CoreAdminHomeAPI; use Piwik\Plugins\SitesManager\API as APISitesManager; use Piwik\Plugins\UsersManager\API as APIUsersManager; @@ -199,6 +201,11 @@ class CronArchive private $formatter; /** + * @var SegmentArchivingRequestUrlProvider + */ + private $segmentArchivingRequestUrlProvider; + + /** * Returns the option name of the option that stores the time core:archive was last executed. * * @param int $idSite @@ -217,14 +224,19 @@ class CronArchive * we determine it using the current request information. * * If invoked via the command line, $piwikUrl cannot be false. + * @param string|null $processNewSegmentsFrom When to archive new segments from. See [General] process_new_segments_from + * for possible values. */ - public function __construct($piwikUrl = false) + public function __construct($piwikUrl = false, $processNewSegmentsFrom = null) { $this->formatter = new Formatter(); $this->initPiwikHost($piwikUrl); $this->initCore(); $this->initTokenAuth(); + + $processNewSegmentsFrom = $processNewSegmentsFrom ?: StaticContainer::get('ini.General.process_new_segments_from'); + $this->segmentArchivingRequestUrlProvider = new SegmentArchivingRequestUrlProvider($processNewSegmentsFrom); } /** @@ -617,9 +629,13 @@ class CronArchive /** * Returns base URL to process reports for the $idSite on a given $period */ - private function getVisitsRequestUrl($idSite, $period, $date) + private function getVisitsRequestUrl($idSite, $period, $date, $segment = false) { - return "?module=API&method=API.get&idSite=$idSite&period=$period&date=" . $date . "&format=php&token_auth=" . $this->token_auth; + $request = "?module=API&method=API.get&idSite=$idSite&period=$period&date=" . $date . "&format=php&token_auth=" . $this->token_auth; + if($segment) { + $request .= '&segment=' . urlencode($segment);; + } + return $request; } private function initSegmentsToArchive() @@ -727,12 +743,18 @@ class CronArchive return true; } - private function getSegmentsForSite($idSite) + private function getSegmentsForSite($idSite, $period) { $segmentsAllSites = $this->segments; $segmentsThisSite = SettingsPiwik::getKnownSegmentsToArchiveForSite($idSite); if (!empty($segmentsThisSite)) { - $this->log("Will pre-process the following " . count($segmentsThisSite) . " Segments for this website (id = $idSite): " . implode(", ", $segmentsThisSite)); + $this->log(sprintf( + "Will pre-process for website id = %s, %s period, the following %d segments: { %s } ", + $idSite, + $period, + count($segmentsThisSite), + implode(", ", $segmentsThisSite) + )); } $segments = array_unique(array_merge($segmentsAllSites, $segmentsThisSite)); return $segments; @@ -752,11 +774,8 @@ class CronArchive { $timer = new Timer(); - $url = $this->piwikUrl; - - $url .= $this->getVisitsRequestUrl($idSite, $period, $date); - - $url .= self::APPEND_TO_API_REQUEST; + $url = $this->getVisitsRequestUrl($idSite, $period, $date, $segment = false); + $url = $this->makeRequestUrl($url); $visitsInLastPeriods = $visitsLastPeriod = 0; $success = true; @@ -767,15 +786,21 @@ class CronArchive // already processed above for "day" if ($period != "day") { $urls[] = $url; - $this->requests++; } - foreach ($this->getSegmentsForSite($idSite) as $segment) { - $urlWithSegment = $url . '&segment=' . urlencode($segment); + foreach ($this->getSegmentsForSite($idSite, $period) as $segment) { + $dateParamForSegment = $this->segmentArchivingRequestUrlProvider->getUrlParameterDateString($idSite, $period, $date, $segment); + + $urlWithSegment = $this->getVisitsRequestUrl($idSite, $period, $dateParamForSegment, $segment); + $urlWithSegment = $this->makeRequestUrl($urlWithSegment); + $urls[] = $urlWithSegment; - $this->requests++; } + // in case several segment URLs for period=range had the date= rewritten to the same value, we only call API once + $urls = array_unique($urls); + $this->requests += count($urls); + $cliMulti = new CliMulti(); $cliMulti->setAcceptInvalidSSLCertificate($this->acceptInvalidSSLCertificate); $cliMulti->setConcurrentProcessesLimit($this->getConcurrentRequestsPerWebsite()); @@ -852,11 +877,12 @@ class CronArchive } /** - * Issues a request to $url + * Issues a request to $url eg. "?module=API&method=API.getDefaultMetricTranslations&format=original&serialize=1" + * */ private function request($url) { - $url = $this->piwikUrl . $url . self::APPEND_TO_API_REQUEST; + $url = $this->makeRequestUrl($url); if ($this->shouldStartProfiler) { $url .= "&xhprof=2"; @@ -1532,4 +1558,12 @@ class CronArchive return $customDateRangesToProcessForSites; } -} + /** + * @param $url + * @return string + */ + private function makeRequestUrl($url) + { + return $this->piwikUrl . $url . self::APPEND_TO_API_REQUEST; + } +}
\ No newline at end of file diff --git a/core/CronArchive/SegmentArchivingRequestUrlProvider.php b/core/CronArchive/SegmentArchivingRequestUrlProvider.php new file mode 100644 index 0000000000..d43fc03ed5 --- /dev/null +++ b/core/CronArchive/SegmentArchivingRequestUrlProvider.php @@ -0,0 +1,138 @@ +<?php +/** + * Piwik - free/libre analytics platform + * + * @link http://piwik.org + * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later + */ +namespace Piwik\CronArchive; + +use Piwik\Cache\Cache; +use Piwik\Cache\Transient; +use Piwik\Date; +use Piwik\Period\Factory as PeriodFactory; +use Piwik\Period\Range; +use Piwik\Plugins\SegmentEditor\Model; + +/** + * Provides URLs that initiate archiving during cron archiving for segments. + * + * Handles the `[General] process_new_segments_from` INI option. + */ +class SegmentArchivingRequestUrlProvider +{ + const BEGINNING_OF_TIME = 'beginning_of_time'; + const CREATION_TIME = 'segment_creation_time'; + + /** + * @var Model + */ + private $segmentEditorModel; + + /** + * @var Cache + */ + private $segmentListCache; + + /** + * @var Date + */ + private $now; + + private $processNewSegmentsFrom; + + public function __construct($processNewSegmentsFrom, Model $segmentEditorModel = null, Cache $segmentListCache = null, Date $now = null) + { + $this->processNewSegmentsFrom = $processNewSegmentsFrom; + $this->segmentEditorModel = $segmentEditorModel ?: new Model(); + $this->segmentListCache = $segmentListCache ?: new Transient(); + $this->now = $now ?: Date::factory('now'); + } + + public function getUrlParameterDateString($idSite, $period, $date, $segment) + { + $segmentCreatedTime = $this->getCreatedTimeOfSegment($idSite, $segment); + if (empty($segmentCreatedTime)) { + return $date; + } + + $oldestDateToProcessForNewSegment = $this->getOldestDateToProcessForNewSegment($segmentCreatedTime); + if (empty($oldestDateToProcessForNewSegment)) { + return $date; + } + + // if the start date for the archiving request is before the minimum date allowed for processing this segment, + // use the minimum allowed date as the start date + $periodObj = PeriodFactory::build($period, $date); + if ($periodObj->getDateStart()->getTimestamp() < $oldestDateToProcessForNewSegment->getTimestamp()) { + $endDate = $periodObj->getDateEnd(); + + // if the creation time of a segment is older than the end date of the archiving request range, we cannot + // blindly rewrite the date string, since the resulting range would be incorrect. instead we make the + // start date equal to the end date, so less archiving occurs, and no fatal error occurs. + if ($oldestDateToProcessForNewSegment->getTimestamp() > $endDate->getTimestamp()) { + $oldestDateToProcessForNewSegment = $endDate; + } + + $date = $oldestDateToProcessForNewSegment->toString().','.$endDate; + } + + return $date; + } + + private function getOldestDateToProcessForNewSegment(Date $segmentCreatedTime) + { + if ($this->processNewSegmentsFrom == self::CREATION_TIME) { + return $segmentCreatedTime; + } else if (preg_match("/^last([0-9]+)$/", $this->processNewSegmentsFrom, $matches)) { + $lastN = $matches[1]; + + list($lastDate, $lastPeriod) = Range::getDateXPeriodsAgo($lastN, $segmentCreatedTime, 'day'); + return Date::factory($lastDate); + } else { + return null; + } + } + + private function getCreatedTimeOfSegment($idSite, $segmentDefinition) + { + $segments = $this->getAllSegments(); + + $earliestCreatedTime = $this->now; + foreach ($segments as $segment) { + if (empty($segment['ts_created']) + || empty($segment['definition']) + || !isset($segment['enable_only_idsite']) + ) { + continue; + } + + if ($this->isSegmentForSite($segment, $idSite) + && $segment['definition'] == $segmentDefinition + ) { + $createdTime = Date::factory($segment['ts_created']); + if ($createdTime->getTimestamp() < $earliestCreatedTime->getTimestamp()) { + $earliestCreatedTime = $createdTime; + } + } + } + return $earliestCreatedTime; + } + + private function getAllSegments() + { + if (!$this->segmentListCache->contains('all')) { + $segments = $this->segmentEditorModel->getAllSegmentsAndIgnoreVisibility(); + + $this->segmentListCache->save('all', $segments); + } + + return $this->segmentListCache->fetch('all'); + } + + private function isSegmentForSite($segment, $idSite) + { + return $segment['enable_only_idsite'] == 0 + || $segment['enable_only_idsite'] == $idSite; + } +}
\ No newline at end of file diff --git a/core/DataTable.php b/core/DataTable.php index 96f136a158..2e94d5e89e 100644 --- a/core/DataTable.php +++ b/core/DataTable.php @@ -374,7 +374,7 @@ class DataTable implements DataTableInterface, \IteratorAggregate, \ArrayAccess usort($this->rows, $functionCallback); if ($this->isSortRecursiveEnabled()) { - foreach ($this->getRows() as $row) { + foreach ($this->getRowsWithoutSummaryRow() as $row) { $subTable = $row->getSubtable(); if ($subTable) { @@ -487,7 +487,7 @@ class DataTable implements DataTableInterface, \IteratorAggregate, \ArrayAccess */ public function filterSubtables($className, $parameters = array()) { - foreach ($this->getRows() as $row) { + foreach ($this->getRowsWithoutSummaryRow() as $row) { $subtable = $row->getSubtable(); if ($subtable) { $subtable->filter($className, $parameters); @@ -508,7 +508,7 @@ class DataTable implements DataTableInterface, \IteratorAggregate, \ArrayAccess */ public function queueFilterSubtables($className, $parameters = array()) { - foreach ($this->getRows() as $row) { + foreach ($this->getRowsWithoutSummaryRow() as $row) { $subtable = $row->getSubtable(); if ($subtable) { $subtable->queueFilter($className, $parameters); @@ -1601,7 +1601,7 @@ class DataTable implements DataTableInterface, \IteratorAggregate, \ArrayAccess public function mergeSubtables($labelColumn = false, $useMetadataColumn = false) { $result = new DataTable(); - foreach ($this->getRows() as $row) { + foreach ($this->getRowsWithoutSummaryRow() as $row) { $subtable = $row->getSubtable(); if ($subtable !== false) { $parentLabel = $row->getColumn('label'); diff --git a/core/DataTable/Filter/Sort.php b/core/DataTable/Filter/Sort.php index 3b11ec370c..53c21f526a 100644 --- a/core/DataTable/Filter/Sort.php +++ b/core/DataTable/Filter/Sort.php @@ -279,7 +279,7 @@ class Sort extends BaseFilter unset($sortedRows); if ($table->isSortRecursiveEnabled()) { - foreach ($table->getRows() as $row) { + foreach ($table->getRowsWithoutSummaryRow() as $row) { $subTable = $row->getSubtable(); if ($subTable) { diff --git a/core/DataTable/Filter/Truncate.php b/core/DataTable/Filter/Truncate.php index 04b4cef2a8..ec95811c50 100644 --- a/core/DataTable/Filter/Truncate.php +++ b/core/DataTable/Filter/Truncate.php @@ -77,7 +77,7 @@ class Truncate extends BaseFilter $table->queueFilter('ReplaceSummaryRowLabel', array($this->labelSummaryRow)); if ($this->filterRecursive) { - foreach ($table->getRows() as $row) { + foreach ($table->getRowsWithoutSummaryRow() as $row) { if ($row->isSubtableLoaded()) { $this->filter($row->getSubtable()); } diff --git a/core/Version.php b/core/Version.php index d937eaafdd..ef5c6be2a6 100644 --- a/core/Version.php +++ b/core/Version.php @@ -20,7 +20,7 @@ final class Version * The current Piwik version. * @var string */ - const VERSION = '2.12.0-b8'; + const VERSION = '2.12.0-rc1'; public function isStableVersion($version) { diff --git a/plugins/Actions/DataTable/Filter/Actions.php b/plugins/Actions/DataTable/Filter/Actions.php index 71abe4190b..8d13311677 100644 --- a/plugins/Actions/DataTable/Filter/Actions.php +++ b/plugins/Actions/DataTable/Filter/Actions.php @@ -43,7 +43,7 @@ class Actions extends BaseFilter return urldecode($label); })); - foreach ($table->getRows() as $row) { + foreach ($table->getRowsWithoutSummaryRow() as $row) { $subtable = $row->getSubtable(); if ($subtable) { $this->filter($subtable); diff --git a/plugins/Referrers/DataTable/Filter/UrlsFromWebsiteId.php b/plugins/Referrers/DataTable/Filter/UrlsFromWebsiteId.php index c35d60b463..ff67c22e40 100644 --- a/plugins/Referrers/DataTable/Filter/UrlsFromWebsiteId.php +++ b/plugins/Referrers/DataTable/Filter/UrlsFromWebsiteId.php @@ -36,7 +36,7 @@ class UrlsFromWebsiteId extends BaseFilter })); $table->queueFilter('ColumnCallbackReplace', array('label', 'Piwik\Plugins\Referrers\getPathFromUrl')); - foreach ($table->getRows() as $row) { + foreach ($table->getRowsWithoutSummaryRow() as $row) { $subtable = $row->getSubtable(); if ($subtable) { $this->filter($subtable); diff --git a/plugins/SegmentEditor/Model.php b/plugins/SegmentEditor/Model.php index 8bfcc957ae..7f312557f9 100644 --- a/plugins/SegmentEditor/Model.php +++ b/plugins/SegmentEditor/Model.php @@ -18,11 +18,25 @@ use Piwik\DbHelper; class Model { private static $rawPrefix = 'segment'; - private $table; - public function __construct() + protected function getTable() { - $this->table = Common::prefixTable(self::$rawPrefix); + return Common::prefixTable(self::$rawPrefix); + } + + /** + * Returns all stored segments that haven't been deleted. Ignores the site the segments are enabled + * for and whether to auto archive or not. + * + * @return array + */ + public function getAllSegmentsAndIgnoreVisibility() + { + $sql = "SELECT * FROM " . $this->getTable() . " WHERE deleted = 0"; + + $segments = $this->getDb()->fetchAll($sql); + + return $segments; } /** @@ -87,7 +101,7 @@ class Model public function deleteSegment($idSegment) { $db = $this->getDb(); - $db->delete($this->table, 'idsegment = ' . (int) $idSegment); + $db->delete($this->getTable(), 'idsegment = ' . (int) $idSegment); } public function updateSegment($idSegment, $segment) @@ -95,7 +109,7 @@ class Model $idSegment = (int) $idSegment; $db = $this->getDb(); - $db->update($this->table, $segment, "idsegment = $idSegment"); + $db->update($this->getTable(), $segment, "idsegment = $idSegment"); return true; } @@ -103,7 +117,7 @@ class Model public function createSegment($segment) { $db = $this->getDb(); - $db->insert($this->table, $segment); + $db->insert($this->getTable(), $segment); $id = $db->lastInsertId(); return $id; @@ -112,7 +126,7 @@ class Model public function getSegment($idSegment) { $db = $this->getDb(); - $segment = $db->fetchRow("SELECT * FROM " . $this->table . " WHERE idsegment = ?", $idSegment); + $segment = $db->fetchRow("SELECT * FROM " . $this->getTable() . " WHERE idsegment = ?", $idSegment); return $segment; } @@ -124,7 +138,7 @@ class Model private function buildQuerySortedByName($where) { - return "SELECT * FROM " . $this->table . " WHERE $where ORDER BY name ASC"; + return "SELECT * FROM " . $this->getTable() . " WHERE $where ORDER BY name ASC"; } public static function install() diff --git a/tests/PHPUnit/Unit/CronArchive/SegmentArchivingRequestUrlProviderTest.php b/tests/PHPUnit/Unit/CronArchive/SegmentArchivingRequestUrlProviderTest.php new file mode 100644 index 0000000000..e449024326 --- /dev/null +++ b/tests/PHPUnit/Unit/CronArchive/SegmentArchivingRequestUrlProviderTest.php @@ -0,0 +1,196 @@ +<?php +/** + * Piwik - free/libre analytics platform + * + * @link http://piwik.org + * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later + */ +namespace Piwik\Tests\Unit\CronArchive; + +use Piwik\Config; +use Piwik\Date; +use Piwik\CronArchive\SegmentArchivingRequestUrlProvider; + +/** + * @group Core + */ +class SegmentArchivingRequestUrlProviderTest extends \PHPUnit_Framework_TestCase +{ + const TEST_NOW = '2015-03-01'; + + private $mockSegmentEntries; + + public function setUp() + { + Config::getInstance()->General['enabled_periods_API'] = 'day,week,month,year,range'; + + $this->mockSegmentEntries = array( + array( + 'ts_created' => '2014-01-01', + 'definition' => 'browserName==FF', + 'enable_only_idsite' => 1 + ), + + array( + 'ts_created' => '2014-01-01', + 'definition' => 'countryCode==us', + 'enable_only_idsite' => 1 + ), + + array( + 'ts_created' => '2012-01-01', + 'definition' => 'countryCode==us', + 'enable_only_idsite' => 1 + ), + + array( + 'ts_created' => '2014-01-01', + 'definition' => 'countryCode==ca', + 'enable_only_idsite' => 2 + ), + + array( + 'ts_created' => '2012-01-01', + 'definition' => 'countryCode==ca', + 'enable_only_idsite' => 2 + ), + + array( + 'ts_created' => '2011-01-01', + 'definition' => 'countryCode==ca', + 'enable_only_idsite' => 0 + ), + + array( + 'ts_created' => '2015-03-01', + 'definition' => 'pageUrl==a', + 'enable_only_idsite' => 1 + ) + ); + } + + /** + * @dataProvider getUrlToArchiveSegmentTestData + */ + public function test_getUrlToArchiveSegment_CorrectlyModifiesDateInOutputUrl($processNewSegmentsFrom, $idSite, $date, $period, $segment, $expected) + { + $urlProvider = $this->createUrlProviderToTest($processNewSegmentsFrom); + + $actual = $urlProvider->getUrlParameterDateString($idSite, $period, $date, $segment); + $this->assertEquals($expected, $actual); + } + + public function getUrlToArchiveSegmentTestData() + { + $dateRange = '2010-02-01,' . self::TEST_NOW; + + return array( + array( // test beginning_of_time does not modify date + 'beginning_of_time', + 1, + $dateRange, + 'week', + 'browserName==FF', + $dateRange + ), + + array( // test garbage string does not modify date + 'salkdfjsdfl', + 1, + $dateRange, + 'week', + 'browserName==FF', + $dateRange + ), + + array( // test creation_time uses creation time of segment + 'segment_creation_time', + 1, + $dateRange, + 'week', + 'browserName==FF', + "2014-01-01,2015-03-01" + ), + + array( // test segment_creation_time uses earliest time of segment if multiple match (multiple for site) + 'segment_creation_time', + 1, + $dateRange, + 'week', + 'countryCode==us', + '2012-01-01,2015-03-01' + ), + + array( // test segment_creation_time uses earliest time of segment if multiple match (multiple for site + one for all) + 'segment_creation_time', + 2, + $dateRange, + 'week', + 'countryCode==ca', + '2011-01-01,2015-03-01' + ), + + array( // test 'now' is used if no site matches (testing w/o any segments) + 'segment_creation_time', + 1, + $dateRange, + 'week', + 'pageTitle==abc', + "2015-03-01,2015-03-01" + ), + + array( // test 'now' is used if no site matches (testing w/ segment for another site) + 'segment_creation_time', + 3, + $dateRange, + 'week', + 'countryCode==us', + "2015-03-01,2015-03-01" + ), + + array( // test lastN rewinds created date by N days + 'last10', + 1, + $dateRange, + 'week', + 'countryCode==us', + "2011-12-22,2015-03-01" + ), + + array( // test lastN rewinds now by N days (testing w/ no found segment) + 'last10', + 3, + $dateRange, + 'week', + 'countryCode==us', + "2015-02-19,2015-03-01" + ), + + array( // test when creation_time is greater than date range end date + 'segment_creation_time', + 1, + '2010-02-01,2015-02-22', + 'week', + 'pageUrl==a', + '2015-02-22,2015-02-22' + ), + + array( + 'segment_creation_time', + 1, + '2015-02-01,' . self::TEST_NOW, + 'week', + 'countryCode==us', + '2015-02-01,2015-03-01' + ), + ); + } + + private function createUrlProviderToTest($processNewSegmentsFrom) + { + $mockSegmentEditorModel = $this->getMock('Piwik\Plugins\SegmentEditor\Model', array('getAllSegmentsAndIgnoreVisibility')); + $mockSegmentEditorModel->expects($this->any())->method('getAllSegmentsAndIgnoreVisibility')->will($this->returnValue($this->mockSegmentEntries)); + + return new SegmentArchivingRequestUrlProvider($processNewSegmentsFrom, $mockSegmentEditorModel, null, Date::factory(self::TEST_NOW)); + } +}
\ No newline at end of file |