diff options
author | Matthieu Aubry <matt@piwik.org> | 2014-12-16 05:53:38 +0300 |
---|---|---|
committer | Matthieu Aubry <matt@piwik.org> | 2014-12-16 05:53:38 +0300 |
commit | 0d55fb656ece0d793cd9374c0aff818823d82bed (patch) | |
tree | 27959fbdda79225adeffca18f1f23faaebe40ff7 | |
parent | 3fe19974b4a9a24583f41852fcba42c386d900f3 (diff) | |
parent | 0ab6b13eed19a12ee0c52a8b459b8ee8fb3a2afe (diff) |
Merge pull request #6856 from piwik/6672_preprocess_ranges2.10.0-b7
during core:archive console command, preprocess any custom date ranges that were selected in any User's Preferences
-rw-r--r-- | core/CronArchive.php | 132 | ||||
-rw-r--r-- | core/Plugin/Controller.php | 1 | ||||
-rw-r--r-- | core/Plugin/Menu.php | 2 | ||||
-rw-r--r-- | core/Site.php | 5 | ||||
-rw-r--r-- | plugins/UsersManager/API.php | 29 | ||||
-rw-r--r-- | plugins/UsersManager/UserPreferences.php | 24 | ||||
-rw-r--r-- | plugins/UsersManager/tests/Integration/APITest.php | 39 |
7 files changed, 199 insertions, 33 deletions
diff --git a/core/CronArchive.php b/core/CronArchive.php index 9cf56dbf8e..d36e988832 100644 --- a/core/CronArchive.php +++ b/core/CronArchive.php @@ -12,10 +12,13 @@ use Exception; use Piwik\ArchiveProcessor\Rules; use Piwik\CronArchive\FixedSiteIds; use Piwik\CronArchive\SharedSiteIds; +use Piwik\Exception\UnexpectedWebsiteFoundException; use Piwik\Metrics\Formatter; use Piwik\Period\Factory as PeriodFactory; use Piwik\DataAccess\InvalidatedReports; use Piwik\Plugins\SitesManager\API as APISitesManager; +use Piwik\Plugins\UsersManager\API as APIUsersManager; +use Piwik\Plugins\UsersManager\UserPreferences; /** * ./console core:archive runs as a cron and is a useful tool for general maintenance, @@ -251,7 +254,7 @@ class CronArchive $this->allWebsites = APISitesManager::getInstance()->getAllSitesId(); if (!empty($this->shouldArchiveOnlySpecificPeriods)) { - $this->log("- Will process the following periods: " . implode(", ", $this->shouldArchiveOnlySpecificPeriods) . " (--force-periods)"); + $this->log("- Will only process the following periods: " . implode(", ", $this->shouldArchiveOnlySpecificPeriods) . " (--force-periods)"); } $websitesIds = $this->initWebsiteIds(); @@ -503,7 +506,14 @@ class CronArchive return false; } - $shouldProceed = $this->processArchiveDays($idSite, $lastTimestampWebsiteProcessedDay, $shouldArchivePeriods, $timerWebsite); + try { + $shouldProceed = $this->processArchiveDays($idSite, $lastTimestampWebsiteProcessedDay, $shouldArchivePeriods, $timerWebsite); + } catch(UnexpectedWebsiteFoundException $e) { + // this website was deleted in the meantime + $shouldProceed = false; + $this->log("Skipped website id $idSite, got: UnexpectedWebsiteFoundException, " . $timerWebsite->__toString()); + } + if (!$shouldProceed) { return false; } @@ -517,18 +527,8 @@ class CronArchive return false; } - $success = true; - foreach (array('week', 'month', 'year') as $period) { - - if (!$this->shouldProcessPeriod($period)) { - // if any period was skipped, we do not mark the Periods archiving as successful - $success = false; - continue; - } + $success = $this->processArchiveForPeriods($idSite, $lastTimestampWebsiteProcessedPeriods); - $success = $this->archiveVisitsAndSegments($idSite, $period, $lastTimestampWebsiteProcessedPeriods) - && $success; - } // Record succesful run of this website's periods archiving if ($success) { Option::set($this->lastRunKey($idSite, "periods"), time()); @@ -556,6 +556,37 @@ class CronArchive } /** + * @param $idSite + * @param $lastTimestampWebsiteProcessedPeriods + * @return bool + */ + private function processArchiveForPeriods($idSite, $lastTimestampWebsiteProcessedPeriods) + { + $success = true; + + foreach (array('week', 'month', 'year') as $period) { + + if (!$this->shouldProcessPeriod($period)) { + // if any period was skipped, we do not mark the Periods archiving as successful + $success = false; + continue; + } + + $date = $this->getApiDateParameter($idSite, $period, $lastTimestampWebsiteProcessedPeriods); + $periodArchiveWasSuccessful = $this->archiveVisitsAndSegments($idSite, $period, $date); + $success = $periodArchiveWasSuccessful && $success; + } + + // period=range + $customDateRangesToPreProcessForSite = $this->getCustomDateRangeToPreProcess($idSite); + foreach ($customDateRangesToPreProcessForSite as $dateRange) { + $periodArchiveWasSuccessful = $this->archiveVisitsAndSegments($idSite, 'range', $dateRange); + $success = $periodArchiveWasSuccessful && $success; + } + return $success; + } + + /** * Checks the config file is found. * * @param $piwikUrl @@ -684,7 +715,8 @@ class CronArchive $this->visitsToday += $visitsToday; $this->websitesWithVisitsSinceLastRun++; - $this->archiveVisitsAndSegments($idSite, "day", $processDaysSince); + + $this->archiveVisitsAndSegments($idSite, "day", $this->getApiDateParameter($idSite, "day", $processDaysSince)); $this->logArchivedWebsite($idSite, "day", $date, $visitsLastDays, $visitsToday, $timerWebsite); return true; @@ -707,17 +739,16 @@ class CronArchive * Requests are triggered using cURL multi handle * * @param $idSite int - * @param $period - * @param $lastTimestampWebsiteProcessed + * @param $period string + * @param $date string * @return bool True on success, false if some request failed */ - private function archiveVisitsAndSegments($idSite, $period, $lastTimestampWebsiteProcessed) + private function archiveVisitsAndSegments($idSite, $period, $date) { $timer = new Timer(); $url = $this->piwikUrl; - $date = $this->getApiDateParameter($idSite, $period, $lastTimestampWebsiteProcessed); $url .= $this->getVisitsRequestUrl($idSite, $period, $date); $url .= self::APPEND_TO_API_REQUEST; @@ -756,6 +787,12 @@ class CronArchive $this->logError("Error unserializing the following response from $url: " . $content); } + if($period == 'range') { + // range returns one dataset (the sum of data between the two dates), + // whereas other periods return lastN which is N datasets in an array. Here we make our period=range dataset look like others: + $stats = array($stats); + } + $visitsInLastPeriods = $this->getVisitsFromApiResponse($stats); $visitsLastPeriod = $this->getVisitsLastPeriodFromApiResponse($stats); } @@ -1281,7 +1318,7 @@ class CronArchive */ private function logArchivedWebsite($idSite, $period, $date, $visitsInLastPeriods, $visitsToday, Timer $timer) { - if (substr($date, 0, 4) === 'last') { + if (strpos($date, 'last') === 0 || strpos($date, 'previous') === 0) { $visitsInLastPeriods = (int)$visitsInLastPeriods . " visits in last " . $date . " " . $period . "s, "; $thisPeriod = $period == "day" ? "today" : "this " . $period; $visitsInLastPeriod = (int)$visitsToday . " visits " . $thisPeriod . ", "; @@ -1361,7 +1398,8 @@ class CronArchive $dateLastMax = self::DEFAULT_DATE_LAST_WEEKS; } if (empty($lastTimestampWebsiteProcessed)) { - $lastTimestampWebsiteProcessed = strtotime(\Piwik\Site::getCreationDateFor($idSite)); + $creationDateFor = \Piwik\Site::getCreationDateFor($idSite); + $lastTimestampWebsiteProcessed = strtotime($creationDateFor); } // Enforcing last2 at minimum to work around timing issues and ensure we make most archives available @@ -1423,4 +1461,58 @@ class CronArchive $now = time(); return ($timestamp < $now) ? $timestamp : $now; } + + /** + * @param $idSite + * @return array of date strings + */ + private function getCustomDateRangeToPreProcess($idSite) + { + static $cache = null; + if(is_null($cache)) { + $cache = $this->loadCustomDateRangeToPreProcess(); + } + if(empty($cache[$idSite])) { + return array(); + } + $dates = array_unique($cache[$idSite]); + return $dates; + } + + /** + * @return array + */ + private function loadCustomDateRangeToPreProcess() + { + $customDateRangesToProcessForSites = array(); + // For all users who have selected this website to load by default, + // we load the default period/date that will be loaded for this user + // and make sure it's pre-archived + $userPreferences = APIUsersManager::getInstance()->getAllUsersPreferences(array(APIUsersManager::PREFERENCE_DEFAULT_REPORT_DATE, APIUsersManager::PREFERENCE_DEFAULT_REPORT)); + foreach ($userPreferences as $userLogin => $userPreferences) { + + $defaultDate = $userPreferences[APIUsersManager::PREFERENCE_DEFAULT_REPORT_DATE]; + $preference = new UserPreferences(); + $period = $preference->getDefaultPeriod($defaultDate); + if ($period != 'range') { + continue; + } + + $defaultReport = $userPreferences[APIUsersManager::PREFERENCE_DEFAULT_REPORT]; + if (is_numeric($defaultReport)) { + // If user selected one particular website ID + $idSites = array($defaultReport); + } else { + // If user selected "All websites" or some other random value, we pre-process all websites that he has access to + $idSites = APISitesManager::getInstance()->getSitesIdWithAtLeastViewAccess($userLogin); + } + + foreach ($idSites as $idSite) { + $customDateRangesToProcessForSites[$idSite][] = $defaultDate; + } + } + + return $customDateRangesToProcessForSites; + } + } diff --git a/core/Plugin/Controller.php b/core/Plugin/Controller.php index 4afcbf25ad..2e60fcd734 100644 --- a/core/Plugin/Controller.php +++ b/core/Plugin/Controller.php @@ -31,7 +31,6 @@ use Piwik\Piwik; use Piwik\Plugins\CoreAdminHome\CustomLogo; use Piwik\Plugins\CoreVisualizations\Visualizations\JqplotGraph\Evolution; use Piwik\Plugins\LanguagesManager\LanguagesManager; -use Piwik\Plugins\UsersManager\UserPreferences; use Piwik\Registry; use Piwik\SettingsPiwik; use Piwik\Site; diff --git a/core/Plugin/Menu.php b/core/Plugin/Menu.php index 0cdc1878df..0ed31e452d 100644 --- a/core/Plugin/Menu.php +++ b/core/Plugin/Menu.php @@ -195,7 +195,7 @@ class Menu $defaultDate = $userPreferences->getDefaultDate(); } if (empty($defaultPeriod)) { - $defaultPeriod = $userPreferences->getDefaultPeriod(); + $defaultPeriod = $userPreferences->getDefaultPeriod($defaultDate); } return array( 'idSite' => $websiteId, diff --git a/core/Site.php b/core/Site.php index deefbd4911..4818590f82 100644 --- a/core/Site.php +++ b/core/Site.php @@ -231,8 +231,11 @@ class Site */ protected function get($name) { + if (!isset(self::$infoSites[$this->id])) { + throw new UnexpectedWebsiteFoundException('The requested website id = ' . (int)$this->id . ' couldn\'t be found'); + } if (!isset(self::$infoSites[$this->id][$name])) { - throw new Exception('The requested website id = ' . (int)$this->id . ' (or its property ' . $name . ') couldn\'t be found'); + throw new Exception("The property $name could not be found on the website ID " . (int)$this->id); } return self::$infoSites[$this->id][$name]; } diff --git a/plugins/UsersManager/API.php b/plugins/UsersManager/API.php index 1e651eef13..75d9190cd1 100644 --- a/plugins/UsersManager/API.php +++ b/plugins/UsersManager/API.php @@ -32,6 +32,8 @@ use Piwik\Tracker\Cache; */ class API extends \Piwik\Plugin\API { + const OPTION_NAME_PREFERENCE_SEPARATOR = '_'; + /** * @var Model */ @@ -107,9 +109,34 @@ class API extends \Piwik\Plugin\API return $this->getDefaultUserPreference($preferenceName, $userLogin); } + /** + * Returns an array of Preferences + * @param $preferenceNames array of preference names + * @return array + * @ignore + */ + public function getAllUsersPreferences(array $preferenceNames) + { + Piwik::checkUserHasSuperUserAccess(); + + $userPreferences = array(); + foreach($preferenceNames as $preferenceName) { + $optionNameMatchAllUsers = $this->getPreferenceId('%', $preferenceName); + $preferences = Option::getLike($optionNameMatchAllUsers); + + foreach($preferences as $optionName => $optionValue) { + $optionName = explode(self::OPTION_NAME_PREFERENCE_SEPARATOR, $optionName); + $userName = $optionName[0]; + $preference = $optionName[1]; + $userPreferences[$userName][$preference] = $optionValue; + } + } + return $userPreferences; + } + private function getPreferenceId($login, $preference) { - return $login . '_' . $preference; + return $login . self::OPTION_NAME_PREFERENCE_SEPARATOR . $preference; } private function getDefaultUserPreference($preferenceName, $login) diff --git a/plugins/UsersManager/UserPreferences.php b/plugins/UsersManager/UserPreferences.php index 430d2b8b65..b1d70f4fcc 100644 --- a/plugins/UsersManager/UserPreferences.php +++ b/plugins/UsersManager/UserPreferences.php @@ -39,6 +39,7 @@ class UserPreferences return false; } + /** * Returns default site ID that Piwik should load. * @@ -91,27 +92,32 @@ class UserPreferences /** * Returns default period type for Piwik reports. * + * @param $defaultDate string the default date string from which the default period will be guessed * @return string `'day'`, `'week'`, `'month'`, `'year'` or `'range'` * @api */ - public function getDefaultPeriod() + public function getDefaultPeriod($defaultDate) { - $userSettingsDate = APIUsersManager::getInstance()->getUserPreference(Piwik::getCurrentUserLogin(), APIUsersManager::PREFERENCE_DEFAULT_REPORT_DATE); - - if ($userSettingsDate === false) { - return Config::getInstance()->General['default_period']; + $defaultPeriod = Config::getInstance()->General['default_period']; + if ($defaultDate === false) { + return $defaultPeriod; } - if (in_array($userSettingsDate, array('today', 'yesterday'))) { + if (in_array($defaultDate, array('today', 'yesterday'))) { return 'day'; } - if (strpos($userSettingsDate, 'last') === 0 - || strpos($userSettingsDate, 'previous') === 0 + if (strpos($defaultDate, 'last') === 0 + || strpos($defaultDate, 'previous') === 0 ) { return 'range'; } - return $userSettingsDate; + return $defaultPeriod; + } + + public function getAllUsersPreferences() + { + } }
\ No newline at end of file diff --git a/plugins/UsersManager/tests/Integration/APITest.php b/plugins/UsersManager/tests/Integration/APITest.php index 1cf2cf71bb..67a6e9f12e 100644 --- a/plugins/UsersManager/tests/Integration/APITest.php +++ b/plugins/UsersManager/tests/Integration/APITest.php @@ -69,4 +69,43 @@ class APITest extends IntegrationTestCase $this->assertFalse($eventTriggered, 'UsersManager.removeSiteAccess event was triggered but should not'); } + public function test_getAllUsersPreferences_isEmpty_whenNoPreference() + { + $preferences = $this->api->getAllUsersPreferences(array('preferenceName')); + $this->assertEmpty($preferences); + } + + public function test_getAllUsersPreferences_isEmpty_whenNoPreferenceAndMultipleRequested() + { + $preferences = $this->api->getAllUsersPreferences(array('preferenceName', 'otherOne')); + $this->assertEmpty($preferences); + } + + public function test_getAllUsersPreferences_shouldGetMultiplePreferences() + { + $user2 = 'userLogin2'; + $user3 = 'userLogin3'; + $this->api->addUser($user2, 'password', 'userlogin2@password.de'); + $this->api->setUserPreference($user2, 'myPreferenceName', 'valueForUser2'); + $this->api->setUserPreference($user2, 'Random_NOT_REQUESTED', 'Random_NOT_REQUESTED'); + + $this->api->addUser($user3, 'password', 'userlogin3@password.de'); + $this->api->setUserPreference($user3, 'myPreferenceName', 'valueForUser3'); + $this->api->setUserPreference($user3, 'otherPreferenceHere', 'otherPreferenceVALUE'); + $this->api->setUserPreference($user3, 'Random_NOT_REQUESTED', 'Random_NOT_REQUESTED'); + + $expected = array( + $user2 => array( + 'myPreferenceName' => 'valueForUser2' + ), + $user3 => array( + 'myPreferenceName' => 'valueForUser3', + 'otherPreferenceHere' => 'otherPreferenceVALUE', + ), + ); + $result = $this->api->getAllUsersPreferences(array('myPreferenceName', 'otherPreferenceHere', 'randomDoesNotExist')); + + $this->assertSame($expected, $result); + } + } |