diff options
author | Matthieu Napoli <matthieu@mnapoli.fr> | 2014-11-27 03:56:56 +0300 |
---|---|---|
committer | Matthieu Napoli <matthieu@mnapoli.fr> | 2014-11-27 03:56:56 +0300 |
commit | 0c80a9be897972650d3e176943c0c63fcddcd6ca (patch) | |
tree | 7115929216c7969ff167cb5deb17777bb02aa7f2 /core | |
parent | 9ea42bb907d9ddbb1ca7e0868e34b3a46de1973e (diff) | |
parent | 7bab10fc9dd46037e39e5cbb3dcca2b420528051 (diff) |
Merge branch 'di-config' into tmp-path
Conflicts:
tests/PHPUnit/Integration/LogTest.php
Diffstat (limited to 'core')
31 files changed, 574 insertions, 88 deletions
diff --git a/core/ArchiveProcessor/PluginsArchiver.php b/core/ArchiveProcessor/PluginsArchiver.php index e0618f8343..d1bb950d38 100644 --- a/core/ArchiveProcessor/PluginsArchiver.php +++ b/core/ArchiveProcessor/PluginsArchiver.php @@ -96,7 +96,7 @@ class PluginsArchiver $archiver = new $archiverClass($this->archiveProcessor); if (!$archiver->isEnabled()) { - Log::debug("PluginsArchiver::%s: Skipping archiving for plugin '%s'.", __FUNCTION__, $pluginName); + Log::verbose("PluginsArchiver::%s: Skipping archiving for plugin '%s'.", __FUNCTION__, $pluginName); continue; } @@ -111,7 +111,7 @@ class PluginsArchiver $archiver->aggregateMultipleReports(); } } else { - Log::debug("PluginsArchiver::%s: Not archiving reports for plugin '%s'.", __FUNCTION__, $pluginName); + Log::verbose("PluginsArchiver::%s: Not archiving reports for plugin '%s'.", __FUNCTION__, $pluginName); } Manager::getInstance()->deleteAll($latestUsedTableId); diff --git a/core/CliMulti.php b/core/CliMulti.php index aee8d7e35d..8a1bffd773 100644 --- a/core/CliMulti.php +++ b/core/CliMulti.php @@ -152,6 +152,14 @@ class CliMulti { return false; } + $pid = $process->getPid(); + foreach ($this->outputs as $output) { + if ($output->getOutputId() === $pid && $output->isAbnormal()) { + $process->finishProcess(); + return true; + } + } + if ($process->hasFinished()) { // prevent from checking this process over and over again unset($this->processes[$index]); diff --git a/core/CliMulti/Output.php b/core/CliMulti/Output.php index 67ecf3543b..e4feb1a805 100644 --- a/core/CliMulti/Output.php +++ b/core/CliMulti/Output.php @@ -13,6 +13,7 @@ use Piwik\Filesystem; class Output { private $tmpFile = ''; + private $outputId = null; public function __construct($outputId) { @@ -23,7 +24,13 @@ class Output { $dir = CliMulti::getTmpPath(); Filesystem::mkdir($dir); - $this->tmpFile = $dir . '/' . $outputId . '.output'; + $this->tmpFile = $dir . '/' . $outputId . '.output'; + $this->outputId = $outputId; + } + + public function getOutputId() + { + return $this->outputId; } public function write($content) @@ -36,6 +43,13 @@ class Output { return $this->tmpFile; } + public function isAbnormal() + { + $size = Filesystem::getFileSize($this->tmpFile, 'MB'); + + return $size !== null && $size >= 100; + } + public function exists() { return file_exists($this->tmpFile); diff --git a/core/CliMulti/Process.php b/core/CliMulti/Process.php index 15dc539e59..cd75a1e6ef 100644 --- a/core/CliMulti/Process.php +++ b/core/CliMulti/Process.php @@ -24,6 +24,7 @@ class Process private $pidFile = ''; private $timeCreation = null; private $isSupported = null; + private $pid = null; public function __construct($pid) { @@ -37,10 +38,16 @@ class Process $this->isSupported = self::isSupported(); $this->pidFile = $pidDir . '/' . $pid . '.pid'; $this->timeCreation = time(); + $this->pid = $pid; $this->markAsNotStarted(); } + public function getPid() + { + return $this->pid; + } + private function markAsNotStarted() { $content = $this->getPidFileContent(); @@ -97,6 +104,11 @@ class Process return false; } + if (!$this->pidFileSizeIsNormal()) { + $this->finishProcess(); + return false; + } + if ($this->isProcessStillRunning($content)) { return true; } @@ -108,6 +120,13 @@ class Process return false; } + private function pidFileSizeIsNormal() + { + $size = Filesystem::getFileSize($this->pidFile); + + return $size !== null && $size < 500; + } + public function finishProcess() { Filesystem::deleteFileIfExists($this->pidFile); diff --git a/core/Common.php b/core/Common.php index 823e884af1..b5ed581d82 100644 --- a/core/Common.php +++ b/core/Common.php @@ -462,7 +462,7 @@ class Common // we deal w/ json differently if ($varType == 'json') { $value = self::undoMagicQuotes($requestArrayToUse[$varName]); - $value = self::json_decode($value, $assoc = true); + $value = json_decode($value, $assoc = true); return self::sanitizeInputValues($value, $alreadyStripslashed = true); } @@ -471,7 +471,13 @@ class Common $ok = false; if ($varType === 'string') { - if (is_string($value)) $ok = true; + if (is_string($value) || is_int($value)) { + $ok = true; + } else if (is_float($value)) { + $value = Common::forceDotAsSeparatorForDecimalPoint($value); + $ok = true; + } + } elseif ($varType === 'integer') { if ($value == (string)(int)$value) $ok = true; } elseif ($varType === 'float') { @@ -1150,6 +1156,48 @@ class Common } /** + * Sends the given response code if supported. + * + * @param int $code Eg 204 + * + * @throws Exception + */ + public static function sendResponseCode($code) + { + $messages = array( + 200 => 'Ok', + 204 => 'No Response', + 301 => 'Moved Permanently', + 302 => 'Found', + 304 => 'Not Modified', + 400 => 'Bad Request', + 401 => 'Unauthorized', + 403 => 'Forbidden', + 404 => 'Not Found', + 500 => 'Internal Server Error' + ); + + if (!array_key_exists($code, $messages)) { + throw new Exception('Response code not supported: ' . $code); + } + + if (strpos(PHP_SAPI, '-fcgi') === false) { + $key = $_SERVER['SERVER_PROTOCOL']; + + if (strlen($key) > 15 || empty($key)) { + $key = 'HTTP/1.1'; + } + + } else { + // FastCGI + $key = 'Status:'; + } + + $message = $messages[$code]; + Common::sendHeader($key . ' ' . $code . ' ' . $message); + } + + /** * Returns the ID of the current LocationProvider (see UserCountry plugin code) from * the Tracker cache. */ diff --git a/core/CronArchive.php b/core/CronArchive.php index 5cf42664a4..151298fc6a 100644 --- a/core/CronArchive.php +++ b/core/CronArchive.php @@ -453,9 +453,9 @@ class CronArchive // (*) If there was some old reports invalidated for this website // we make sure all these old reports are triggered at least once - $websiteIsOldDataInvalidate = $this->isOldReportInvalidatedForWebsite($idSite); + $websiteInvalidatedShouldReprocess = $this->isOldReportInvalidatedForWebsite($idSite); - if ($websiteIsOldDataInvalidate) { + if ($websiteInvalidatedShouldReprocess) { $shouldArchivePeriods = true; } @@ -473,7 +473,7 @@ class CronArchive $skipDayArchive = $existingArchiveIsValid; // Invalidate old website forces the archiving for this site - $skipDayArchive = $skipDayArchive && !$websiteIsOldDataInvalidate; + $skipDayArchive = $skipDayArchive && !$websiteInvalidatedShouldReprocess; // Also reprocess when day has ended since last run if ($dayHasEndedMustReprocess @@ -527,6 +527,14 @@ class CronArchive Option::set($this->lastRunKey($idSite, "periods"), time()); } + if(!$success) { + // cancel marking the site as reprocessed + if($websiteInvalidatedShouldReprocess) { + $store = new InvalidatedReports(); + $store->addInvalidatedSitesToReprocess(array($idSite)); + } + } + $this->archivedPeriodsArchivesWebsite++; $requestsWebsite = $this->requests - $requestsBefore; diff --git a/core/DataArray.php b/core/DataArray.php index 7515ef8699..2eab0bbab5 100644 --- a/core/DataArray.php +++ b/core/DataArray.php @@ -99,6 +99,11 @@ class DataArray return; } + // Edge case fail safe + if(!isset($oldRowToUpdate[Metrics::INDEX_NB_VISITS])) { + return; + } + $oldRowToUpdate[Metrics::INDEX_NB_VISITS] += $newRowToAdd[Metrics::INDEX_NB_VISITS]; $oldRowToUpdate[Metrics::INDEX_NB_ACTIONS] += $newRowToAdd[Metrics::INDEX_NB_ACTIONS]; $oldRowToUpdate[Metrics::INDEX_NB_UNIQ_VISITORS] += $newRowToAdd[Metrics::INDEX_NB_UNIQ_VISITORS]; diff --git a/core/Db.php b/core/Db.php index 556bd2ada5..fa26f34f02 100644 --- a/core/Db.php +++ b/core/Db.php @@ -46,7 +46,7 @@ class Db return Tracker::getDatabase(); } - if (self::$connection === null) { + if (!self::hasDatabaseObject()) { self::createDatabaseObject(); } @@ -104,6 +104,16 @@ class Db } /** + * Detect whether a database object is initialized / created or not. + * + * @internal + */ + public static function hasDatabaseObject() + { + return isset(self::$connection); + } + + /** * Disconnects and destroys the database connection. * * For tests. diff --git a/core/Exception/InvalidRequestParameterException.php b/core/Exception/InvalidRequestParameterException.php new file mode 100644 index 0000000000..13ead2e716 --- /dev/null +++ b/core/Exception/InvalidRequestParameterException.php @@ -0,0 +1,13 @@ +<?php +/** + * Piwik - free/libre analytics platform + * + * @link http://piwik.org + * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later + * + */ +namespace Piwik\Exception; + +class InvalidRequestParameterException extends Exception +{ +}
\ No newline at end of file diff --git a/core/Exception/UnexpectedWebsiteFoundException.php b/core/Exception/UnexpectedWebsiteFoundException.php new file mode 100644 index 0000000000..68178d8599 --- /dev/null +++ b/core/Exception/UnexpectedWebsiteFoundException.php @@ -0,0 +1,13 @@ +<?php +/** + * Piwik - free/libre analytics platform + * + * @link http://piwik.org + * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later + * + */ +namespace Piwik\Exception; + +class UnexpectedWebsiteFoundException extends Exception +{ +}
\ No newline at end of file diff --git a/core/Filesystem.php b/core/Filesystem.php index ffeef1f3be..125c47bfda 100644 --- a/core/Filesystem.php +++ b/core/Filesystem.php @@ -375,6 +375,40 @@ class Filesystem } /** + * Get the size of a file in the specified unit. + * + * @param string $pathToFile + * @param string $unit eg 'B' for Byte, 'KB', 'MB', 'GB', 'TB'. + * + * @return float|null Returns null if file does not exist or the size of the file in the specified unit + * + * @throws Exception In case the unit is invalid + */ + public static function getFileSize($pathToFile, $unit = 'B') + { + $unit = strtoupper($unit); + $units = array('TB' => pow(1024, 4), + 'GB' => pow(1024, 3), + 'MB' => pow(1024, 2), + 'KB' => 1024, + 'B' => 1); + + if (!array_key_exists($unit, $units)) { + throw new Exception('Invalid unit given'); + } + + if (!file_exists($pathToFile)) { + return; + } + + $filesize = filesize($pathToFile); + $factor = $units[$unit]; + $converted = $filesize / $factor; + + return $converted; + } + + /** * @param $path * @return int */ diff --git a/core/Mail.php b/core/Mail.php index 179b1221ab..1e9ef0bb52 100644 --- a/core/Mail.php +++ b/core/Mail.php @@ -107,7 +107,7 @@ class Mail extends Zend_Mail $tr = new \Zend_Mail_Transport_Smtp($mailConfig['host'], $smtpConfig); Mail::setDefaultTransport($tr); - ini_set("smtp_port", $mailConfig['port']); + @ini_set("smtp_port", $mailConfig['port']); } public function send($transport = NULL) diff --git a/core/Piwik.php b/core/Piwik.php index 625a7cf9f8..80ed01cd8c 100644 --- a/core/Piwik.php +++ b/core/Piwik.php @@ -468,6 +468,7 @@ class Piwik * in case another Login plugin is being used. * * @return string + * @api */ public static function getLoginPluginName() { diff --git a/core/Plugin/MetadataLoader.php b/core/Plugin/MetadataLoader.php index 34bb90dcdc..e75bc58338 100644 --- a/core/Plugin/MetadataLoader.php +++ b/core/Plugin/MetadataLoader.php @@ -95,7 +95,7 @@ class MetadataLoader return array(); } - $info = Common::json_decode($json, $assoc = true); + $info = json_decode($json, $assoc = true); if (!is_array($info) || empty($info) ) { diff --git a/core/Plugin/Settings.php b/core/Plugin/Settings.php index 57f8c813fb..0b0bb0953a 100644 --- a/core/Plugin/Settings.php +++ b/core/Plugin/Settings.php @@ -58,6 +58,9 @@ abstract class Settings implements StorageInterface private $introduction; private $pluginName; + // for lazy loading of setting values + private $settingValuesLoaded = false; + /** * Constructor. */ @@ -76,7 +79,6 @@ abstract class Settings implements StorageInterface } $this->init(); - $this->loadSettings(); } /** @@ -126,7 +128,7 @@ abstract class Settings implements StorageInterface }); $settings2 = $settings; - + uasort($settings, function ($setting1, $setting2) use ($settings2) { /** @var Setting $setting1 */ /** @var Setting $setting2 */ @@ -166,6 +168,8 @@ abstract class Settings implements StorageInterface */ public function save() { + $this->loadSettingsIfNotDoneYet(); + Option::set($this->getOptionKey(), serialize($this->settingsValues)); $pluginName = $this->getPluginName(); @@ -195,6 +199,7 @@ abstract class Settings implements StorageInterface Option::delete($this->getOptionKey()); $this->settingsValues = array(); + $this->settingValuesLoaded = false; } /** @@ -210,6 +215,7 @@ abstract class Settings implements StorageInterface { $this->checkIsValidSetting($setting->getName()); $this->checkHasEnoughReadPermission($setting); + $this->loadSettingsIfNotDoneYet(); if (array_key_exists($setting->getKey(), $this->settingsValues)) { @@ -247,6 +253,7 @@ abstract class Settings implements StorageInterface settype($value, $setting->type); } + $this->loadSettingsIfNotDoneYet(); $this->settingsValues[$setting->getKey()] = $value; } @@ -259,6 +266,7 @@ abstract class Settings implements StorageInterface public function removeSettingValue(Setting $setting) { $this->checkHasEnoughWritePermission($setting); + $this->loadSettingsIfNotDoneYet(); $key = $setting->getKey(); @@ -298,6 +306,16 @@ abstract class Settings implements StorageInterface return 'Plugin_' . $this->pluginName . '_Settings'; } + private function loadSettingsIfNotDoneYet() + { + if ($this->settingValuesLoaded) { + return; + } + + $this->settingValuesLoaded = true; + $this->loadSettings(); + } + private function loadSettings() { $values = Option::get($this->getOptionKey()); @@ -412,7 +430,7 @@ abstract class Settings implements StorageInterface $setting->validate = function ($value) use ($setting, $pluginName) { $errorMsg = Piwik::translate('CoreAdminHome_PluginSettingsValueNotAllowed', - array($setting->title, $pluginName)); + array($setting->title, $pluginName)); if (is_array($value) && $setting->type == Settings::TYPE_ARRAY) { foreach ($value as $val) { diff --git a/core/ProxyHttp.php b/core/ProxyHttp.php index 69f7144b44..b2939e9a39 100644 --- a/core/ProxyHttp.php +++ b/core/ProxyHttp.php @@ -66,7 +66,7 @@ class ProxyHttp { // if the file cannot be found return HTTP status code '404' if (!file_exists($file)) { - self::setHttpStatus('404 Not Found'); + Common::sendResponseCode(404); return; } @@ -87,7 +87,7 @@ class ProxyHttp // Return 304 if the file has not modified since if ($modifiedSince === $lastModified) { - self::setHttpStatus('304 Not Modified'); + Common::sendResponseCode(304); return; } @@ -158,7 +158,7 @@ class ProxyHttp } if (!_readfile($file, $byteStart, $byteEnd)) { - self::setHttpStatus('505 Internal server error'); + Common::sendResponseCode(500); } } @@ -220,24 +220,6 @@ class ProxyHttp } /** - * Set response header, e.g., HTTP/1.0 200 Ok - * - * @param string $status Status - * @return bool - */ - protected static function setHttpStatus($status) - { - if (strpos(PHP_SAPI, '-fcgi') === false) { - $key = $_SERVER['SERVER_PROTOCOL']; - } else { - // FastCGI - $key = 'Status:'; - } - - Common::sendHeader($key . ' ' . $status); - } - - /** * Returns a formatted Expires HTTP header for a certain number of days in the future. The result * can be used in a call to `header()`. */ diff --git a/core/Sequence.php b/core/Sequence.php index fa95b0b07c..7a25c23959 100644 --- a/core/Sequence.php +++ b/core/Sequence.php @@ -22,6 +22,10 @@ use Piwik\Db\AdapterInterface; */ class Sequence { + const TABLE_NAME = 'sequence'; + /** + * @var string + */ private $name; /** @@ -30,20 +34,22 @@ class Sequence private $db; /** + * @var string + */ + private $table; + + /** * The name of the table or sequence you want to get an id for. * * @param string $name eg 'archive_numeric_2014_11' * @param AdapterInterface $db You can optionally pass a DB adapter to make it work against another database. + * @param string|null $tablePrefix */ - public function __construct($name, $db = null) + public function __construct($name, $db = null, $tablePrefix = null) { $this->name = $name; $this->db = $db ?: Db::get(); - } - - private function getTableName() - { - return Common::prefixTable('sequence'); + $this->table = $this->getTableName($tablePrefix); } /** @@ -58,9 +64,7 @@ class Sequence { $initialValue = (int) $initialValue; - $table = $this->getTableName(); - - $this->db->insert($table, array('name' => $this->name, 'value' => $initialValue)); + $this->db->insert($this->table, array('name' => $this->name, 'value' => $initialValue)); return $initialValue; } @@ -72,7 +76,7 @@ class Sequence */ public function exists() { - $query = $this->db->query('SELECT * FROM ' . $this->getTableName() . ' WHERE name = ?', $this->name); + $query = $this->db->query('SELECT * FROM ' . $this->table . ' WHERE name = ?', $this->name); return $query->rowCount() > 0; } @@ -86,8 +90,7 @@ class Sequence */ public function getNextId() { - $table = $this->getTableName(); - $sql = 'UPDATE ' . $table . ' SET value = LAST_INSERT_ID(value + 1) WHERE name = ?'; + $sql = 'UPDATE ' . $this->table . ' SET value = LAST_INSERT_ID(value + 1) WHERE name = ?'; $result = $this->db->query($sql, array($this->name)); $rowCount = $result->rowCount(); @@ -108,8 +111,7 @@ class Sequence */ public function getCurrentId() { - $table = $this->getTableName(); - $sql = 'SELECT value FROM ' . $table . ' WHERE name = ?'; + $sql = 'SELECT value FROM ' . $this->table . ' WHERE name = ?'; $id = $this->db->fetchOne($sql, array($this->name)); @@ -117,4 +119,9 @@ class Sequence return (int) $id; } } + + private function getTableName($prefix) + { + return ($prefix !== null) ? $prefix . self::TABLE_NAME : Common::prefixTable(self::TABLE_NAME); + } } diff --git a/core/SettingsPiwik.php b/core/SettingsPiwik.php index 1d972a7f88..1291c7fe13 100644 --- a/core/SettingsPiwik.php +++ b/core/SettingsPiwik.php @@ -331,6 +331,17 @@ class SettingsPiwik } } + /** + * Returns true if Piwik is deployed using git + * FAQ: http://piwik.org/faq/how-to-install/faq_18271/ + * + * @return bool + */ + public static function isGitDeployment() + { + return file_exists(PIWIK_INCLUDE_PATH . '/.git/HEAD'); + } + public static function getCurrentGitBranch() { $file = PIWIK_INCLUDE_PATH . '/.git/HEAD'; @@ -417,4 +428,5 @@ class SettingsPiwik { return Config::getInstance()->General['force_ssl'] == 1; } + } diff --git a/core/Site.php b/core/Site.php index 96b7316b76..deefbd4911 100644 --- a/core/Site.php +++ b/core/Site.php @@ -10,6 +10,7 @@ namespace Piwik; use Exception; +use Piwik\Exception\UnexpectedWebsiteFoundException; use Piwik\Plugins\SitesManager\API; /** @@ -95,7 +96,7 @@ class Site protected static function setSite($idSite, $infoSite) { if (empty($idSite) || empty($infoSite)) { - throw new Exception("An unexpected website was found, check idSite in the request."); + throw new UnexpectedWebsiteFoundException("An unexpected website was found, check idSite in the request."); } /** diff --git a/core/Tracker.php b/core/Tracker.php index ed911e55f6..d6b2d9236f 100644 --- a/core/Tracker.php +++ b/core/Tracker.php @@ -9,6 +9,8 @@ namespace Piwik; use Exception; +use Piwik\Exception\InvalidRequestParameterException; +use Piwik\Exception\UnexpectedWebsiteFoundException; use Piwik\Plugins\PrivacyManager\Config as PrivacyManagerConfig; use Piwik\Plugins\SitesManager\SiteUrls; use Piwik\Tracker\Cache; @@ -412,15 +414,16 @@ class Tracker * * @param Exception $e * @param bool $authenticated + * @param int $statusCode eg 500 */ - protected function exitWithException($e, $authenticated = false) + protected function exitWithException($e, $authenticated = false, $statusCode = 500) { if ($this->hasRedirectUrl()) { $this->performRedirectToUrlIfSet(); exit; } - Common::sendHeader('HTTP/1.1 500 Internal Server Error'); + Common::sendResponseCode($statusCode); error_log(sprintf("Error in Piwik (tracker): %s", str_replace("\n", " ", $this->getMessageFromException($e)))); if ($this->usingBulkTracking) { @@ -436,7 +439,7 @@ class Tracker $result['message'] = $this->getMessageFromException($e); } Common::sendHeader('Content-Type: application/json'); - echo Common::json_encode($result); + echo json_encode($result); die(1); exit; } @@ -454,8 +457,9 @@ class Tracker Common::sendHeader('Content-Type: text/html; charset=utf-8'); echo $this->getMessageFromException($e); } else { - $this->outputTransparentGif(); + $this->sendResponse(); } + die(1); exit; } @@ -496,12 +500,12 @@ class Tracker $this->outputAccessControlHeaders(); Common::sendHeader('Content-Type: application/json'); - echo Common::json_encode($result); + echo json_encode($result); exit; } switch ($this->getState()) { case self::STATE_LOGGING_DISABLE: - $this->outputTransparentGif (); + $this->sendResponse(); Common::printDebug("Logging disabled, display transparent logo"); break; @@ -513,7 +517,7 @@ class Tracker case self::STATE_NOSCRIPT_REQUEST: case self::STATE_NOTHING_TO_NOTICE: default: - $this->outputTransparentGif (); + $this->sendResponse(); Common::printDebug("Nothing to notice => default behaviour"); break; } @@ -648,7 +652,7 @@ class Tracker return $visit; } - protected function outputTransparentGif () + private function sendResponse() { if (isset($GLOBALS['PIWIK_TRACKER_DEBUG']) && $GLOBALS['PIWIK_TRACKER_DEBUG'] @@ -660,11 +664,25 @@ class Tracker // If there was an error during tracker, return so errors can be flushed return; } - $transGifBase64 = "R0lGODlhAQABAIAAAAAAAAAAACH5BAEAAAAALAAAAAABAAEAAAICRAEAOw=="; - Common::sendHeader('Content-Type: image/gif'); $this->outputAccessControlHeaders(); + $request = $_GET + $_POST; + + if (array_key_exists('send_image', $request) && $request['send_image'] === '0') { + Common::sendResponseCode(204); + + return; + } + + $this->outputTransparentGif(); + } + + protected function outputTransparentGif () + { + $transGifBase64 = "R0lGODlhAQABAIAAAAAAAAAAACH5BAEAAAAALAAAAAABAAEAAAICRAEAOw=="; + Common::sendHeader('Content-Type: image/gif'); + print(base64_decode($transGifBase64)); } @@ -828,6 +846,12 @@ class Tracker } else { Common::printDebug("The request is invalid: empty request, or maybe tracking is disabled in the config.ini.php via record_statistics=0"); } + } catch (UnexpectedWebsiteFoundException $e) { + Common::printDebug("Exception: " . $e->getMessage()); + $this->exitWithException($e, $isAuthenticated, 400); + } catch (InvalidRequestParameterException $e) { + Common::printDebug("Exception: " . $e->getMessage()); + $this->exitWithException($e, $isAuthenticated, 400); } catch (DbException $e) { Common::printDebug("Exception: " . $e->getMessage()); $this->exitWithException($e, $isAuthenticated); diff --git a/core/Tracker/Request.php b/core/Tracker/Request.php index b733ae30ea..8853800fa6 100644 --- a/core/Tracker/Request.php +++ b/core/Tracker/Request.php @@ -12,6 +12,8 @@ use Exception; use Piwik\Common; use Piwik\Config; use Piwik\Cookie; +use Piwik\Exception\InvalidRequestParameterException; +use Piwik\Exception\UnexpectedWebsiteFoundException; use Piwik\IP; use Piwik\Network\IPUtils; use Piwik\Piwik; @@ -390,7 +392,7 @@ class Request Piwik::postEvent('Tracker.Request.getIdSite', array(&$idSite, $this->params)); if ($idSite <= 0) { - throw new Exception('Invalid idSite: \'' . $idSite . '\''); + throw new UnexpectedWebsiteFoundException('Invalid idSite: \'' . $idSite . '\''); } return $idSite; @@ -523,7 +525,7 @@ class Request $idVisitor = $this->getForcedVisitorId(); if (!empty($idVisitor)) { if (strlen($idVisitor) != Tracker::LENGTH_HEX_ID_STRING) { - throw new Exception("Visitor ID (cid) $idVisitor must be " . Tracker::LENGTH_HEX_ID_STRING . " characters long"); + throw new InvalidRequestParameterException("Visitor ID (cid) $idVisitor must be " . Tracker::LENGTH_HEX_ID_STRING . " characters long"); } Common::printDebug("Request will be recorded for this idvisitor = " . $idVisitor); $found = true; diff --git a/core/Tracker/Visitor.php b/core/Tracker/Visitor.php index afec61849e..d9aadb640e 100644 --- a/core/Tracker/Visitor.php +++ b/core/Tracker/Visitor.php @@ -204,20 +204,17 @@ class Visitor $fields[] = $dimension->getColumnName(); } - /** - * This event collects a list of [visit entity]() properties that should be loaded when reading - * the existing visit. Properties that appear in this list will be available in other tracking - * events such as 'onExistingVisit'. - * - * Plugins can use this event to load additional visit entity properties for later use during tracking. - */ foreach ($dimension->getRequiredVisitFields() as $field) { $fields[] = $field; } } /** - * @ignore + * This event collects a list of [visit entity](/guides/persistence-and-the-mysql-backend#visits) properties that should be loaded when reading + * the existing visit. Properties that appear in this list will be available in other tracking + * events such as 'onExistingVisit'. + * + * Plugins can use this event to load additional visit entity properties for later use during tracking. */ Piwik::postEvent('Tracker.getVisitFieldsToPersist', array(&$fields)); diff --git a/core/Translate.php b/core/Translate.php index 65381c0a2d..c29d0f0339 100644 --- a/core/Translate.php +++ b/core/Translate.php @@ -173,7 +173,7 @@ class Translate $clientSideTranslations[$key] = $translations[$plugin][$stringName]; } - $js = 'var translations = ' . Common::json_encode($clientSideTranslations) . ';'; + $js = 'var translations = ' . json_encode($clientSideTranslations) . ';'; $js .= "\n" . 'if (typeof(piwik_translations) == \'undefined\') { var piwik_translations = new Object; }' . 'for(var i in translations) { piwik_translations[i] = translations[i];} '; return $js; diff --git a/core/Translate/Filter/ByParameterCount.php b/core/Translate/Filter/ByParameterCount.php index 74be50d26d..357ab5ba33 100644 --- a/core/Translate/Filter/ByParameterCount.php +++ b/core/Translate/Filter/ByParameterCount.php @@ -43,7 +43,8 @@ class ByParameterCount extends FilterAbstract if (isset($this->baseTranslations[$pluginName][$key])) { $baseTranslation = $this->baseTranslations[$pluginName][$key]; } else { - $baseTranslation = ''; + // english string was deleted, do not error + continue; } // ensure that translated strings have the same number of %s as the english source strings diff --git a/core/Twig.php b/core/Twig.php index bf840950a5..1ce277e235 100755 --- a/core/Twig.php +++ b/core/Twig.php @@ -11,7 +11,6 @@ namespace Piwik; use Exception; use Piwik\Container\StaticContainer; use Piwik\DataTable\Filter\SafeDecodeLabel; -use Piwik\Period\Range; use Piwik\Translate; use Piwik\View\RenderTokenParser; use Piwik\Visualization\Sparkline; @@ -245,7 +244,7 @@ class Twig protected function addFilter_prettyDate() { $prettyDate = new Twig_SimpleFilter('prettyDate', function ($dateString, $period) { - return Range::factory($period, $dateString)->getLocalizedShortString(); + return Period\Factory::build($period, $dateString)->getLocalizedShortString(); }); $this->twig->addFilter($prettyDate); } diff --git a/core/Updates/1.8.3-b1.php b/core/Updates/1.8.3-b1.php index 7a00ec20b3..ee8e6c4d00 100644 --- a/core/Updates/1.8.3-b1.php +++ b/core/Updates/1.8.3-b1.php @@ -98,8 +98,8 @@ class Updates_1_8_3_b1 extends Updates is_null($period) ? ScheduledReports::DEFAULT_PERIOD : $period, ScheduledReports::EMAIL_TYPE, is_null($format) ? ScheduledReports::DEFAULT_REPORT_FORMAT : $format, - Common::json_encode(preg_split('/,/', $reports)), - Common::json_encode($parameters), + json_encode(preg_split('/,/', $reports)), + json_encode($parameters), $ts_created, $ts_last_sent, $deleted diff --git a/core/Updates/2.0.3-b7.php b/core/Updates/2.0.3-b7.php index add119e462..9903d999c1 100644 --- a/core/Updates/2.0.3-b7.php +++ b/core/Updates/2.0.3-b7.php @@ -23,9 +23,11 @@ class Updates_2_0_3_b7 extends Updates $errors = array(); try { + $checker = new DoNotTrackHeaderChecker(); + // enable DoNotTrack check in PrivacyManager if DoNotTrack plugin was enabled if (\Piwik\Plugin\Manager::getInstance()->isPluginActivated('DoNotTrack')) { - DoNotTrackHeaderChecker::activate(); + $checker->activate(); } // enable IP anonymization if AnonymizeIP plugin was enabled diff --git a/core/Updates/2.10.0-b1.php b/core/Updates/2.10.0-b1.php new file mode 100644 index 0000000000..6a739b6468 --- /dev/null +++ b/core/Updates/2.10.0-b1.php @@ -0,0 +1,252 @@ +<?php +/** + * Piwik - free/libre analytics platform + * + * @link http://piwik.org + * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later + * + */ + +namespace Piwik\Updates; + +use Piwik\Common; +use Piwik\DataAccess\ArchiveTableCreator; +use Piwik\DataTable; +use Piwik\Db; +use Piwik\Updater; +use Piwik\Updates; +use DeviceDetector\Parser\Client\Browser AS BrowserParser; +use Piwik\Plugins\Dashboard\Model AS DashboardModel; + +/** + * This Update script will update all browser and os archives of UserSettings and DevicesDetection plugin + * + * In the future only DevicesDetection will handle browser and os archives, so we try to rename all existing archives + * of UserSettings plugin to their corresponding archive name in DevicesDetection plugin: + * - *UserSettings_browser* will now be *DevicesDetection_browserVersions* + * - *UserSettings_os* will now be *DevicesDetection_osVersions* + * + * Unlike DevicesDetection plugin, the UserSettings plugin did not store archives holding the os and browser data without + * their version number. The "version-less" reports were always generated out of the "version-containing" archives . + * For big archives (month/year) that ment that some of the data was truncated, due to the datatable entry limit. + * To avoid that data loss / inaccuracy in the future, DevicesDetection plugin will also store archives without the version. + * For data archived after DevicesDetection plugin was enabled, those archive already exist. As we are removing the + * UserSettings reports, we need to move the existing old data to the new archives, which means we need to build up + * those archives, where they do not exist. + * + * NOTE: Some archives might not contain "all" data. + * That might have happened directly after the day DevicesDetection plugin was enabled. For the days before, there were + * no archives calculated. So week/month/year archives will only contain data for the days, where archives were generated + * To find a date after which it is safe to use DevicesDetection archives we need to find the first day-archive that + * contains DevicesDetection data. Day archives will always contain full data, but week/month/year archives may not. + * So we need to recreate those week/month/year archives. + */ +class Updates_2_10_0_b1 extends Updates +{ + + static function getSql() + { + $sqls = array('# ATTENTION: This update script will execute some more SQL queries than that below as it is necessary to rebuilt some archives #' => false); + + // update scheduled reports to use new plugin + $reportsToReplace = array( + 'UserSettings_getBrowserVersion' => 'DevicesDetection_getBrowserVersions', + 'UserSettings_getBrowser' => 'DevicesDetection_getBrowsers', + 'UserSettings_getOSFamily' => 'DevicesDetection_getOsFamilies', + 'UserSettings_getOS' => 'DevicesDetection_getOsVersions', + 'UserSettings_getMobileVsDesktop' => 'DevicesDetection_getType', + 'UserSettings_getBrowserType' => 'DevicesDetection_getBrowserEngines', + 'UserSettings_getWideScreen' => 'UserSettings_getScreenType', + ); + + foreach ($reportsToReplace as $old => $new) { + $sqls["UPDATE " . Common::prefixTable('report') . " SET reports = REPLACE(reports, '".$old."', '".$new."')"] = false; + } + + // update dashboard to use new widgets + $oldWidgets = array( + array('module' => 'UserSettings', 'action' => 'getBrowserVersion', 'params' => array()), + array('module' => 'UserSettings', 'action' => 'getBrowser', 'params' => array()), + array('module' => 'UserSettings', 'action' => 'getOSFamily', 'params' => array()), + array('module' => 'UserSettings', 'action' => 'getOS', 'params' => array()), + array('module' => 'UserSettings', 'action' => 'getMobileVsDesktop', 'params' => array()), + array('module' => 'UserSettings', 'action' => 'getBrowserType', 'params' => array()), + array('module' => 'UserSettings', 'action' => 'getWideScreen', 'params' => array()), + ); + + $newWidgets = array( + array('module' => 'DevicesDetection', 'action' => 'getBrowserVersions', 'params' => array()), + array('module' => 'DevicesDetection', 'action' => 'getBrowsers', 'params' => array()), + array('module' => 'DevicesDetection', 'action' => 'getOsFamilies', 'params' => array()), + array('module' => 'DevicesDetection', 'action' => 'getOsVersions', 'params' => array()), + array('module' => 'DevicesDetection', 'action' => 'getType', 'params' => array()), + array('module' => 'DevicesDetection', 'action' => 'getBrowserEngines', 'params' => array()), + array('module' => 'UserSettings', 'action' => 'getScreenType', 'params' => array()), + ); + + $allDashboards = Db::get()->fetchAll(sprintf("SELECT * FROM %s", Common::prefixTable('user_dashboard'))); + + foreach($allDashboards AS $dashboard) { + + $dashboardLayout = json_decode($dashboard['layout']); + + $dashboardLayout = DashboardModel::replaceDashboardWidgets($dashboardLayout, $oldWidgets, $newWidgets); + + $newLayout = json_encode($dashboardLayout); + if ($newLayout != $dashboard['layout']) { + $sqls["UPDATE " . Common::prefixTable('user_dashboard') . " SET layout = '".addslashes($newLayout)."' WHERE iddashboard = ".$dashboard['iddashboard']] = false; + } + } + + return $sqls; + } + + static function update() + { + Updater::updateDatabase(__FILE__, self::getSql()); + + $archiveBlobTables = self::getAllArchiveBlobTables(); + + foreach ($archiveBlobTables as $table) { + self::updateBrowserArchives($table); + self::updateOsArchives($table); + } + } + + /** + * Returns all available archive blob tables + * + * @return array + */ + public static function getAllArchiveBlobTables() + { + static $archiveBlobTables; + + if (empty($archiveBlobTables)) { + + $archiveTables = ArchiveTableCreator::getTablesArchivesInstalled(); + + $archiveBlobTables = array_filter($archiveTables, function($name) { + return ArchiveTableCreator::getTypeFromTableName($name) == ArchiveTableCreator::BLOB_TABLE; + }); + + // sort tables so we have them in order of their date + rsort($archiveBlobTables); + } + + return (array) $archiveBlobTables; + } + + /** + * Find the first day on which DevicesDetection archives were generated + * + * @return int Timestamp + */ + public static function getFirstDayOfArchivedDeviceDetectorData() + { + static $deviceDetectionBlobAvailableDate; + + if (empty($deviceDetectionBlobAvailableDate)) { + + $archiveBlobTables = self::getAllArchiveBlobTables(); + + $deviceDetectionBlobAvailableDate = null; + foreach ($archiveBlobTables as $table) { + + // Look for all day archives and try to find that with the lowest date + $deviceDetectionBlobAvailableDate = Db::get()->fetchOne(sprintf("SELECT date1 FROM %s WHERE name = 'DevicesDetection_browserVersions' AND period = 1 ORDER BY date1 ASC LIMIT 1", $table)); + + if (!empty($deviceDetectionBlobAvailableDate)) { + break; + } + + } + + $deviceDetectionBlobAvailableDate = strtotime($deviceDetectionBlobAvailableDate); + } + + return $deviceDetectionBlobAvailableDate; + } + + /** + * Updates all browser archives to new structure + * @param string $table + * @throws \Exception + */ + public static function updateBrowserArchives($table) + { + // rename old UserSettings archives where no DeviceDetection archives exists + Db::exec(sprintf("UPDATE IGNORE %s SET name='DevicesDetection_browserVersions' WHERE name = 'UserSettings_browser'", $table)); + + /* + * check dates of remaining (non-day) archives with calculated safe date + * archives before or within that week/month/year of that date will be replaced + */ + $oldBrowserBlobs = Db::get()->fetchAll(sprintf("SELECT * FROM %s WHERE name = 'UserSettings_browser' AND `period` > 1", $table)); + foreach ($oldBrowserBlobs as $blob) { + + // if start date of blob is before calculated date us old usersettings archive instead of already existing DevicesDetection archive + if (strtotime($blob['date1']) < self::getFirstDayOfArchivedDeviceDetectorData()) { + + Db::get()->query(sprintf("DELETE FROM %s WHERE idarchive = ? AND name = ?", $table), array($blob['idarchive'], 'DevicesDetection_browserVersions')); + Db::get()->query(sprintf("UPDATE %s SET name = ? WHERE idarchive = ? AND name = ?", $table), array('DevicesDetection_browserVersions', $blob['idarchive'], 'UserSettings_browser')); + } + } + + // rebuild archives without versions + $browserBlobs = Db::get()->fetchAll(sprintf("SELECT * FROM %s WHERE name = 'DevicesDetection_browserVersions'", $table)); + foreach ($browserBlobs as $blob) { + self::createArchiveBlobWithoutVersions($blob, 'DevicesDetection_browsers', $table); + } + } + + public static function updateOsArchives($table) { + Db::exec(sprintf("UPDATE IGNORE %s SET name='DevicesDetection_osVersions' WHERE name = 'UserSettings_os'", $table)); + + /* + * check dates of remaining (non-day) archives with calculated safe date + * archives before or within that week/month/year of that date will be replaced + */ + $oldOsBlobs = Db::get()->fetchAll(sprintf("SELECT * FROM %s WHERE name = 'UserSettings_os' AND `period` > 1", $table)); + foreach ($oldOsBlobs as $blob) { + + // if start date of blob is before calculated date us old usersettings archive instead of already existing DevicesDetection archive + if (strtotime($blob['date1']) < self::getFirstDayOfArchivedDeviceDetectorData()) { + + Db::get()->query(sprintf("DELETE FROM %s WHERE idarchive = ? AND name = ?", $table), array($blob['idarchive'], 'DevicesDetection_osVersions')); + Db::get()->query(sprintf("UPDATE %s SET name = ? WHERE idarchive = ? AND name = ?", $table), array('DevicesDetection_osVersions', $blob['idarchive'], 'UserSettings_os')); + } + } + + // rebuild archives without versions + $osBlobs = Db::get()->fetchAll(sprintf("SELECT * FROM %s WHERE name = 'DevicesDetection_osVersions'", $table)); + foreach ($osBlobs as $blob) { + self::createArchiveBlobWithoutVersions($blob, 'DevicesDetection_os', $table); + } + } + + protected static function createArchiveBlobWithoutVersions($blob, $newName, $table) + { + $blob['value'] = @gzuncompress($blob['value']); + + $datatable = DataTable::fromSerializedArray($blob['value']); + $datatable->filter('GroupBy', array('label', function ($label) { + if (preg_match("/(.+) [0-9]+(?:\.[0-9]+)?$/", $label, $matches)) { + return $matches[1]; // should match for browsers + } + + if (strpos($label, ';')) { + return substr($label, 0, 3); // should match for os + } + + return $label; + })); + + $newData = $datatable->getSerialized(); + + $blob['value'] = @gzcompress($newData[0]); + $blob['name'] = $newName; + + Db::get()->query(sprintf('REPLACE INTO %s (`idarchive`, `name`, `idsite`, `date1`, `date2`, `period`, `ts_archived`, `value`) VALUES (?, ? , ?, ?, ?, ?, ?, ?)', $table), array_values($blob)); + } +} diff --git a/core/Updates/2.9.0-b7.php b/core/Updates/2.9.0-b7.php index 1efcf8360c..110545aaa8 100644 --- a/core/Updates/2.9.0-b7.php +++ b/core/Updates/2.9.0-b7.php @@ -47,7 +47,8 @@ class Updates_2_9_0_b7 extends Updates } $query = self::getQueryToCreateSequence($table, $maxId); - $sql[$query] = false; + // refs #6696, ignores Integrity constraint violation: 1062 Duplicate entry 'piwik_archive_numeric_2010_01' for key 'PRIMARY' + $sql[$query] = '1062'; } } diff --git a/core/Version.php b/core/Version.php index fa7f0f29b7..fcf7e8f40d 100644 --- a/core/Version.php +++ b/core/Version.php @@ -20,5 +20,5 @@ final class Version * The current Piwik version. * @var string */ - const VERSION = '2.9.0'; + const VERSION = '2.10.0-b1'; } diff --git a/core/WidgetsList.php b/core/WidgetsList.php index e67f6b1b77..943dfceb68 100644 --- a/core/WidgetsList.php +++ b/core/WidgetsList.php @@ -159,18 +159,15 @@ class WidgetsList extends Singleton } /** - * Adds a report to the list of dashboard widgets. + * Returns the unique id of an widget with the given parameters * - * @param string $widgetCategory The widget category. This can be a translation token. - * @param string $widgetName The name of the widget. This can be a translation token. - * @param string $controllerName The report's controller name (same as the plugin name). - * @param string $controllerAction The report's controller action method name. - * @param array $customParameters Extra query parameters that should be sent while getting - * this report. + * @param $controllerName + * @param $controllerAction + * @param array $customParameters + * @return string */ - public static function add($widgetCategory, $widgetName, $controllerName, $controllerAction, $customParameters = array()) + public static function getWidgetUniqueId($controllerName, $controllerAction, $customParameters = array()) { - $widgetName = Piwik::translate($widgetName); $widgetUniqueId = 'widget' . $controllerName . $controllerAction; foreach ($customParameters as $name => $value) { @@ -182,6 +179,24 @@ class WidgetsList extends Singleton $widgetUniqueId .= $name . $value; } + return $widgetUniqueId; + } + + /** + * Adds a report to the list of dashboard widgets. + * + * @param string $widgetCategory The widget category. This can be a translation token. + * @param string $widgetName The name of the widget. This can be a translation token. + * @param string $controllerName The report's controller name (same as the plugin name). + * @param string $controllerAction The report's controller action method name. + * @param array $customParameters Extra query parameters that should be sent while getting + * this report. + */ + public static function add($widgetCategory, $widgetName, $controllerName, $controllerAction, $customParameters = array()) + { + $widgetName = Piwik::translate($widgetName); + $widgetUniqueId = self::getWidgetUniqueId($controllerName, $controllerAction, $customParameters); + if (!array_key_exists($widgetCategory, self::$widgets)) { self::$widgets[$widgetCategory] = array(); } |