diff options
author | mattab <matthieu.aubry@gmail.com> | 2013-03-28 03:42:39 +0400 |
---|---|---|
committer | mattab <matthieu.aubry@gmail.com> | 2013-03-28 03:42:40 +0400 |
commit | ae4b03163792f0b6e933933e5d37df87dc3fd566 (patch) | |
tree | d1d7510a9728f587d3d63ebd03e4ecf3d904838b /core/Http.php | |
parent | 158c2150f5f2e13ece459b8d131244c11b763997 (diff) |
Mass conversion of all files to the newly agreed coding standard: PSR 1/2
Converting Piwik core source files, PHP, JS, TPL, CSS
More info: http://piwik.org/participate/coding-standards/
Diffstat (limited to 'core/Http.php')
-rw-r--r-- | core/Http.php | 1427 |
1 files changed, 679 insertions, 748 deletions
diff --git a/core/Http.php b/core/Http.php index 7e921d354d..1b0e328184 100644 --- a/core/Http.php +++ b/core/Http.php @@ -17,752 +17,683 @@ */ class Piwik_Http { - /** - * Get "best" available transport method for sendHttpRequest() calls. - * - * @return string - */ - static public function getTransportMethod() - { - $method = 'curl'; - if(!self::isCurlEnabled()) - { - $method = 'fopen'; - if(@ini_get('allow_url_fopen') != '1') - { - $method = 'socket'; - if(!self::isSocketEnabled()) - { - return null; - } - } - } - return $method; - } - - protected static function isSocketEnabled() - { - return function_exists('fsockopen'); - } - - protected static function isCurlEnabled() - { - return function_exists('curl_init'); - } - - /** - * Sends http request ensuring the request will fail before $timeout seconds - * - * If no $destinationPath is specified, the trimmed response (without header) is returned as a string. - * If a $destinationPath is specified, the response (without header) is saved to a file. - * - * @param string $aUrl - * @param int $timeout - * @param string $userAgent - * @param string $destinationPath - * @param int $followDepth - * @param bool $acceptLanguage - * @param array $byteRange For Range: header. Should be two element array of bytes, eg, array(0, 1024) - * Doesn't work w/ fopen method. - * @param bool $getExtendedInfo True to return status code, headers & response, false if just response. - * @param string $httpMethod The HTTP method to use. Defaults to 'GET'. - * @throws Exception - * @return bool true (or string) on success; false on HTTP response error code (1xx or 4xx) - */ - static public function sendHttpRequest($aUrl, $timeout, $userAgent = null, $destinationPath = null, $followDepth = 0, $acceptLanguage = false, $byteRange = false, $getExtendedInfo = false, $httpMethod = 'GET' -) - { - // create output file - $file = null; - if($destinationPath) - { - // Ensure destination directory exists - Piwik_Common::mkdir(dirname($destinationPath)); - if (($file = @fopen($destinationPath, 'wb')) === false || !is_resource($file)) - { - throw new Exception('Error while creating the file: ' . $destinationPath); - } - } - - $acceptLanguage = $acceptLanguage ? 'Accept-Language: '.$acceptLanguage : ''; - return self::sendHttpRequestBy(self::getTransportMethod(), $aUrl, $timeout, $userAgent, $destinationPath, $file, $followDepth, $acceptLanguage, $acceptInvalidSslCertificate = false, $byteRange, $getExtendedInfo, $httpMethod); - } - - /** - * Sends http request using the specified transport method - * - * @param string $method - * @param string $aUrl - * @param int $timeout - * @param string $userAgent - * @param string $destinationPath - * @param resource $file - * @param int $followDepth - * @param bool|string $acceptLanguage Accept-language header - * @param bool $acceptInvalidSslCertificate Only used with $method == 'curl'. If set to true (NOT recommended!) the SSL certificate will not be checked - * @param array $byteRange For Range: header. Should be two element array of bytes, eg, array(0, 1024) - * Doesn't work w/ fopen method. - * @param bool $getExtendedInfo True to return status code, headers & response, false if just response. - * @param string $httpMethod The HTTP method to use. Defaults to 'GET'. - * @throws Exception - * @return bool true (or string/array) on success; false on HTTP response error code (1xx or 4xx) - */ - static public function sendHttpRequestBy( - $method = 'socket', - $aUrl, - $timeout, - $userAgent = null, - $destinationPath = null, - $file = null, - $followDepth = 0, - $acceptLanguage = false, - $acceptInvalidSslCertificate = false, - $byteRange = false, - $getExtendedInfo = false, - $httpMethod = 'GET' - ) - { - if ($followDepth > 5) - { - throw new Exception('Too many redirects ('.$followDepth.')'); - } - - $contentLength = 0; - $fileLength = 0; - - // Piwik services behave like a proxy, so we should act like one. - $xff = 'X-Forwarded-For: ' - . (isset($_SERVER['HTTP_X_FORWARDED_FOR']) && !empty($_SERVER['HTTP_X_FORWARDED_FOR']) ? $_SERVER['HTTP_X_FORWARDED_FOR'] . ',' : '') - . Piwik_IP::getIpFromHeader(); - - if(empty($userAgent)) - { - $userAgent = self::getUserAgent(); - } - - $via = 'Via: ' - . (isset($_SERVER['HTTP_VIA']) && !empty($_SERVER['HTTP_VIA']) ? $_SERVER['HTTP_VIA'] . ', ' : '') - . Piwik_Version::VERSION . ' ' - . ($userAgent ? " ($userAgent)" : ''); - - // range header - $rangeHeader = ''; - if (!empty($byteRange)) - { - $rangeHeader = 'Range: bytes='.$byteRange[0].'-'.$byteRange[1]."\r\n"; - } - - // proxy configuration - $proxyHost = Piwik_Config::getInstance()->proxy['host']; - $proxyPort = Piwik_Config::getInstance()->proxy['port']; - $proxyUser = Piwik_Config::getInstance()->proxy['username']; - $proxyPassword = Piwik_Config::getInstance()->proxy['password']; - - // other result data - $status = null; - $headers = array(); - - if($method == 'socket') - { - if(!self::isSocketEnabled()) { - // can be triggered in tests - throw new Exception("HTTP socket support is not enabled (php function fsockopen is not available) "); - } - // initialization - $url = @parse_url($aUrl); - if($url === false || !isset($url['scheme'])) - { - throw new Exception('Malformed URL: '.$aUrl); - } - - if($url['scheme'] != 'http') - { - throw new Exception('Invalid protocol/scheme: '.$url['scheme']); - } - $host = $url['host']; - $port = isset($url['port)']) ? $url['port'] : 80; - $path = isset($url['path']) ? $url['path'] : '/'; - if(isset($url['query'])) - { - $path .= '?'.$url['query']; - } - $errno = null; - $errstr = null; - - if ((!empty($proxyHost) && !empty($proxyPort)) - || !empty($byteRange)) - { - $httpVer = '1.1'; - } - else - { - $httpVer = '1.0'; - } - - $proxyAuth = null; - if(!empty($proxyHost) && !empty($proxyPort)) - { - $connectHost = $proxyHost; - $connectPort = $proxyPort; - if(!empty($proxyUser) && !empty($proxyPassword)) - { - $proxyAuth = 'Proxy-Authorization: Basic '.base64_encode("$proxyUser:$proxyPassword") ."\r\n"; - } - $requestHeader = "$httpMethod $aUrl HTTP/$httpVer\r\n"; - } - else - { - $connectHost = $host; - $connectPort = $port; - $requestHeader = "$httpMethod $path HTTP/$httpVer\r\n"; - } - - // connection attempt - if (($fsock = @fsockopen($connectHost, $connectPort, $errno, $errstr, $timeout)) === false || !is_resource($fsock)) - { - if(is_resource($file)) { @fclose($file); } - throw new Exception("Error while connecting to: $host. Please try again later. $errstr"); - } - - // send HTTP request header - $requestHeader .= - "Host: $host".($port != 80 ? ':'.$port : '')."\r\n" - .($proxyAuth ? $proxyAuth : '') - .'User-Agent: '.$userAgent."\r\n" - . ($acceptLanguage ? $acceptLanguage ."\r\n" : '') - .$xff."\r\n" - .$via."\r\n" - .$rangeHeader - ."Connection: close\r\n" - ."\r\n"; - fwrite($fsock, $requestHeader); - - $streamMetaData = array('timed_out' => false); - @stream_set_blocking($fsock, true); - - if (function_exists('stream_set_timeout')) - { - @stream_set_timeout($fsock, $timeout); - } - elseif (function_exists('socket_set_timeout')) - { - @socket_set_timeout($fsock, $timeout); - } - - // process header - $status = null; - $expectRedirect = false; - - while(!feof($fsock)) - { - $line = fgets($fsock, 4096); - - $streamMetaData = @stream_get_meta_data($fsock); - if($streamMetaData['timed_out']) - { - if(is_resource($file)) { @fclose($file); } - @fclose($fsock); - throw new Exception('Timed out waiting for server response'); - } - - // a blank line marks the end of the server response header - if(rtrim($line, "\r\n") == '') - { - break; - } - - // parse first line of server response header - if(!$status) - { - // expect first line to be HTTP response status line, e.g., HTTP/1.1 200 OK - if(!preg_match('~^HTTP/(\d\.\d)\s+(\d+)(\s*.*)?~', $line, $m)) - { - if(is_resource($file)) { @fclose($file); } - @fclose($fsock); - throw new Exception('Expected server response code. Got '.rtrim($line, "\r\n")); - } - - $status = (integer) $m[2]; - - // Informational 1xx or Client Error 4xx - if ($status < 200 || $status >= 400) - { - if(is_resource($file)) { @fclose($file); } - @fclose($fsock); - - if (!$getExtendedInfo) - { - return false; - } - else - { - return array('status' => $status); - } - } - - continue; - } - - // handle redirect - if(preg_match('/^Location:\s*(.+)/', rtrim($line, "\r\n"), $m)) - { - if(is_resource($file)) { @fclose($file); } - @fclose($fsock); - // Successful 2xx vs Redirect 3xx - if($status < 300) - { - throw new Exception('Unexpected redirect to Location: '.rtrim($line).' for status code '.$status); - } - return self::sendHttpRequestBy( - $method, - trim($m[1]), - $timeout, - $userAgent, - $destinationPath, - $file, - $followDepth+1, - $acceptLanguage, - $acceptInvalidSslCertificate = false, - $byteRange, - $getExtendedInfo, - $httpMethod - ); - } - - // save expected content length for later verification - if(preg_match('/^Content-Length:\s*(\d+)/', $line, $m)) - { - $contentLength = (integer) $m[1]; - } - - self::parseHeaderLine($headers, $line); - } - - if (feof($fsock) - && $httpMethod != 'HEAD') - { - throw new Exception('Unexpected end of transmission'); - } - - // process content/body - $response = ''; - - while(!feof($fsock)) - { - $line = fread($fsock, 8192); - - $streamMetaData = @stream_get_meta_data($fsock); - if($streamMetaData['timed_out']) - { - if(is_resource($file)) { @fclose($file); } - @fclose($fsock); - throw new Exception('Timed out waiting for server response'); - } - - $fileLength += Piwik_Common::strlen($line); - - if(is_resource($file)) - { - // save to file - fwrite($file, $line); - } - else - { - // concatenate to response string - $response .= $line; - } - } - - // determine success or failure - @fclose(@$fsock); - } - else if($method == 'fopen') - { - $response = false; - - // we make sure the request takes less than a few seconds to fail - // we create a stream_context (works in php >= 5.2.1) - // we also set the socket_timeout (for php < 5.2.1) - $default_socket_timeout = @ini_get('default_socket_timeout'); - @ini_set('default_socket_timeout', $timeout); - - $ctx = null; - if(function_exists('stream_context_create')) { - $stream_options = array( - 'http' => array( - 'header' => 'User-Agent: '.$userAgent."\r\n" - .($acceptLanguage ? $acceptLanguage."\r\n" : '') - .$xff."\r\n" - .$via."\r\n" - .$rangeHeader, - 'max_redirects' => 5, // PHP 5.1.0 - 'timeout' => $timeout, // PHP 5.2.1 - ) - ); - - if(!empty($proxyHost) && !empty($proxyPort)) - { - $stream_options['http']['proxy'] = 'tcp://'.$proxyHost.':'.$proxyPort; - $stream_options['http']['request_fulluri'] = true; // required by squid proxy - if(!empty($proxyUser) && !empty($proxyPassword)) - { - $stream_options['http']['header'] .= 'Proxy-Authorization: Basic '.base64_encode("$proxyUser:$proxyPassword")."\r\n"; - } - } - - $ctx = stream_context_create($stream_options); - } - - // save to file - if(is_resource($file)) - { - $handle = fopen($aUrl, 'rb', false, $ctx); - while(!feof($handle)) - { - $response = fread($handle, 8192); - $fileLength += Piwik_Common::strlen($response); - fwrite($file, $response); - } - fclose($handle); - } - else - { - $response = file_get_contents($aUrl, 0, $ctx); - $fileLength = Piwik_Common::strlen($response); - } - - // restore the socket_timeout value - if(!empty($default_socket_timeout)) - { - @ini_set('default_socket_timeout', $default_socket_timeout); - } - } - else if($method == 'curl') - { - if(!self::isCurlEnabled()) { - // can be triggered in tests - throw new Exception("CURL is not enabled in php.ini, but is being used."); - } - $ch = @curl_init(); - - if(!empty($proxyHost) && !empty($proxyPort)) - { - @curl_setopt($ch, CURLOPT_PROXY, $proxyHost.':'.$proxyPort); - if(!empty($proxyUser) && !empty($proxyPassword)) - { - // PROXYAUTH defaults to BASIC - @curl_setopt($ch, CURLOPT_PROXYUSERPWD, $proxyUser.':'.$proxyPassword); - } - } - - $curl_options = array( - // internal to ext/curl - CURLOPT_BINARYTRANSFER => is_resource($file), - - // curl options (sorted oldest to newest) - CURLOPT_URL => $aUrl, - CURLOPT_USERAGENT => $userAgent, - CURLOPT_HTTPHEADER => array( - $xff, - $via, - $rangeHeader, - $acceptLanguage - ), - // only get header info if not saving directly to file - CURLOPT_HEADER => is_resource($file) ? false : true, - CURLOPT_CONNECTTIMEOUT => $timeout, - ); - // Case archive.php is triggering archiving on https:// and the certificate is not valid - if($acceptInvalidSslCertificate) - { - $curl_options += array( - CURLOPT_SSL_VERIFYHOST => false, - CURLOPT_SSL_VERIFYPEER => false, - ); - } - @curl_setopt($ch, CURLOPT_CUSTOMREQUEST, $httpMethod); - if ($httpMethod == 'HEAD') - { - @curl_setopt($ch, CURLOPT_NOBODY, true); - } - - @curl_setopt_array($ch, $curl_options); - self::configCurlCertificate($ch); - - - /* - * as of php 5.2.0, CURLOPT_FOLLOWLOCATION can't be set if - * in safe_mode or open_basedir is set - */ - if((string)ini_get('safe_mode') == '' && ini_get('open_basedir') == '') - { - $curl_options = array( - // curl options (sorted oldest to newest) - CURLOPT_FOLLOWLOCATION => true, - CURLOPT_MAXREDIRS => 5, - ); - @curl_setopt_array($ch, $curl_options); - } - - if(is_resource($file)) - { - // write output directly to file - @curl_setopt($ch, CURLOPT_FILE, $file); - } - else - { - // internal to ext/curl - @curl_setopt($ch, CURLOPT_RETURNTRANSFER, true); - } - - ob_start(); - $response = @curl_exec($ch); - ob_end_clean(); - - if($response === true) - { - $response = ''; - } - else if($response === false) - { - $errstr = curl_error($ch); - if($errstr != '') - { - throw new Exception('curl_exec: '.$errstr); - } - $response = ''; - } - else - { - $header = ''; - // redirects are included in the output html, so we look for the last line that starts w/ HTTP/... - // to split the response - while (substr($response, 0, 5) == "HTTP/") - { - list($header, $response) = explode("\r\n\r\n", $response, 2); - } - - foreach (explode("\r\n", $header) as $line) - { - self::parseHeaderLine($headers, $line); - } - } - - $contentLength = @curl_getinfo($ch, CURLINFO_CONTENT_LENGTH_DOWNLOAD); - $fileLength = is_resource($file) ? @curl_getinfo($ch, CURLINFO_SIZE_DOWNLOAD) : Piwik_Common::strlen($response); - $status = @curl_getinfo($ch, CURLINFO_HTTP_CODE); - - @curl_close($ch); - unset($ch); - } - else - { - throw new Exception('Invalid request method: '.$method); - } - - if(is_resource($file)) - { - fflush($file); - @fclose($file); - - $fileSize = filesize($destinationPath); - if((($contentLength > 0) && ($fileLength != $contentLength)) - || ($fileSize != $fileLength)) - { - throw new Exception('File size error: '.$destinationPath.'; expected '.$contentLength.' bytes; received '.$fileLength.' bytes; saved '.$fileSize.' bytes to file'); - } - return true; - } - - if (!$getExtendedInfo) - { - return trim($response); - } - else - { - return array( - 'status' => $status, - 'headers' => $headers, - 'data' => $response - ); - } - } - - /** - * Downloads the next chunk of a specific file. The next chunk's byte range - * is determined by the existing file's size and the expected file size, which - * is stored in the piwik_option table before starting a download. - * - * Note this function uses the Range HTTP header to accomplish downloading in - * parts. - * - * @param string $url The url to download from. - * @param string $outputPath The path to the file to save/append to. - * @param bool $isContinuation True if this is the continuation of a download, - * or if we're starting a fresh one. - */ - public static function downloadChunk( $url, $outputPath, $isContinuation ) - { - // make sure file doesn't already exist if we're starting a new download - if (!$isContinuation - && file_exists($outputPath)) - { - throw new Exception( - Piwik_Translate('General_DownloadFail_FileExists', "'".$outputPath."'") - . ' ' . Piwik_Translate('General_DownloadPleaseRemoveExisting')); - } - - // if we're starting a download, get the expected file size & save as an option - $downloadOption = $outputPath.'_expectedDownloadSize'; - if (!$isContinuation) - { - $expectedFileSizeResult = Piwik_Http::sendHttpRequest( - $url, - $timeout = 300, - $userAgent = null, - $destinationPath = null, - $followDepth = 0, - $acceptLanguage = false, - $byteRange = false, - $getExtendedInfo = true, - $httpMethod = 'HEAD' - ); - - $expectedFileSize = 0; - if (isset($expectedFileSizeResult['headers']['Content-Length'])) - { - $expectedFileSize = (int)$expectedFileSizeResult['headers']['Content-Length']; - } - - if ($expectedFileSize == 0) - { - Piwik::log(sprintf("HEAD request for '%s' failed, got following: %s", $url, print_r($expectedFileSizeResult, true))); - throw new Exception(Piwik_Translate('General_DownloadFail_HttpRequestFail')); - } - - Piwik_SetOption($downloadOption, $expectedFileSize); - } - else - { - $expectedFileSize = (int)Piwik_GetOption($downloadOption); - if ($expectedFileSize === false) // sanity check - { - throw new Exception("Trying to continue a download that never started?! That's not supposed to happen..."); - } - } - - // if existing file is already big enough, then fail so we don't accidentally overwrite - // existing DB - $existingSize = file_exists($outputPath) ? filesize($outputPath) : 0; - if ($existingSize >= $expectedFileSize) - { - throw new Exception( - Piwik_Translate('General_DownloadFail_FileExistsContinue', "'".$outputPath."'") - . ' ' . Piwik_Translate('General_DownloadPleaseRemoveExisting')); - } - - // download a chunk of the file - $result = Piwik_Http::sendHttpRequest( - $url, - $timeout = 300, - $userAgent = null, - $destinationPath = null, - $followDepth = 0, - $acceptLanguage = false, - $byteRange = array($existingSize, min($existingSize + 1024 * 1024 - 1, $expectedFileSize)), - $getExtendedInfo = true - ); - - if ($result === false - || $result['status'] < 200 - || $result['status'] > 299) - { - $result['data'] = self::truncateStr($result['data'], 1024); - Piwik::log("Failed to download range '".$byteRange[0]."-".$byteRange[1] - . "' of file from url '$url'. Got result: ".print_r($result, true)); - - throw new Exception(Piwik_Translate('General_DownloadFail_HttpRequestFail')); - } - - // write chunk to file - $f = fopen($outputPath, 'ab'); - fwrite($f, $result['data']); - fclose($f); - - clearstatcache($clear_realpath_cache = true, $outputPath); - return array( - 'current_size' => filesize($outputPath), - 'expected_file_size' => $expectedFileSize, - ); - } - - /** - * Will configure CURL handle $ch - * to use local list of Certificate Authorities, - */ - public static function configCurlCertificate( &$ch ) - { - if (file_exists(PIWIK_INCLUDE_PATH . '/core/DataFiles/cacert.pem')) - { - @curl_setopt($ch, CURLOPT_CAINFO, PIWIK_INCLUDE_PATH . '/core/DataFiles/cacert.pem'); - } - } - - public static function getUserAgent() - { - return !empty($_SERVER['HTTP_USER_AGENT']) - ? $_SERVER['HTTP_USER_AGENT'] - : 'Piwik/' . Piwik_Version::VERSION; - } - - /** - * Fetch the file at $url in the destination $destinationPath - * - * @param string $url - * @param string $destinationPath - * @param int $tries - * @throws Exception - * @return bool true on success, throws Exception on failure - */ - static public function fetchRemoteFile($url, $destinationPath = null, $tries = 0) - { - @ignore_user_abort(true); - Piwik::setMaxExecutionTime(0); - return self::sendHttpRequest($url, 10, 'Update', $destinationPath, $tries); - } - - /** - * Utility function, parses an HTTP header line into key/value & sets header - * array with them. - * - * @param array $headers - * @param string $line - */ - private static function parseHeaderLine( &$headers, $line ) - { - $parts = explode(':', $line, 2); - if (count($parts) == 1) - { - return; - } - - list($name, $value) = $parts; - $headers[trim($name)] = trim($value); - } - - /** - * Utility function that truncates a string to an arbitrary limit. - * - * @param string $str The string to truncate. - * @param int $limit The maximum length of the truncated string. - * @return string - */ - private static function truncateStr( $str, $limit ) - { - if (strlen($str) > $limit) - { - return substr($str, 0, $limit).'...'; - } - return $str; - } + /** + * Get "best" available transport method for sendHttpRequest() calls. + * + * @return string + */ + static public function getTransportMethod() + { + $method = 'curl'; + if (!self::isCurlEnabled()) { + $method = 'fopen'; + if (@ini_get('allow_url_fopen') != '1') { + $method = 'socket'; + if (!self::isSocketEnabled()) { + return null; + } + } + } + return $method; + } + + protected static function isSocketEnabled() + { + return function_exists('fsockopen'); + } + + protected static function isCurlEnabled() + { + return function_exists('curl_init'); + } + + /** + * Sends http request ensuring the request will fail before $timeout seconds + * + * If no $destinationPath is specified, the trimmed response (without header) is returned as a string. + * If a $destinationPath is specified, the response (without header) is saved to a file. + * + * @param string $aUrl + * @param int $timeout + * @param string $userAgent + * @param string $destinationPath + * @param int $followDepth + * @param bool $acceptLanguage + * @param array $byteRange For Range: header. Should be two element array of bytes, eg, array(0, 1024) + * Doesn't work w/ fopen method. + * @param bool $getExtendedInfo True to return status code, headers & response, false if just response. + * @param string $httpMethod The HTTP method to use. Defaults to 'GET'. + * @throws Exception + * @return bool true (or string) on success; false on HTTP response error code (1xx or 4xx) + */ + static public function sendHttpRequest($aUrl, $timeout, $userAgent = null, $destinationPath = null, $followDepth = 0, $acceptLanguage = false, $byteRange = false, $getExtendedInfo = false, $httpMethod = 'GET' + ) + { + // create output file + $file = null; + if ($destinationPath) { + // Ensure destination directory exists + Piwik_Common::mkdir(dirname($destinationPath)); + if (($file = @fopen($destinationPath, 'wb')) === false || !is_resource($file)) { + throw new Exception('Error while creating the file: ' . $destinationPath); + } + } + + $acceptLanguage = $acceptLanguage ? 'Accept-Language: ' . $acceptLanguage : ''; + return self::sendHttpRequestBy(self::getTransportMethod(), $aUrl, $timeout, $userAgent, $destinationPath, $file, $followDepth, $acceptLanguage, $acceptInvalidSslCertificate = false, $byteRange, $getExtendedInfo, $httpMethod); + } + + /** + * Sends http request using the specified transport method + * + * @param string $method + * @param string $aUrl + * @param int $timeout + * @param string $userAgent + * @param string $destinationPath + * @param resource $file + * @param int $followDepth + * @param bool|string $acceptLanguage Accept-language header + * @param bool $acceptInvalidSslCertificate Only used with $method == 'curl'. If set to true (NOT recommended!) the SSL certificate will not be checked + * @param array $byteRange For Range: header. Should be two element array of bytes, eg, array(0, 1024) + * Doesn't work w/ fopen method. + * @param bool $getExtendedInfo True to return status code, headers & response, false if just response. + * @param string $httpMethod The HTTP method to use. Defaults to 'GET'. + * @throws Exception + * @return bool true (or string/array) on success; false on HTTP response error code (1xx or 4xx) + */ + static public function sendHttpRequestBy( + $method = 'socket', + $aUrl, + $timeout, + $userAgent = null, + $destinationPath = null, + $file = null, + $followDepth = 0, + $acceptLanguage = false, + $acceptInvalidSslCertificate = false, + $byteRange = false, + $getExtendedInfo = false, + $httpMethod = 'GET' + ) + { + if ($followDepth > 5) { + throw new Exception('Too many redirects (' . $followDepth . ')'); + } + + $contentLength = 0; + $fileLength = 0; + + // Piwik services behave like a proxy, so we should act like one. + $xff = 'X-Forwarded-For: ' + . (isset($_SERVER['HTTP_X_FORWARDED_FOR']) && !empty($_SERVER['HTTP_X_FORWARDED_FOR']) ? $_SERVER['HTTP_X_FORWARDED_FOR'] . ',' : '') + . Piwik_IP::getIpFromHeader(); + + if (empty($userAgent)) { + $userAgent = self::getUserAgent(); + } + + $via = 'Via: ' + . (isset($_SERVER['HTTP_VIA']) && !empty($_SERVER['HTTP_VIA']) ? $_SERVER['HTTP_VIA'] . ', ' : '') + . Piwik_Version::VERSION . ' ' + . ($userAgent ? " ($userAgent)" : ''); + + // range header + $rangeHeader = ''; + if (!empty($byteRange)) { + $rangeHeader = 'Range: bytes=' . $byteRange[0] . '-' . $byteRange[1] . "\r\n"; + } + + // proxy configuration + $proxyHost = Piwik_Config::getInstance()->proxy['host']; + $proxyPort = Piwik_Config::getInstance()->proxy['port']; + $proxyUser = Piwik_Config::getInstance()->proxy['username']; + $proxyPassword = Piwik_Config::getInstance()->proxy['password']; + + // other result data + $status = null; + $headers = array(); + + if ($method == 'socket') { + if (!self::isSocketEnabled()) { + // can be triggered in tests + throw new Exception("HTTP socket support is not enabled (php function fsockopen is not available) "); + } + // initialization + $url = @parse_url($aUrl); + if ($url === false || !isset($url['scheme'])) { + throw new Exception('Malformed URL: ' . $aUrl); + } + + if ($url['scheme'] != 'http') { + throw new Exception('Invalid protocol/scheme: ' . $url['scheme']); + } + $host = $url['host']; + $port = isset($url['port)']) ? $url['port'] : 80; + $path = isset($url['path']) ? $url['path'] : '/'; + if (isset($url['query'])) { + $path .= '?' . $url['query']; + } + $errno = null; + $errstr = null; + + if ((!empty($proxyHost) && !empty($proxyPort)) + || !empty($byteRange) + ) { + $httpVer = '1.1'; + } else { + $httpVer = '1.0'; + } + + $proxyAuth = null; + if (!empty($proxyHost) && !empty($proxyPort)) { + $connectHost = $proxyHost; + $connectPort = $proxyPort; + if (!empty($proxyUser) && !empty($proxyPassword)) { + $proxyAuth = 'Proxy-Authorization: Basic ' . base64_encode("$proxyUser:$proxyPassword") . "\r\n"; + } + $requestHeader = "$httpMethod $aUrl HTTP/$httpVer\r\n"; + } else { + $connectHost = $host; + $connectPort = $port; + $requestHeader = "$httpMethod $path HTTP/$httpVer\r\n"; + } + + // connection attempt + if (($fsock = @fsockopen($connectHost, $connectPort, $errno, $errstr, $timeout)) === false || !is_resource($fsock)) { + if (is_resource($file)) { + @fclose($file); + } + throw new Exception("Error while connecting to: $host. Please try again later. $errstr"); + } + + // send HTTP request header + $requestHeader .= + "Host: $host" . ($port != 80 ? ':' . $port : '') . "\r\n" + . ($proxyAuth ? $proxyAuth : '') + . 'User-Agent: ' . $userAgent . "\r\n" + . ($acceptLanguage ? $acceptLanguage . "\r\n" : '') + . $xff . "\r\n" + . $via . "\r\n" + . $rangeHeader + . "Connection: close\r\n" + . "\r\n"; + fwrite($fsock, $requestHeader); + + $streamMetaData = array('timed_out' => false); + @stream_set_blocking($fsock, true); + + if (function_exists('stream_set_timeout')) { + @stream_set_timeout($fsock, $timeout); + } elseif (function_exists('socket_set_timeout')) { + @socket_set_timeout($fsock, $timeout); + } + + // process header + $status = null; + $expectRedirect = false; + + while (!feof($fsock)) { + $line = fgets($fsock, 4096); + + $streamMetaData = @stream_get_meta_data($fsock); + if ($streamMetaData['timed_out']) { + if (is_resource($file)) { + @fclose($file); + } + @fclose($fsock); + throw new Exception('Timed out waiting for server response'); + } + + // a blank line marks the end of the server response header + if (rtrim($line, "\r\n") == '') { + break; + } + + // parse first line of server response header + if (!$status) { + // expect first line to be HTTP response status line, e.g., HTTP/1.1 200 OK + if (!preg_match('~^HTTP/(\d\.\d)\s+(\d+)(\s*.*)?~', $line, $m)) { + if (is_resource($file)) { + @fclose($file); + } + @fclose($fsock); + throw new Exception('Expected server response code. Got ' . rtrim($line, "\r\n")); + } + + $status = (integer)$m[2]; + + // Informational 1xx or Client Error 4xx + if ($status < 200 || $status >= 400) { + if (is_resource($file)) { + @fclose($file); + } + @fclose($fsock); + + if (!$getExtendedInfo) { + return false; + } else { + return array('status' => $status); + } + } + + continue; + } + + // handle redirect + if (preg_match('/^Location:\s*(.+)/', rtrim($line, "\r\n"), $m)) { + if (is_resource($file)) { + @fclose($file); + } + @fclose($fsock); + // Successful 2xx vs Redirect 3xx + if ($status < 300) { + throw new Exception('Unexpected redirect to Location: ' . rtrim($line) . ' for status code ' . $status); + } + return self::sendHttpRequestBy( + $method, + trim($m[1]), + $timeout, + $userAgent, + $destinationPath, + $file, + $followDepth + 1, + $acceptLanguage, + $acceptInvalidSslCertificate = false, + $byteRange, + $getExtendedInfo, + $httpMethod + ); + } + + // save expected content length for later verification + if (preg_match('/^Content-Length:\s*(\d+)/', $line, $m)) { + $contentLength = (integer)$m[1]; + } + + self::parseHeaderLine($headers, $line); + } + + if (feof($fsock) + && $httpMethod != 'HEAD' + ) { + throw new Exception('Unexpected end of transmission'); + } + + // process content/body + $response = ''; + + while (!feof($fsock)) { + $line = fread($fsock, 8192); + + $streamMetaData = @stream_get_meta_data($fsock); + if ($streamMetaData['timed_out']) { + if (is_resource($file)) { + @fclose($file); + } + @fclose($fsock); + throw new Exception('Timed out waiting for server response'); + } + + $fileLength += Piwik_Common::strlen($line); + + if (is_resource($file)) { + // save to file + fwrite($file, $line); + } else { + // concatenate to response string + $response .= $line; + } + } + + // determine success or failure + @fclose(@$fsock); + } else if ($method == 'fopen') { + $response = false; + + // we make sure the request takes less than a few seconds to fail + // we create a stream_context (works in php >= 5.2.1) + // we also set the socket_timeout (for php < 5.2.1) + $default_socket_timeout = @ini_get('default_socket_timeout'); + @ini_set('default_socket_timeout', $timeout); + + $ctx = null; + if (function_exists('stream_context_create')) { + $stream_options = array( + 'http' => array( + 'header' => 'User-Agent: ' . $userAgent . "\r\n" + . ($acceptLanguage ? $acceptLanguage . "\r\n" : '') + . $xff . "\r\n" + . $via . "\r\n" + . $rangeHeader, + 'max_redirects' => 5, // PHP 5.1.0 + 'timeout' => $timeout, // PHP 5.2.1 + ) + ); + + if (!empty($proxyHost) && !empty($proxyPort)) { + $stream_options['http']['proxy'] = 'tcp://' . $proxyHost . ':' . $proxyPort; + $stream_options['http']['request_fulluri'] = true; // required by squid proxy + if (!empty($proxyUser) && !empty($proxyPassword)) { + $stream_options['http']['header'] .= 'Proxy-Authorization: Basic ' . base64_encode("$proxyUser:$proxyPassword") . "\r\n"; + } + } + + $ctx = stream_context_create($stream_options); + } + + // save to file + if (is_resource($file)) { + $handle = fopen($aUrl, 'rb', false, $ctx); + while (!feof($handle)) { + $response = fread($handle, 8192); + $fileLength += Piwik_Common::strlen($response); + fwrite($file, $response); + } + fclose($handle); + } else { + $response = file_get_contents($aUrl, 0, $ctx); + $fileLength = Piwik_Common::strlen($response); + } + + // restore the socket_timeout value + if (!empty($default_socket_timeout)) { + @ini_set('default_socket_timeout', $default_socket_timeout); + } + } else if ($method == 'curl') { + if (!self::isCurlEnabled()) { + // can be triggered in tests + throw new Exception("CURL is not enabled in php.ini, but is being used."); + } + $ch = @curl_init(); + + if (!empty($proxyHost) && !empty($proxyPort)) { + @curl_setopt($ch, CURLOPT_PROXY, $proxyHost . ':' . $proxyPort); + if (!empty($proxyUser) && !empty($proxyPassword)) { + // PROXYAUTH defaults to BASIC + @curl_setopt($ch, CURLOPT_PROXYUSERPWD, $proxyUser . ':' . $proxyPassword); + } + } + + $curl_options = array( + // internal to ext/curl + CURLOPT_BINARYTRANSFER => is_resource($file), + + // curl options (sorted oldest to newest) + CURLOPT_URL => $aUrl, + CURLOPT_USERAGENT => $userAgent, + CURLOPT_HTTPHEADER => array( + $xff, + $via, + $rangeHeader, + $acceptLanguage + ), + // only get header info if not saving directly to file + CURLOPT_HEADER => is_resource($file) ? false : true, + CURLOPT_CONNECTTIMEOUT => $timeout, + ); + // Case archive.php is triggering archiving on https:// and the certificate is not valid + if ($acceptInvalidSslCertificate) { + $curl_options += array( + CURLOPT_SSL_VERIFYHOST => false, + CURLOPT_SSL_VERIFYPEER => false, + ); + } + @curl_setopt($ch, CURLOPT_CUSTOMREQUEST, $httpMethod); + if ($httpMethod == 'HEAD') { + @curl_setopt($ch, CURLOPT_NOBODY, true); + } + + @curl_setopt_array($ch, $curl_options); + self::configCurlCertificate($ch); + + + /* + * as of php 5.2.0, CURLOPT_FOLLOWLOCATION can't be set if + * in safe_mode or open_basedir is set + */ + if ((string)ini_get('safe_mode') == '' && ini_get('open_basedir') == '') { + $curl_options = array( + // curl options (sorted oldest to newest) + CURLOPT_FOLLOWLOCATION => true, + CURLOPT_MAXREDIRS => 5, + ); + @curl_setopt_array($ch, $curl_options); + } + + if (is_resource($file)) { + // write output directly to file + @curl_setopt($ch, CURLOPT_FILE, $file); + } else { + // internal to ext/curl + @curl_setopt($ch, CURLOPT_RETURNTRANSFER, true); + } + + ob_start(); + $response = @curl_exec($ch); + ob_end_clean(); + + if ($response === true) { + $response = ''; + } else if ($response === false) { + $errstr = curl_error($ch); + if ($errstr != '') { + throw new Exception('curl_exec: ' . $errstr); + } + $response = ''; + } else { + $header = ''; + // redirects are included in the output html, so we look for the last line that starts w/ HTTP/... + // to split the response + while (substr($response, 0, 5) == "HTTP/") { + list($header, $response) = explode("\r\n\r\n", $response, 2); + } + + foreach (explode("\r\n", $header) as $line) { + self::parseHeaderLine($headers, $line); + } + } + + $contentLength = @curl_getinfo($ch, CURLINFO_CONTENT_LENGTH_DOWNLOAD); + $fileLength = is_resource($file) ? @curl_getinfo($ch, CURLINFO_SIZE_DOWNLOAD) : Piwik_Common::strlen($response); + $status = @curl_getinfo($ch, CURLINFO_HTTP_CODE); + + @curl_close($ch); + unset($ch); + } else { + throw new Exception('Invalid request method: ' . $method); + } + + if (is_resource($file)) { + fflush($file); + @fclose($file); + + $fileSize = filesize($destinationPath); + if ((($contentLength > 0) && ($fileLength != $contentLength)) + || ($fileSize != $fileLength) + ) { + throw new Exception('File size error: ' . $destinationPath . '; expected ' . $contentLength . ' bytes; received ' . $fileLength . ' bytes; saved ' . $fileSize . ' bytes to file'); + } + return true; + } + + if (!$getExtendedInfo) { + return trim($response); + } else { + return array( + 'status' => $status, + 'headers' => $headers, + 'data' => $response + ); + } + } + + /** + * Downloads the next chunk of a specific file. The next chunk's byte range + * is determined by the existing file's size and the expected file size, which + * is stored in the piwik_option table before starting a download. + * + * Note this function uses the Range HTTP header to accomplish downloading in + * parts. + * + * @param string $url The url to download from. + * @param string $outputPath The path to the file to save/append to. + * @param bool $isContinuation True if this is the continuation of a download, + * or if we're starting a fresh one. + */ + public static function downloadChunk($url, $outputPath, $isContinuation) + { + // make sure file doesn't already exist if we're starting a new download + if (!$isContinuation + && file_exists($outputPath) + ) { + throw new Exception( + Piwik_Translate('General_DownloadFail_FileExists', "'" . $outputPath . "'") + . ' ' . Piwik_Translate('General_DownloadPleaseRemoveExisting')); + } + + // if we're starting a download, get the expected file size & save as an option + $downloadOption = $outputPath . '_expectedDownloadSize'; + if (!$isContinuation) { + $expectedFileSizeResult = Piwik_Http::sendHttpRequest( + $url, + $timeout = 300, + $userAgent = null, + $destinationPath = null, + $followDepth = 0, + $acceptLanguage = false, + $byteRange = false, + $getExtendedInfo = true, + $httpMethod = 'HEAD' + ); + + $expectedFileSize = 0; + if (isset($expectedFileSizeResult['headers']['Content-Length'])) { + $expectedFileSize = (int)$expectedFileSizeResult['headers']['Content-Length']; + } + + if ($expectedFileSize == 0) { + Piwik::log(sprintf("HEAD request for '%s' failed, got following: %s", $url, print_r($expectedFileSizeResult, true))); + throw new Exception(Piwik_Translate('General_DownloadFail_HttpRequestFail')); + } + + Piwik_SetOption($downloadOption, $expectedFileSize); + } else { + $expectedFileSize = (int)Piwik_GetOption($downloadOption); + if ($expectedFileSize === false) // sanity check + { + throw new Exception("Trying to continue a download that never started?! That's not supposed to happen..."); + } + } + + // if existing file is already big enough, then fail so we don't accidentally overwrite + // existing DB + $existingSize = file_exists($outputPath) ? filesize($outputPath) : 0; + if ($existingSize >= $expectedFileSize) { + throw new Exception( + Piwik_Translate('General_DownloadFail_FileExistsContinue', "'" . $outputPath . "'") + . ' ' . Piwik_Translate('General_DownloadPleaseRemoveExisting')); + } + + // download a chunk of the file + $result = Piwik_Http::sendHttpRequest( + $url, + $timeout = 300, + $userAgent = null, + $destinationPath = null, + $followDepth = 0, + $acceptLanguage = false, + $byteRange = array($existingSize, min($existingSize + 1024 * 1024 - 1, $expectedFileSize)), + $getExtendedInfo = true + ); + + if ($result === false + || $result['status'] < 200 + || $result['status'] > 299 + ) { + $result['data'] = self::truncateStr($result['data'], 1024); + Piwik::log("Failed to download range '" . $byteRange[0] . "-" . $byteRange[1] + . "' of file from url '$url'. Got result: " . print_r($result, true)); + + throw new Exception(Piwik_Translate('General_DownloadFail_HttpRequestFail')); + } + + // write chunk to file + $f = fopen($outputPath, 'ab'); + fwrite($f, $result['data']); + fclose($f); + + clearstatcache($clear_realpath_cache = true, $outputPath); + return array( + 'current_size' => filesize($outputPath), + 'expected_file_size' => $expectedFileSize, + ); + } + + /** + * Will configure CURL handle $ch + * to use local list of Certificate Authorities, + */ + public static function configCurlCertificate(&$ch) + { + if (file_exists(PIWIK_INCLUDE_PATH . '/core/DataFiles/cacert.pem')) { + @curl_setopt($ch, CURLOPT_CAINFO, PIWIK_INCLUDE_PATH . '/core/DataFiles/cacert.pem'); + } + } + + public static function getUserAgent() + { + return !empty($_SERVER['HTTP_USER_AGENT']) + ? $_SERVER['HTTP_USER_AGENT'] + : 'Piwik/' . Piwik_Version::VERSION; + } + + /** + * Fetch the file at $url in the destination $destinationPath + * + * @param string $url + * @param string $destinationPath + * @param int $tries + * @throws Exception + * @return bool true on success, throws Exception on failure + */ + static public function fetchRemoteFile($url, $destinationPath = null, $tries = 0) + { + @ignore_user_abort(true); + Piwik::setMaxExecutionTime(0); + return self::sendHttpRequest($url, 10, 'Update', $destinationPath, $tries); + } + + /** + * Utility function, parses an HTTP header line into key/value & sets header + * array with them. + * + * @param array $headers + * @param string $line + */ + private static function parseHeaderLine(&$headers, $line) + { + $parts = explode(':', $line, 2); + if (count($parts) == 1) { + return; + } + + list($name, $value) = $parts; + $headers[trim($name)] = trim($value); + } + + /** + * Utility function that truncates a string to an arbitrary limit. + * + * @param string $str The string to truncate. + * @param int $limit The maximum length of the truncated string. + * @return string + */ + private static function truncateStr($str, $limit) + { + if (strlen($str) > $limit) { + return substr($str, 0, $limit) . '...'; + } + return $str; + } } |