diff options
author | Morris Jobke <hey@morrisjobke.de> | 2020-05-25 14:38:04 +0300 |
---|---|---|
committer | GitHub <noreply@github.com> | 2020-05-25 14:38:04 +0300 |
commit | 1d816add44fb0058d1fd927585a4e894c1d39132 (patch) | |
tree | c7cb8d854e318515d5a19fa8f832fb9b434f2530 /lib | |
parent | 4e8f72726ae7ede288cef5073527b6b7df737d88 (diff) | |
parent | 3d3ee1bfaefb16a5fe912932abff90bf972f3736 (diff) |
Merge pull request #20965 from nextcloud/backport/20033/stable18
[stable18] Enable fseek for files in S3 storage
Diffstat (limited to 'lib')
-rw-r--r-- | lib/composer/composer/autoload_classmap.php | 1 | ||||
-rw-r--r-- | lib/composer/composer/autoload_static.php | 1 | ||||
-rw-r--r-- | lib/private/Files/ObjectStore/S3ObjectTrait.php | 45 | ||||
-rw-r--r-- | lib/private/Files/Stream/SeekableHttpStream.php | 192 |
4 files changed, 218 insertions, 21 deletions
diff --git a/lib/composer/composer/autoload_classmap.php b/lib/composer/composer/autoload_classmap.php index d66039ae1fd..998fe176967 100644 --- a/lib/composer/composer/autoload_classmap.php +++ b/lib/composer/composer/autoload_classmap.php @@ -1007,6 +1007,7 @@ return array( 'OC\\Files\\Storage\\Wrapper\\Wrapper' => $baseDir . '/lib/private/Files/Storage/Wrapper/Wrapper.php', 'OC\\Files\\Stream\\Encryption' => $baseDir . '/lib/private/Files/Stream/Encryption.php', 'OC\\Files\\Stream\\Quota' => $baseDir . '/lib/private/Files/Stream/Quota.php', + 'OC\\Files\\Stream\\SeekableHttpStream' => $baseDir . '/lib/private/Files/Stream/SeekableHttpStream.php', 'OC\\Files\\Type\\Detection' => $baseDir . '/lib/private/Files/Type/Detection.php', 'OC\\Files\\Type\\Loader' => $baseDir . '/lib/private/Files/Type/Loader.php', 'OC\\Files\\Type\\TemplateManager' => $baseDir . '/lib/private/Files/Type/TemplateManager.php', diff --git a/lib/composer/composer/autoload_static.php b/lib/composer/composer/autoload_static.php index 0a65eae24bb..7dd0e15e51d 100644 --- a/lib/composer/composer/autoload_static.php +++ b/lib/composer/composer/autoload_static.php @@ -1036,6 +1036,7 @@ class ComposerStaticInit53792487c5a8370acc0b06b1a864ff4c 'OC\\Files\\Storage\\Wrapper\\Wrapper' => __DIR__ . '/../../..' . '/lib/private/Files/Storage/Wrapper/Wrapper.php', 'OC\\Files\\Stream\\Encryption' => __DIR__ . '/../../..' . '/lib/private/Files/Stream/Encryption.php', 'OC\\Files\\Stream\\Quota' => __DIR__ . '/../../..' . '/lib/private/Files/Stream/Quota.php', + 'OC\\Files\\Stream\\SeekableHttpStream' => __DIR__ . '/../../..' . '/lib/private/Files/Stream/SeekableHttpStream.php', 'OC\\Files\\Type\\Detection' => __DIR__ . '/../../..' . '/lib/private/Files/Type/Detection.php', 'OC\\Files\\Type\\Loader' => __DIR__ . '/../../..' . '/lib/private/Files/Type/Loader.php', 'OC\\Files\\Type\\TemplateManager' => __DIR__ . '/../../..' . '/lib/private/Files/Type/TemplateManager.php', diff --git a/lib/private/Files/ObjectStore/S3ObjectTrait.php b/lib/private/Files/ObjectStore/S3ObjectTrait.php index 6188357521b..0939dd2a23c 100644 --- a/lib/private/Files/ObjectStore/S3ObjectTrait.php +++ b/lib/private/Files/ObjectStore/S3ObjectTrait.php @@ -30,6 +30,7 @@ use Aws\S3\MultipartUploader; use Aws\S3\ObjectUploader; use Aws\S3\S3Client; use Icewind\Streams\CallbackWrapper; +use OC\Files\Stream\SeekableHttpStream; const S3_UPLOAD_PART_SIZE = 524288000; // 500MB @@ -49,27 +50,29 @@ trait S3ObjectTrait { * @since 7.0.0 */ function readObject($urn) { - $client = $this->getConnection(); - $command = $client->getCommand('GetObject', [ - 'Bucket' => $this->bucket, - 'Key' => $urn - ]); - $request = \Aws\serialize($command); - $headers = []; - foreach ($request->getHeaders() as $key => $values) { - foreach ($values as $value) { - $headers[] = "$key: $value"; + return SeekableHttpStream::open(function ($range) use ($urn) { + $command = $this->getConnection()->getCommand('GetObject', [ + 'Bucket' => $this->bucket, + 'Key' => $urn, + 'Range' => 'bytes=' . $range, + ]); + $request = \Aws\serialize($command); + $headers = []; + foreach ($request->getHeaders() as $key => $values) { + foreach ($values as $value) { + $headers[] = "$key: $value"; + } } - } - $opts = [ - 'http' => [ - 'protocol_version' => 1.1, - 'header' => $headers - ] - ]; + $opts = [ + 'http' => [ + 'protocol_version' => 1.1, + 'header' => $headers, + ], + ]; - $context = stream_context_create($opts); - return fopen($request->getUri(), 'r', false, $context); + $context = stream_context_create($opts); + return fopen($request->getUri(), 'r', false, $context); + }); } /** @@ -87,7 +90,7 @@ trait S3ObjectTrait { $uploader = new MultipartUploader($this->getConnection(), $countStream, [ 'bucket' => $this->bucket, 'key' => $urn, - 'part_size' => S3_UPLOAD_PART_SIZE + 'part_size' => S3_UPLOAD_PART_SIZE, ]); try { @@ -114,7 +117,7 @@ trait S3ObjectTrait { function deleteObject($urn) { $this->getConnection()->deleteObject([ 'Bucket' => $this->bucket, - 'Key' => $urn + 'Key' => $urn, ]); } diff --git a/lib/private/Files/Stream/SeekableHttpStream.php b/lib/private/Files/Stream/SeekableHttpStream.php new file mode 100644 index 00000000000..113ba19a17a --- /dev/null +++ b/lib/private/Files/Stream/SeekableHttpStream.php @@ -0,0 +1,192 @@ +<?php +/** + * + * @copyright Copyright (c) 2020, Lukas Stabe (lukas@stabe.de) + * + * @license GNU AGPL version 3 or any later version + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + * + */ + +namespace OC\Files\Stream; + +use Icewind\Streams\File; + +/** + * A stream wrapper that uses http range requests to provide a seekable stream for http reading + */ +class SeekableHttpStream implements File { + private const PROTOCOL = 'httpseek'; + + private static $registered = false; + + /** + * Registers the stream wrapper using the `httpseek://` url scheme + * $return void + */ + private static function registerIfNeeded() { + if (!self::$registered) { + stream_wrapper_register( + self::PROTOCOL, + self::class + ); + self::$registered = true; + } + } + + /** + * Open a readonly-seekable http stream + * + * The provided callback will be called with byte range and should return an http stream for the requested range + * + * @param callable $callback + * @return false|resource + */ + public static function open(callable $callback) { + $context = stream_context_create([ + SeekableHttpStream::PROTOCOL => [ + 'callback' => $callback + ], + ]); + + SeekableHttpStream::registerIfNeeded(); + return fopen(SeekableHttpStream::PROTOCOL . '://', 'r', false, $context); + } + + /** @var resource */ + public $context; + + /** @var callable */ + private $openCallback; + + /** @var resource */ + private $current; + /** @var int */ + private $offset = 0; + + private function reconnect(int $start) { + $range = $start . '-'; + if ($this->current != null) { + fclose($this->current); + } + + $this->current = ($this->openCallback)($range); + + if ($this->current === false) { + return false; + } + + $responseHead = stream_get_meta_data($this->current)['wrapper_data']; + $rangeHeaders = array_values(array_filter($responseHead, function ($v) { + return preg_match('#^content-range:#i', $v) === 1; + })); + if (!$rangeHeaders) { + return false; + } + $contentRange = $rangeHeaders[0]; + + $content = trim(explode(':', $contentRange)[1]); + $range = trim(explode(' ', $content)[1]); + $begin = intval(explode('-', $range)[0]); + + if ($begin !== $start) { + return false; + } + + $this->offset = $begin; + + return true; + } + + public function stream_open($path, $mode, $options, &$opened_path) { + $options = stream_context_get_options($this->context)[self::PROTOCOL]; + $this->openCallback = $options['callback']; + + return $this->reconnect(0); + } + + public function stream_read($count) { + if (!$this->current) { + return false; + } + $ret = fread($this->current, $count); + $this->offset += strlen($ret); + return $ret; + } + + public function stream_seek($offset, $whence = SEEK_SET) { + switch ($whence) { + case SEEK_SET: + if ($offset === $this->offset) { + return true; + } + return $this->reconnect($offset); + case SEEK_CUR: + if ($offset === 0) { + return true; + } + return $this->reconnect($this->offset + $offset); + case SEEK_END: + return false; + } + return false; + } + + public function stream_tell() { + return $this->offset; + } + + public function stream_stat() { + if (is_resource($this->current)) { + return fstat($this->current); + } else { + return false; + } + } + + public function stream_eof() { + if (is_resource($this->current)) { + return feof($this->current); + } else { + return true; + } + } + + public function stream_close() { + if (is_resource($this->current)) { + fclose($this->current); + } + } + + public function stream_write($data) { + return false; + } + + public function stream_set_option($option, $arg1, $arg2) { + return false; + } + + public function stream_truncate($size) { + return false; + } + + public function stream_lock($operation) { + return false; + } + + public function stream_flush() { + return; //noop because readonly stream + } +} |