Send images to imaginary docker to generate previewsimaginary-prototype

Signed-off-by: Carl Schwan <carl@carlschwan.eu>
Co-Authored-by: Vincent Petry <vincent@nextcloud.com>

10 files changed, 421 insertions, 4 deletions

diff --git a/config/config.sample.php b/config/config.sample.php
index f861df2a7ce..5bc1cc52cf0 100644
--- a/config/config.sample.php
+++ b/config/config.sample.php
@@ -1102,6 +1102,14 @@ $CONFIG = [
 	'--convert-to png --outdir ',
 
 /**
+ * Set the URL of the Imaginary service to send image previews to.
+ * Also requires the OC\Preview\Imaginary provider to be enabled.
+ *
+ * See https://github.com/h2non/imaginary
+ */
+'preview_imaginary_url' => 'http://previews_hpb:8088/',
+
+/**
  * Only register providers that have been explicitly enabled
  *
  * The following providers are disabled by default due to performance or privacy
diff --git a/lib/composer/composer/autoload_classmap.php b/lib/composer/composer/autoload_classmap.php
index 81e02389e85..5e136ddecd9 100644
--- a/lib/composer/composer/autoload_classmap.php
+++ b/lib/composer/composer/autoload_classmap.php
@@ -414,6 +414,7 @@ return array(
     'OCP\\ISearch' => $baseDir . '/lib/public/ISearch.php',
     'OCP\\IServerContainer' => $baseDir . '/lib/public/IServerContainer.php',
     'OCP\\ISession' => $baseDir . '/lib/public/ISession.php',
+    'OCP\\IStreamImage' => $baseDir . '/lib/public/IStreamImage.php',
     'OCP\\ITagManager' => $baseDir . '/lib/public/ITagManager.php',
     'OCP\\ITags' => $baseDir . '/lib/public/ITags.php',
     'OCP\\ITempManager' => $baseDir . '/lib/public/ITempManager.php',
@@ -1299,6 +1300,7 @@ return array(
     'OC\\Preview\\HEIC' => $baseDir . '/lib/private/Preview/HEIC.php',
     'OC\\Preview\\Illustrator' => $baseDir . '/lib/private/Preview/Illustrator.php',
     'OC\\Preview\\Image' => $baseDir . '/lib/private/Preview/Image.php',
+    'OC\\Preview\\Imaginary' => $baseDir . '/lib/private/Preview/Imaginary.php',
     'OC\\Preview\\JPEG' => $baseDir . '/lib/private/Preview/JPEG.php',
     'OC\\Preview\\Krita' => $baseDir . '/lib/private/Preview/Krita.php',
     'OC\\Preview\\MP3' => $baseDir . '/lib/private/Preview/MP3.php',
@@ -1455,6 +1457,7 @@ return array(
     'OC\\Share\\Helper' => $baseDir . '/lib/private/Share/Helper.php',
     'OC\\Share\\SearchResultSorter' => $baseDir . '/lib/private/Share/SearchResultSorter.php',
     'OC\\Share\\Share' => $baseDir . '/lib/private/Share/Share.php',
+    'OC\\StreamImage' => $baseDir . '/lib/private/StreamImage.php',
     'OC\\Streamer' => $baseDir . '/lib/private/Streamer.php',
     'OC\\SubAdmin' => $baseDir . '/lib/private/SubAdmin.php',
     'OC\\Support\\CrashReport\\Registry' => $baseDir . '/lib/private/Support/CrashReport/Registry.php',
diff --git a/lib/composer/composer/autoload_static.php b/lib/composer/composer/autoload_static.php
index abaa64aa3c0..c148b10ba45 100644
--- a/lib/composer/composer/autoload_static.php
+++ b/lib/composer/composer/autoload_static.php
@@ -443,6 +443,7 @@ class ComposerStaticInit53792487c5a8370acc0b06b1a864ff4c
         'OCP\\ISearch' => __DIR__ . '/../../..' . '/lib/public/ISearch.php',
         'OCP\\IServerContainer' => __DIR__ . '/../../..' . '/lib/public/IServerContainer.php',
         'OCP\\ISession' => __DIR__ . '/../../..' . '/lib/public/ISession.php',
+        'OCP\\IStreamImage' => __DIR__ . '/../../..' . '/lib/public/IStreamImage.php',
         'OCP\\ITagManager' => __DIR__ . '/../../..' . '/lib/public/ITagManager.php',
         'OCP\\ITags' => __DIR__ . '/../../..' . '/lib/public/ITags.php',
         'OCP\\ITempManager' => __DIR__ . '/../../..' . '/lib/public/ITempManager.php',
@@ -1328,6 +1329,7 @@ class ComposerStaticInit53792487c5a8370acc0b06b1a864ff4c
         'OC\\Preview\\HEIC' => __DIR__ . '/../../..' . '/lib/private/Preview/HEIC.php',
         'OC\\Preview\\Illustrator' => __DIR__ . '/../../..' . '/lib/private/Preview/Illustrator.php',
         'OC\\Preview\\Image' => __DIR__ . '/../../..' . '/lib/private/Preview/Image.php',
+        'OC\\Preview\\Imaginary' => __DIR__ . '/../../..' . '/lib/private/Preview/Imaginary.php',
         'OC\\Preview\\JPEG' => __DIR__ . '/../../..' . '/lib/private/Preview/JPEG.php',
         'OC\\Preview\\Krita' => __DIR__ . '/../../..' . '/lib/private/Preview/Krita.php',
         'OC\\Preview\\MP3' => __DIR__ . '/../../..' . '/lib/private/Preview/MP3.php',
@@ -1484,6 +1486,7 @@ class ComposerStaticInit53792487c5a8370acc0b06b1a864ff4c
         'OC\\Share\\Helper' => __DIR__ . '/../../..' . '/lib/private/Share/Helper.php',
         'OC\\Share\\SearchResultSorter' => __DIR__ . '/../../..' . '/lib/private/Share/SearchResultSorter.php',
         'OC\\Share\\Share' => __DIR__ . '/../../..' . '/lib/private/Share/Share.php',
+        'OC\\StreamImage' => __DIR__ . '/../../..' . '/lib/private/StreamImage.php',
         'OC\\Streamer' => __DIR__ . '/../../..' . '/lib/private/Streamer.php',
         'OC\\SubAdmin' => __DIR__ . '/../../..' . '/lib/private/SubAdmin.php',
         'OC\\Support\\CrashReport\\Registry' => __DIR__ . '/../../..' . '/lib/private/Support/CrashReport/Registry.php',
diff --git a/lib/private/Http/Client/Client.php b/lib/private/Http/Client/Client.php
index 55a44eb41ba..673f566e354 100644
--- a/lib/private/Http/Client/Client.php
+++ b/lib/private/Http/Client/Client.php
@@ -292,7 +292,8 @@ class Client implements IClient {
 			unset($options['body']);
 		}
 		$response = $this->client->request('post', $uri, $this->buildRequestOptions($options));
-		return new Response($response);
+		$isStream = isset($options['stream']) && $options['stream'];
+		return new Response($response, $isStream);
 	}
 
 	/**
diff --git a/lib/private/Preview/Generator.php b/lib/private/Preview/Generator.php
index 6e1e0997a68..a770fe3b459 100644
--- a/lib/private/Preview/Generator.php
+++ b/lib/private/Preview/Generator.php
@@ -38,6 +38,7 @@ use OCP\Files\SimpleFS\ISimpleFolder;
 use OCP\IConfig;
 use OCP\IImage;
 use OCP\IPreview;
+use OCP\IStreamImage;
 use OCP\Preview\IProviderV2;
 use OCP\Preview\IVersionedPreviewFile;
 use Symfony\Component\EventDispatcher\EventDispatcherInterface;
@@ -136,6 +137,22 @@ class Generator {
 			$previewVersion = $file->getPreviewVersion() . '-';
 		}
 
+		if (count($specifications) === 1
+			&& (($specifications[0]['width'] === 250 && $specifications[0]['height'] === 250)
+				|| ($specifications[0]['width'] === 150 && $specifications[0]['height'] === 150))
+			&& preg_match(Imaginary::supportedMimeTypes(), $mimeType)
+			&& $this->config->getSystemValueString('preview_imaginary_url', 'invalid') !== 'invalid') {
+			$crop = $specifications[0]['crop'] ?? false;
+			$preview = $this->getSmallImagePreview($previewFolder, $file, $mimeType, $previewVersion, $crop);
+
+			if ($preview->getSize() === 0) {
+				$preview->delete();
+				throw new NotFoundException('Cached preview size 0, invalid!');
+			}
+
+			return $preview;
+		}
+
 		// Get the max preview and infer the max preview sizes from that
 		$maxPreview = $this->getMaxPreview($previewFolder, $file, $mimeType, $previewVersion);
 		$maxPreviewImage = null; // only load the image when we need it
@@ -204,6 +221,65 @@ class Generator {
 		return $preview;
 	}
 
+	private function getSmallImagePreview(ISimpleFolder $previewFolder, File $file, string $mimeType, string $prefix, bool $crop) {
+		$nodes = $previewFolder->getDirectoryListing();
+
+		foreach ($nodes as $node) {
+			$name = $node->getName();
+			if (($prefix === '' || strpos($name, $prefix) === 0)
+				&& (str_starts_with($name, '256-256-crop') && $crop || str_starts_with($name, '256-256') && !$crop)) {
+				return $node;
+			}
+		}
+
+		$previewProviders = $this->previewManager->getProviders();
+		foreach ($previewProviders as $supportedMimeType => $providers) {
+			// Filter out providers that does not support this mime
+			if (!preg_match($supportedMimeType, $mimeType)) {
+				continue;
+			}
+
+			foreach ($providers as $providerClosure) {
+				$provider = $this->helper->getProvider($providerClosure);
+				if (!($provider instanceof IProviderV2)) {
+					continue;
+				}
+
+				if (!$provider->isAvailable($file)) {
+					continue;
+				}
+
+				$preview = $this->helper->getThumbnail($provider, $file, 256, 256, true);
+
+				if (!($preview instanceof IImage)) {
+					continue;
+				}
+
+				// Try to get the extension.
+				try {
+					$ext = $this->getExtention($preview->dataMimeType());
+				} catch (\InvalidArgumentException $e) {
+					// Just continue to the next iteration if this preview doesn't have a valid mimetype
+					continue;
+				}
+
+				$path = $this->generatePath(256, 256, $crop, $preview->dataMimeType(), $prefix);
+				try {
+					$file = $previewFolder->newFile($path);
+					if ($preview instanceof IStreamImage) {
+						$file->putContent($preview->resource());
+					} else {
+						$file->putContent($preview->data());
+					}
+				} catch (NotPermittedException $e) {
+					throw new NotFoundException();
+				}
+
+				return $file;
+			}
+		}
+	}
+
 	/**
 	 * @param ISimpleFolder $previewFolder
 	 * @param File $file
@@ -259,7 +335,11 @@ class Generator {
 				$path = $prefix . (string)$preview->width() . '-' . (string)$preview->height() . '-max.' . $ext;
 				try {
 					$file = $previewFolder->newFile($path);
-					$file->putContent($preview->data());
+					if ($preview instanceof IStreamImage) {
+						$file->putContent($preview->resource());
+					} else {
+						$file->putContent($preview->data());
+					}
 				} catch (NotPermittedException $e) {
 					throw new NotFoundException();
 				}
diff --git a/lib/private/Preview/GeneratorHelper.php b/lib/private/Preview/GeneratorHelper.php
index a5a9dacf63b..6a94b948241 100644
--- a/lib/private/Preview/GeneratorHelper.php
+++ b/lib/private/Preview/GeneratorHelper.php
@@ -58,8 +58,11 @@ class GeneratorHelper {
 	 *
 	 * @return bool|IImage
 	 */
-	public function getThumbnail(IProviderV2 $provider, File $file, $maxWidth, $maxHeight) {
-		return $provider->getThumbnail($file, $maxWidth, $maxHeight);
+	public function getThumbnail(IProviderV2 $provider, File $file, $maxWidth, $maxHeight, bool $crop = false) {
+		if ($provider instanceof Imaginary) {
+			return $provider->getCroppedThumbnail($file, $maxWidth, $maxHeight, $crop) ?? false;
+		}
+		return $provider->getThumbnail($file, $maxWidth, $maxHeight) ?? false;
 	}
 
 	/**
diff --git a/lib/private/Preview/Imaginary.php b/lib/private/Preview/Imaginary.php
new file mode 100644
index 00000000000..7e6ce86d4eb
--- /dev/null
+++ b/lib/private/Preview/Imaginary.php
@@ -0,0 +1,136 @@
+<?php
+/**
+ * @copyright Copyright (c) 2020, Nextcloud, GmbH.
+ *
+ * @author Vincent Petry <vincent@nextcloud.com>
+ * @author Carl Schwan <carl@carlschwan.eu>
+ *
+ * @license AGPL-3.0-or-later
+ *
+ * This code is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3,
+ * as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License, version 3,
+ * along with this program. If not, see <http://www.gnu.org/licenses/>
+ *
+ */
+
+namespace OC\Preview;
+
+use OCP\Files\File;
+use OCP\Http\Client\IClientService;
+use OCP\IConfig;
+use OCP\IImage;
+
+use OC\StreamImage;
+use Psr\Log\LoggerInterface;
+
+class Imaginary extends ProviderV2 {
+	/** @var IConfig */
+	private $config;
+
+	/** @var IClientService */
+	private $service;
+
+	/** @var LoggerInterface */
+	private $logger;
+
+	public function __construct(array $config) {
+		parent::__construct($config);
+		$this->config = \OC::$server->get(IConfig::class);
+		$this->service = \OC::$server->get(IClientService::class);
+		$this->logger = \OC::$server->get(LoggerInterface::class);
+	}
+
+	/**
+	 * {@inheritDoc}
+	 */
+	public function getMimeType(): string {
+		return self::supportedMimeTypes();
+	}
+
+	public static function supportedMimeTypes(): string {
+		return '/image\/(bmp|x-bitmap|png|jpeg|gif|heic|svg|webp)/';
+	}
+
+	public function getCroppedThumbnail(File $file, int $maxX, int $maxY, bool $crop): ?IImage {
+		$maxSizeForImages = $this->config->getSystemValue('preview_max_filesize_image', 50);
+
+		$size = $file->getSize();
+
+		if ($maxSizeForImages !== -1 && $size > ($maxSizeForImages * 1024 * 1024)) {
+			return null;
+		}
+
+		$imaginaryUrl = $this->config->getSystemValueString('preview_imaginary_url', 'invalid');
+		if ($imaginaryUrl === 'invalid') {
+			$this->logger->error('Imaginary preview provider is enabled, but no url is configured. Please provide the url of your imaginary server to the \'preview_imaginary_url\' config variable.');
+			return null;
+		}
+		$imaginaryUrl = rtrim($imaginaryUrl, '/');
+
+		// Object store
+		$stream = $file->fopen('r');
+
+		$httpClient = $this->service->newClient();
+
+		switch ($file->getMimeType()) {
+			case 'image/gif':
+			case 'image/png':
+				$mimeType = 'png';
+				break;
+			default:
+				$mimeType = 'jpeg';
+		}
+
+		$parameters = [
+			'width' => $maxX,
+			'height' => $maxY,
+			'stripmeta' => 'true',
+			'type' => $mimeType,
+		];
+
+
+		try {
+			$response = $httpClient->post(
+				$imaginaryUrl . ($crop ? '/smartcrop' : '/fit'), [
+					'query' => $parameters,
+					'stream' => true,
+					'content-type' => $file->getMimeType(),
+					'body' => $stream,
+					'nextcloud' => ['allow_local_address' => true],
+				]);
+		} catch (\Exception $e) {
+			$this->logger->error('Imaginary preview generation failed: ' . $e->getMessage(), [
+				'exception' => $e,
+			]);
+			return null;
+		}
+
+		if ($response->getStatusCode() !== 200) {
+			$this->logger->error('Imaginary preview generation failed: ' . json_decode($response->getBody())['message']);
+			return null;
+		}
+
+		if ($response->getHeader('X-Image-Width') && $response->getHeader('X-Image-Height')) {
+			$maxX = (int)$response->getHeader('X-Image-Width');
+			$maxY = (int)$response->getHeader('X-Image-Height');
+		}
+
+		$image = new StreamImage($response->getBody(), $response->getHeader('Content-Type'), $maxX, $maxY);
+		return $image->valid() ? $image : null;
+	}
+
+	/**
+	 * {@inheritDoc}
+	 */
+	public function getThumbnail(File $file, int $maxX, int $maxY): ?IImage {
+		return $this->getCroppedThumbnail($file, $maxX, $maxY, false);
+	}
+}
diff --git a/lib/private/PreviewManager.php b/lib/private/PreviewManager.php
index 18d4427d346..6c17dd58b4b 100644
--- a/lib/private/PreviewManager.php
+++ b/lib/private/PreviewManager.php
@@ -387,6 +387,7 @@ class PreviewManager implements IPreview {
 		$this->registerCoreProvider(Preview\Krita::class, '/application\/x-krita/');
 		$this->registerCoreProvider(Preview\MP3::class, '/audio\/mpeg/');
 		$this->registerCoreProvider(Preview\OpenDocument::class, '/application\/vnd.oasis.opendocument.*/');
+		$this->registerCoreProvider(Preview\Imaginary::class, Preview\Imaginary::supportedMimeTypes());
 
 		// SVG, Office and Bitmap require imagick
 		if (extension_loaded('imagick')) {
diff --git a/lib/private/StreamImage.php b/lib/private/StreamImage.php
new file mode 100644
index 00000000000..38fd114df17
--- /dev/null
+++ b/lib/private/StreamImage.php
@@ -0,0 +1,152 @@
+<?php
+/**
+ * @copyright Copyright (c) 2021 Carl Schwan <carl@carlschwan.eu>
+ *
+ * @author Carl Schwan <carl@carlschwan.eu>
+ *
+ * @license AGPL-3.0-or-later
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+namespace OC;
+
+use OCP\IStreamImage;
+use OCP\IImage;
+
+/**
+ * Only useful when dealing with transferring streamed previews from an external
+ * service to an object store.
+ *
+ * Only width/height/resource and mimeType are implemented and will give you a
+ * valid result.
+ */
+class StreamImage implements IStreamImage {
+	/** @var resource The internal stream */
+	private $stream;
+
+	/** @var string */
+	private $mimeType;
+
+	/** @var int */
+	private $width;
+
+	/** @var int */
+	private $height;
+
+	/** @param resource $stream */
+	public function __construct($stream, string $mimeType, int $width, int $height) {
+		$this->stream = $stream;
+		$this->mimeType = $mimeType;
+		$this->width = $width;
+		$this->height = $height;
+	}
+
+	/** @inheritDoc */
+	public function valid() {
+		return is_resource($this->stream);
+	}
+
+	/** @inheritDoc */
+	public function mimeType() {
+		return $this->mimeType;
+	}
+
+	/** @inheritDoc */
+	public function width() {
+		return $this->width;
+	}
+
+	/** @inheritDoc */
+	public function height() {
+		return $this->height;
+	}
+
+	public function widthTopLeft() {
+		throw new \BadMethodCallException('Not implemented');
+	}
+
+	public function heightTopLeft() {
+		throw new \BadMethodCallException('Not implemented');
+	}
+
+	public function show($mimeType = null) {
+		throw new \BadMethodCallException('Not implemented');
+	}
+
+	public function save($filePath = null, $mimeType = null) {
+		throw new \BadMethodCallException('Not implemented');
+	}
+
+	public function resource() {
+		return $this->stream;
+	}
+
+	public function dataMimeType() {
+		return $this->mimeType;
+	}
+
+	public function data() {
+		return '';
+	}
+
+	public function getOrientation() {
+		throw new \BadMethodCallException('Not implemented');
+	}
+
+	public function fixOrientation() {
+		throw new \BadMethodCallException('Not implemented');
+	}
+
+	public function resize($maxSize) {
+		throw new \BadMethodCallException('Not implemented');
+	}
+
+	public function preciseResize(int $width, int $height): bool {
+		throw new \BadMethodCallException('Not implemented');
+	}
+
+	public function centerCrop($size = 0) {
+		throw new \BadMethodCallException('Not implemented');
+	}
+
+	public function crop(int $x, int $y, int $w, int $h): bool {
+		throw new \BadMethodCallException('Not implemented');
+	}
+
+	public function fitIn($maxWidth, $maxHeight) {
+		throw new \BadMethodCallException('Not implemented');
+	}
+
+	public function scaleDownToFit($maxWidth, $maxHeight) {
+		throw new \BadMethodCallException('Not implemented');
+	}
+
+	public function copy(): IImage {
+		throw new \BadMethodCallException('Not implemented');
+	}
+
+	public function cropCopy(int $x, int $y, int $w, int $h): IImage {
+		throw new \BadMethodCallException('Not implemented');
+	}
+
+	public function preciseResizeCopy(int $width, int $height): IImage {
+		throw new \BadMethodCallException('Not implemented');
+	}
+
+	public function resizeCopy(int $maxSize): IImage {
+		throw new \BadMethodCallException('Not implemented');
+	}
+}
diff --git a/lib/public/IStreamImage.php b/lib/public/IStreamImage.php
new file mode 100644
index 00000000000..a93381dd5ed
--- /dev/null
+++ b/lib/public/IStreamImage.php
@@ -0,0 +1,30 @@
+<?php
+/**
+ * @copyright Copyright (c) 2021 Carl Schwan <carl@carlschwan.eu>
+ *
+ * @author Carl Schwan <carl@carlschwan.eu>
+ *
+ * @license AGPL-3.0-or-later
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+namespace OCP;
+
+/**
+ * @since 24.0.0
+ */
+interface IStreamImage extends IImage {
+}