diff options
39 files changed, 896 insertions, 54 deletions
diff --git a/3rdparty b/3rdparty -Subproject d80ec1fa2dad1c3ede272583e3c4f1f77f40141 +Subproject 6176112be9428026897d958dc2b558d1bde4fec diff --git a/apps/dav/composer/autoload.php b/apps/dav/composer/autoload.php index 06b2e993e94..a3040af8caa 100644 --- a/apps/dav/composer/autoload.php +++ b/apps/dav/composer/autoload.php @@ -2,6 +2,11 @@ // autoload.php @generated by Composer +if (PHP_VERSION_ID < 50600) { + echo 'Composer 2.3.0 dropped support for autoloading on PHP <5.6 and you are running '.PHP_VERSION.', please upgrade PHP or use Composer 2.2 LTS via "composer self-update --2.2". Aborting.'.PHP_EOL; + exit(1); +} + require_once __DIR__ . '/composer/autoload_real.php'; return ComposerAutoloaderInitDAV::getLoader(); diff --git a/apps/dav/composer/composer/autoload_real.php b/apps/dav/composer/composer/autoload_real.php index 8416efa9d7e..4b2290344f5 100644 --- a/apps/dav/composer/composer/autoload_real.php +++ b/apps/dav/composer/composer/autoload_real.php @@ -27,7 +27,7 @@ class ComposerAutoloaderInitDAV spl_autoload_unregister(array('ComposerAutoloaderInitDAV', 'loadClassLoader')); require __DIR__ . '/autoload_static.php'; - \Composer\Autoload\ComposerStaticInitDAV::getInitializer($loader)(); + call_user_func(\Composer\Autoload\ComposerStaticInitDAV::getInitializer($loader)); $loader->setClassMapAuthoritative(true); $loader->register(true); diff --git a/apps/dav/composer/composer/installed.php b/apps/dav/composer/composer/installed.php index baf72c4fb34..628db5d793b 100644 --- a/apps/dav/composer/composer/installed.php +++ b/apps/dav/composer/composer/installed.php @@ -5,7 +5,7 @@ 'type' => 'library', 'install_path' => __DIR__ . '/../', 'aliases' => array(), - 'reference' => 'e2c675724fc4ea50f1275bf0027b96f277c32578', + 'reference' => '9586920c0ec4016864a2219e838fb272127822d8', 'name' => '__root__', 'dev' => false, ), @@ -16,7 +16,7 @@ 'type' => 'library', 'install_path' => __DIR__ . '/../', 'aliases' => array(), - 'reference' => 'e2c675724fc4ea50f1275bf0027b96f277c32578', + 'reference' => '9586920c0ec4016864a2219e838fb272127822d8', 'dev_requirement' => false, ), ), diff --git a/apps/dav/lib/Connector/Sabre/Directory.php b/apps/dav/lib/Connector/Sabre/Directory.php index 8b616b0cb8a..9e0b89596cd 100644 --- a/apps/dav/lib/Connector/Sabre/Directory.php +++ b/apps/dav/lib/Connector/Sabre/Directory.php @@ -34,6 +34,8 @@ namespace OCA\DAV\Connector\Sabre; use OC\Files\Mount\MoveableMount; use OC\Files\View; +use OC\Metadata\FileMetadata; +use OC\Metadata\MetadataGroup; use OCA\DAV\Connector\Sabre\Exception\FileLocked; use OCA\DAV\Connector\Sabre\Exception\Forbidden; use OCA\DAV\Connector\Sabre\Exception\InvalidPath; @@ -73,6 +75,9 @@ class Directory extends \OCA\DAV\Connector\Sabre\Node implements \Sabre\DAV\ICol */ private $tree; + /** @var array<string, array<int, FileMetadata>> */ + private array $metadata = []; + /** * Sets up the node, expects a full path name * diff --git a/apps/dav/lib/Connector/Sabre/File.php b/apps/dav/lib/Connector/Sabre/File.php index a46ca372be7..6c379984995 100644 --- a/apps/dav/lib/Connector/Sabre/File.php +++ b/apps/dav/lib/Connector/Sabre/File.php @@ -43,6 +43,7 @@ use OC\AppFramework\Http\Request; use OC\Files\Filesystem; use OC\Files\Stream\HashWrapper; use OC\Files\View; +use OC\Metadata\FileMetadata; use OCA\DAV\AppInfo\Application; use OCA\DAV\Connector\Sabre\Exception\EntityTooLarge; use OCA\DAV\Connector\Sabre\Exception\FileLocked; @@ -80,6 +81,9 @@ class File extends Node implements IFile { protected IL10N $l10n; + /** @var array<string, FileMetadata> */ + private array $metadata = []; + /** * Sets up the node, expects a full path name * @@ -757,4 +761,16 @@ class File extends Node implements IFile { public function getNode(): \OCP\Files\File { return $this->node; } + + public function getMetadata(string $group): FileMetadata { + return $this->metadata[$group]; + } + + public function setMetadata(string $group, FileMetadata $metadata): void { + $this->metadata[$group] = $metadata; + } + + public function hasMetadata(string $group) { + return array_key_exists($group, $this->metadata); + } } diff --git a/apps/dav/lib/Connector/Sabre/FilesPlugin.php b/apps/dav/lib/Connector/Sabre/FilesPlugin.php index 180f05c0e7e..5cc562e0ff8 100644 --- a/apps/dav/lib/Connector/Sabre/FilesPlugin.php +++ b/apps/dav/lib/Connector/Sabre/FilesPlugin.php @@ -34,6 +34,7 @@ namespace OCA\DAV\Connector\Sabre; use OC\AppFramework\Http\Request; +use OC\Metadata\IMetadataManager; use OCP\Constants; use OCP\Files\ForbiddenException; use OCP\Files\StorageNotAvailableException; @@ -41,6 +42,7 @@ use OCP\IConfig; use OCP\IPreview; use OCP\IRequest; use OCP\IUserSession; +use Psr\Log\LoggerInterface; use Sabre\DAV\Exception\Forbidden; use Sabre\DAV\Exception\NotFound; use Sabre\DAV\IFile; @@ -50,6 +52,7 @@ use Sabre\DAV\ServerPlugin; use Sabre\DAV\Tree; use Sabre\HTTP\RequestInterface; use Sabre\HTTP\ResponseInterface; +use Sabre\Uri; class FilesPlugin extends ServerPlugin { @@ -79,6 +82,7 @@ class FilesPlugin extends ServerPlugin { public const SHARE_NOTE = '{http://nextcloud.org/ns}note'; public const SUBFOLDER_COUNT_PROPERTYNAME = '{http://nextcloud.org/ns}contained-folder-count'; public const SUBFILE_COUNT_PROPERTYNAME = '{http://nextcloud.org/ns}contained-file-count'; + public const FILE_METADATA_SIZE = '{http://nextcloud.org/ns}file-metadata-size'; /** * Reference to main server object @@ -436,6 +440,29 @@ class FilesPlugin extends ServerPlugin { $propFind->handle(self::UPLOAD_TIME_PROPERTYNAME, function () use ($node) { return $node->getFileInfo()->getUploadTime(); }); + + if ($this->config->getSystemValueBool('enable_file_metadata', true)) { + $propFind->handle(self::FILE_METADATA_SIZE, function () use ($node) { + if (!str_starts_with($node->getFileInfo()->getMimetype(), 'image')) { + return json_encode([]); + } + + if ($node->hasMetadata('size')) { + $sizeMetadata = $node->getMetadata('size'); + } else { + // This code path should not be called since we try to preload + // the metadata when loading the folder or the search results + // in one go + $metadataManager = \OC::$server->get(IMetadataManager::class); + $sizeMetadata = $metadataManager->fetchMetadataFor('size', [$node->getId()])[$node->getId()]; + + // TODO would be nice to display this in the profiler... + \OC::$server->get(LoggerInterface::class)->warning('Inefficient fetching of metadata'); + } + + return json_encode($sizeMetadata->getMetadata()); + }); + } } if ($node instanceof Directory) { @@ -448,6 +475,32 @@ class FilesPlugin extends ServerPlugin { }); $requestProperties = $propFind->getRequestedProperties(); + + // TODO detect dynamically which metadata groups are requested and + // preload all of them and not just size + if ($this->config->getSystemValueBool('enable_file_metadata', true) + && in_array(self::FILE_METADATA_SIZE, $requestProperties, true)) { + // Preloading of the metadata + $fileIds = []; + foreach ($node->getChildren() as $child) { + /** @var \OCP\Files\Node|Node $child */ + if (str_starts_with($child->getFileInfo()->getMimeType(), 'image/')) { + /** @var File $child */ + $fileIds[] = $child->getFileInfo()->getId(); + } + } + /** @var IMetaDataManager $metadataManager */ + $metadataManager = \OC::$server->get(IMetadataManager::class); + $preloadedMetadata = $metadataManager->fetchMetadataFor('size', $fileIds); + foreach ($node->getChildren() as $child) { + /** @var \OCP\Files\Node|Node $child */ + if (str_starts_with($child->getFileInfo()->getMimeType(), 'image')) { + /** @var File $child */ + $child->setMetadata('size', $preloadedMetadata[$child->getFileInfo()->getId()]); + } + } + } + if (in_array(self::SUBFILE_COUNT_PROPERTYNAME, $requestProperties, true) || in_array(self::SUBFOLDER_COUNT_PROPERTYNAME, $requestProperties, true)) { $nbFiles = 0; diff --git a/apps/dav/lib/Files/FileSearchBackend.php b/apps/dav/lib/Files/FileSearchBackend.php index 45e911db182..21eb14a29bd 100644 --- a/apps/dav/lib/Files/FileSearchBackend.php +++ b/apps/dav/lib/Files/FileSearchBackend.php @@ -30,6 +30,7 @@ use OC\Files\Search\SearchComparison; use OC\Files\Search\SearchOrder; use OC\Files\Search\SearchQuery; use OC\Files\View; +use OC\Metadata\IMetadataManager; use OCA\DAV\Connector\Sabre\CachingTree; use OCA\DAV\Connector\Sabre\Directory; use OCA\DAV\Connector\Sabre\FilesPlugin; @@ -44,6 +45,7 @@ use OCP\Files\Search\ISearchQuery; use OCP\IUser; use OCP\Share\IManager; use Sabre\DAV\Exception\NotFound; +use Sabre\DAV\INode; use SearchDAV\Backend\ISearchBackend; use SearchDAV\Backend\SearchPropertyDefinition; use SearchDAV\Backend\SearchResult; @@ -88,14 +90,12 @@ class FileSearchBackend implements ISearchBackend { /** * Search endpoint will be remote.php/dav - * - * @return string */ - public function getArbiterPath() { + public function getArbiterPath(): string { return ''; } - public function isValidScope($href, $depth, $path) { + public function isValidScope(string $href, $depth, ?string $path): bool { // only allow scopes inside the dav server if (is_null($path)) { return false; @@ -109,7 +109,7 @@ class FileSearchBackend implements ISearchBackend { } } - public function getPropertyDefinitionsForScope($href, $path) { + public function getPropertyDefinitionsForScope(string $href, ?string $path): array { // all valid scopes support the same schema //todo dynamically load all propfind properties that are supported @@ -132,15 +132,44 @@ class FileSearchBackend implements ISearchBackend { new SearchPropertyDefinition(FilesPlugin::OWNER_DISPLAY_NAME_PROPERTYNAME, false, true, false), new SearchPropertyDefinition(FilesPlugin::DATA_FINGERPRINT_PROPERTYNAME, false, true, false), new SearchPropertyDefinition(FilesPlugin::HAS_PREVIEW_PROPERTYNAME, false, true, false, SearchPropertyDefinition::DATATYPE_BOOLEAN), + new SearchPropertyDefinition(FilesPlugin::FILE_METADATA_SIZE, false, true, false, SearchPropertyDefinition::DATATYPE_STRING), new SearchPropertyDefinition(FilesPlugin::FILEID_PROPERTYNAME, false, true, false, SearchPropertyDefinition::DATATYPE_NONNEGATIVE_INTEGER), ]; } /** + * @param INode[] $nodes + * @param string[] $requestProperties + */ + public function preloadPropertyFor(array $nodes, array $requestProperties): void { + if (in_array(FilesPlugin::FILE_METADATA_SIZE, $requestProperties, true)) { + // Preloading of the metadata + $fileIds = []; + foreach ($nodes as $node) { + /** @var \OCP\Files\Node|\OCA\DAV\Connector\Sabre\Node $node */ + if (str_starts_with($node->getFileInfo()->getMimeType(), 'image/')) { + /** @var \OCA\DAV\Connector\Sabre\File $node */ + $fileIds[] = $node->getFileInfo()->getId(); + } + } + /** @var IMetaDataManager $metadataManager */ + $metadataManager = \OC::$server->get(IMetadataManager::class); + $preloadedMetadata = $metadataManager->fetchMetadataFor('size', $fileIds); + foreach ($nodes as $node) { + /** @var \OCP\Files\Node|\OCA\DAV\Connector\Sabre\Node $node */ + if (str_starts_with($node->getFileInfo()->getMimeType(), 'image/')) { + /** @var \OCA\DAV\Connector\Sabre\File $node */ + $node->setMetadata('size', $preloadedMetadata[$node->getFileInfo()->getId()]); + } + } + } + } + + /** * @param Query $search * @return SearchResult[] */ - public function search(Query $search) { + public function search(Query $search): array { if (count($search->from) !== 1) { throw new \InvalidArgumentException('Searching more than one folder is not supported'); } diff --git a/apps/dav/lib/Files/LazySearchBackend.php b/apps/dav/lib/Files/LazySearchBackend.php index d84c11306e3..c3b2f27d72a 100644 --- a/apps/dav/lib/Files/LazySearchBackend.php +++ b/apps/dav/lib/Files/LazySearchBackend.php @@ -22,6 +22,7 @@ */ namespace OCA\DAV\Files; +use Sabre\DAV\INode; use SearchDAV\Backend\ISearchBackend; use SearchDAV\Query\Query; @@ -35,7 +36,7 @@ class LazySearchBackend implements ISearchBackend { $this->backend = $backend; } - public function getArbiterPath() { + public function getArbiterPath(): string { if ($this->backend) { return $this->backend->getArbiterPath(); } else { @@ -43,27 +44,30 @@ class LazySearchBackend implements ISearchBackend { } } - public function isValidScope($href, $depth, $path) { + public function isValidScope(string $href, $depth, ?string $path): bool { if ($this->backend) { return $this->backend->getArbiterPath(); - } else { - return false; } + return false; } - public function getPropertyDefinitionsForScope($href, $path) { + public function getPropertyDefinitionsForScope(string $href, ?String $path): array { if ($this->backend) { return $this->backend->getPropertyDefinitionsForScope($href, $path); - } else { - return []; } + return []; } - public function search(Query $query) { + public function search(Query $query): array { if ($this->backend) { return $this->backend->search($query); - } else { - return []; + } + return []; + } + + public function preloadPropertyFor(array $nodes, array $requestProperties): void { + if ($this->backend) { + $this->backend->preloadPropertyFor($nodes, $requestProperties); } } } diff --git a/build/psalm-baseline.xml b/build/psalm-baseline.xml index 73891b59918..863226922d8 100644 --- a/build/psalm-baseline.xml +++ b/build/psalm-baseline.xml @@ -826,10 +826,6 @@ </InvalidScalarArgument> </file> <file src="apps/dav/lib/Files/FileSearchBackend.php"> - <InvalidArgument occurrences="2"> - <code>$argument</code> - <code>$operator->arguments</code> - </InvalidArgument> <InvalidReturnStatement occurrences="1"> <code>$value</code> </InvalidReturnStatement> @@ -839,9 +835,6 @@ <ParamNameMismatch occurrences="1"> <code>$search</code> </ParamNameMismatch> - <UndefinedDocblockClass occurrences="1"> - <code>$operator->arguments[0]->name</code> - </UndefinedDocblockClass> <UndefinedPropertyFetch occurrences="1"> <code>$operator->arguments[0]->name</code> </UndefinedPropertyFetch> @@ -855,9 +848,7 @@ <InvalidReturnStatement occurrences="1"> <code>$this->backend->getArbiterPath()</code> </InvalidReturnStatement> - <InvalidReturnType occurrences="1"> - <code>isValidScope</code> - </InvalidReturnType> + <InvalidReturnType occurrences="1"/> </file> <file src="apps/dav/lib/Files/RootCollection.php"> <UndefinedFunction occurrences="1"> diff --git a/config/config.sample.php b/config/config.sample.php index 4d8dcfa5660..378d88168cd 100644 --- a/config/config.sample.php +++ b/config/config.sample.php @@ -2125,4 +2125,15 @@ $CONFIG = [ * Defaults to ``true`` */ 'profile.enabled' => true, + +/** + * Enable file metadata collection + * + * This is helpful for the mobile clients and will enable a few optimization in + * the future for the preview generation. + * + * Note that when enabled, this data will be stored in the database and might increase + * the database storage. + */ +'enable_file_metadata' => true, ]; diff --git a/core/Application.php b/core/Application.php index 545588ab208..34932cab183 100644 --- a/core/Application.php +++ b/core/Application.php @@ -48,12 +48,17 @@ use OC\DB\MissingColumnInformation; use OC\DB\MissingIndexInformation; use OC\DB\MissingPrimaryKeyInformation; use OC\DB\SchemaWrapper; +use OC\Metadata\FileEventListener; use OCP\AppFramework\App; use OCP\EventDispatcher\IEventDispatcher; +use OCP\Files\Events\Node\NodeDeletedEvent; +use OCP\Files\Events\Node\NodeWrittenEvent; +use OCP\Files\Events\NodeRemovedFromCache; use OCP\IDBConnection; use OCP\User\Events\BeforeUserDeletedEvent; use OCP\User\Events\UserDeletedEvent; use OCP\Util; +use OCP\IConfig; use Symfony\Component\EventDispatcher\GenericEvent; /** @@ -301,5 +306,15 @@ class Application extends App { $eventDispatcher->addServiceListener(BeforeUserDeletedEvent::class, UserDeletedFilesCleanupListener::class); $eventDispatcher->addServiceListener(UserDeletedEvent::class, UserDeletedFilesCleanupListener::class); $eventDispatcher->addServiceListener(UserDeletedEvent::class, UserDeletedWebAuthnCleanupListener::class); + + // Metadata + /** @var IConfig $config */ + $config = $container->get(IConfig::class); + if ($config->getSystemValueBool('enable_file_metadata', true)) { + $eventDispatcher = \OC::$server->get(IEventDispatcher::class); + $eventDispatcher->addServiceListener(NodeDeletedEvent::class, FileEventListener::class); + $eventDispatcher->addServiceListener(NodeRemovedFromCache::class, FileEventListener::class); + $eventDispatcher->addServiceListener(NodeWrittenEvent::class, FileEventListener::class); + } } } diff --git a/core/Migrations/Version240000Date20220404230027.php b/core/Migrations/Version240000Date20220404230027.php new file mode 100644 index 00000000000..f45f8d5b500 --- /dev/null +++ b/core/Migrations/Version240000Date20220404230027.php @@ -0,0 +1,62 @@ +<?php + +declare(strict_types=1); +/** + * @copyright Copyright 2022 Carl Schwan <carl@carlschwan.eu> + * + * @license AGPL-3.0 + * + * This code is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License, version 3, + * along with this program. If not, see <http://www.gnu.org/licenses/> + * + */ + +namespace OC\Core\Migrations; + +use Closure; +use OCP\DB\ISchemaWrapper; +use OCP\DB\Types; +use OCP\Migration\IOutput; +use OCP\Migration\SimpleMigrationStep; + +/** + * Add oc_file_metadata table + * @see OC\Metadata\FileMetadata + */ +class Version240000Date20220404230027 extends SimpleMigrationStep { + /** + * @param IOutput $output + * @param Closure $schemaClosure The `\Closure` returns a `ISchemaWrapper` + * @param array $options + * @return null|ISchemaWrapper + */ + public function changeSchema(IOutput $output, Closure $schemaClosure, array $options): ?ISchemaWrapper { + /** @var ISchemaWrapper $schema */ + $schema = $schemaClosure(); + + if (!$schema->hasTable('file_metadata')) { + $table = $schema->createTable('file_metadata'); + $table->addColumn('id', Types::INTEGER, [ + 'notnull' => true, + ]); + $table->addColumn('group_name', Types::STRING, [ + 'notnull' => true, + 'length' => 50, + ]); + $table->addColumn('metadata', Types::JSON, [ + 'notnull' => true, + ]); + $table->setPrimaryKey(['id', 'group_name'], 'file_metadata_idx'); + } + return $schema; + } +} diff --git a/lib/composer/composer/InstalledVersions.php b/lib/composer/composer/InstalledVersions.php index d50e0c9fcc4..fc50a9f8622 100644 --- a/lib/composer/composer/InstalledVersions.php +++ b/lib/composer/composer/InstalledVersions.php @@ -264,7 +264,7 @@ class InstalledVersions if (null === self::$installed) { // only require the installed.php file if this file is loaded from its dumped location, // and not from its source location in the composer/composer package, see https://github.com/composer/composer/issues/9937 - if (substr(__DIR__, -8, 1) !== 'C') { + if (substr(__DIR__, -8, 1) !== 'C' && is_file(__DIR__ . '/installed.php')) { self::$installed = include __DIR__ . '/installed.php'; } else { self::$installed = array(); @@ -337,7 +337,7 @@ class InstalledVersions if (null === self::$installed) { // only require the installed.php file if this file is loaded from its dumped location, // and not from its source location in the composer/composer package, see https://github.com/composer/composer/issues/9937 - if (substr(__DIR__, -8, 1) !== 'C') { + if (substr(__DIR__, -8, 1) !== 'C' && is_file(__DIR__ . '/installed.php')) { self::$installed = require __DIR__ . '/installed.php'; } else { self::$installed = array(); diff --git a/lib/composer/composer/LICENSE b/lib/composer/composer/LICENSE index f27399a042d..62ecfd8d004 100644 --- a/lib/composer/composer/LICENSE +++ b/lib/composer/composer/LICENSE @@ -1,4 +1,3 @@ - Copyright (c) Nils Adermann, Jordi Boggiano Permission is hereby granted, free of charge, to any person obtaining a copy @@ -18,4 +17,3 @@ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - diff --git a/lib/composer/composer/autoload_classmap.php b/lib/composer/composer/autoload_classmap.php index acc0f6bf2ad..be40cd4c607 100644 --- a/lib/composer/composer/autoload_classmap.php +++ b/lib/composer/composer/autoload_classmap.php @@ -2,7 +2,7 @@ // autoload_classmap.php @generated by Composer -$vendorDir = dirname(__DIR__); +$vendorDir = dirname(dirname(__FILE__)); $baseDir = dirname(dirname($vendorDir)); return array( @@ -1021,6 +1021,7 @@ return array( 'OC\\Core\\Migrations\\Version23000Date20211203110726' => $baseDir . '/core/Migrations/Version23000Date20211203110726.php', 'OC\\Core\\Migrations\\Version23000Date20211213203940' => $baseDir . '/core/Migrations/Version23000Date20211213203940.php', 'OC\\Core\\Migrations\\Version240000Date20220202150027' => $baseDir . '/core/Migrations/Version240000Date20220202150027.php', + 'OC\\Core\\Migrations\\Version240000Date20220404230027' => $baseDir . '/core/Migrations/Version240000Date20220404230027.php', 'OC\\Core\\Migrations\\Version24000Date20211210141942' => $baseDir . '/core/Migrations/Version24000Date20211210141942.php', 'OC\\Core\\Migrations\\Version24000Date20211213081506' => $baseDir . '/core/Migrations/Version24000Date20211213081506.php', 'OC\\Core\\Migrations\\Version24000Date20211213081604' => $baseDir . '/core/Migrations/Version24000Date20211213081604.php', @@ -1302,6 +1303,14 @@ return array( 'OC\\Memcache\\ProfilerWrapperCache' => $baseDir . '/lib/private/Memcache/ProfilerWrapperCache.php', 'OC\\Memcache\\Redis' => $baseDir . '/lib/private/Memcache/Redis.php', 'OC\\MemoryInfo' => $baseDir . '/lib/private/MemoryInfo.php', + 'OC\\Metadata\\Capabilities' => $baseDir . '/lib/private/Metadata/Capabilities.php', + 'OC\\Metadata\\FileEventListener' => $baseDir . '/lib/private/Metadata/FileEventListener.php', + 'OC\\Metadata\\FileMetadata' => $baseDir . '/lib/private/Metadata/FileMetadata.php', + 'OC\\Metadata\\FileMetadataMapper' => $baseDir . '/lib/private/Metadata/FileMetadataMapper.php', + 'OC\\Metadata\\IMetadataManager' => $baseDir . '/lib/private/Metadata/IMetadataManager.php', + 'OC\\Metadata\\IMetadataProvider' => $baseDir . '/lib/private/Metadata/IMetadataProvider.php', + 'OC\\Metadata\\MetadataManager' => $baseDir . '/lib/private/Metadata/MetadataManager.php', + 'OC\\Metadata\\Provider\\ExifProvider' => $baseDir . '/lib/private/Metadata/Provider/ExifProvider.php', 'OC\\Migration\\BackgroundRepair' => $baseDir . '/lib/private/Migration/BackgroundRepair.php', 'OC\\Migration\\ConsoleOutput' => $baseDir . '/lib/private/Migration/ConsoleOutput.php', 'OC\\Migration\\SimpleOutput' => $baseDir . '/lib/private/Migration/SimpleOutput.php', diff --git a/lib/composer/composer/autoload_namespaces.php b/lib/composer/composer/autoload_namespaces.php index f1ae7a0ffec..4a9c20beed0 100644 --- a/lib/composer/composer/autoload_namespaces.php +++ b/lib/composer/composer/autoload_namespaces.php @@ -2,7 +2,7 @@ // autoload_namespaces.php @generated by Composer -$vendorDir = dirname(__DIR__); +$vendorDir = dirname(dirname(__FILE__)); $baseDir = dirname(dirname($vendorDir)); return array( diff --git a/lib/composer/composer/autoload_psr4.php b/lib/composer/composer/autoload_psr4.php index 74e48cf69ae..b641d9c6a03 100644 --- a/lib/composer/composer/autoload_psr4.php +++ b/lib/composer/composer/autoload_psr4.php @@ -2,7 +2,7 @@ // autoload_psr4.php @generated by Composer -$vendorDir = dirname(__DIR__); +$vendorDir = dirname(dirname(__FILE__)); $baseDir = dirname(dirname($vendorDir)); return array( diff --git a/lib/composer/composer/autoload_real.php b/lib/composer/composer/autoload_real.php index 4b1ab7678ec..a5748c7a891 100644 --- a/lib/composer/composer/autoload_real.php +++ b/lib/composer/composer/autoload_real.php @@ -23,11 +23,30 @@ class ComposerAutoloaderInit53792487c5a8370acc0b06b1a864ff4c } spl_autoload_register(array('ComposerAutoloaderInit53792487c5a8370acc0b06b1a864ff4c', 'loadClassLoader'), true, true); - self::$loader = $loader = new \Composer\Autoload\ClassLoader(\dirname(__DIR__)); + self::$loader = $loader = new \Composer\Autoload\ClassLoader(\dirname(\dirname(__FILE__))); spl_autoload_unregister(array('ComposerAutoloaderInit53792487c5a8370acc0b06b1a864ff4c', 'loadClassLoader')); - require __DIR__ . '/autoload_static.php'; - \Composer\Autoload\ComposerStaticInit53792487c5a8370acc0b06b1a864ff4c::getInitializer($loader)(); + $useStaticLoader = PHP_VERSION_ID >= 50600 && !defined('HHVM_VERSION') && (!function_exists('zend_loader_file_encoded') || !zend_loader_file_encoded()); + if ($useStaticLoader) { + require __DIR__ . '/autoload_static.php'; + + call_user_func(\Composer\Autoload\ComposerStaticInit53792487c5a8370acc0b06b1a864ff4c::getInitializer($loader)); + } else { + $map = require __DIR__ . '/autoload_namespaces.php'; + foreach ($map as $namespace => $path) { + $loader->set($namespace, $path); + } + + $map = require __DIR__ . '/autoload_psr4.php'; + foreach ($map as $namespace => $path) { + $loader->setPsr4($namespace, $path); + } + + $classMap = require __DIR__ . '/autoload_classmap.php'; + if ($classMap) { + $loader->addClassMap($classMap); + } + } $loader->register(true); diff --git a/lib/composer/composer/autoload_static.php b/lib/composer/composer/autoload_static.php index 09e8d3a627e..7e778d73b83 100644 --- a/lib/composer/composer/autoload_static.php +++ b/lib/composer/composer/autoload_static.php @@ -1050,6 +1050,7 @@ class ComposerStaticInit53792487c5a8370acc0b06b1a864ff4c 'OC\\Core\\Migrations\\Version23000Date20211203110726' => __DIR__ . '/../../..' . '/core/Migrations/Version23000Date20211203110726.php', 'OC\\Core\\Migrations\\Version23000Date20211213203940' => __DIR__ . '/../../..' . '/core/Migrations/Version23000Date20211213203940.php', 'OC\\Core\\Migrations\\Version240000Date20220202150027' => __DIR__ . '/../../..' . '/core/Migrations/Version240000Date20220202150027.php', + 'OC\\Core\\Migrations\\Version240000Date20220404230027' => __DIR__ . '/../../..' . '/core/Migrations/Version240000Date20220404230027.php', 'OC\\Core\\Migrations\\Version24000Date20211210141942' => __DIR__ . '/../../..' . '/core/Migrations/Version24000Date20211210141942.php', 'OC\\Core\\Migrations\\Version24000Date20211213081506' => __DIR__ . '/../../..' . '/core/Migrations/Version24000Date20211213081506.php', 'OC\\Core\\Migrations\\Version24000Date20211213081604' => __DIR__ . '/../../..' . '/core/Migrations/Version24000Date20211213081604.php', @@ -1331,6 +1332,14 @@ class ComposerStaticInit53792487c5a8370acc0b06b1a864ff4c 'OC\\Memcache\\ProfilerWrapperCache' => __DIR__ . '/../../..' . '/lib/private/Memcache/ProfilerWrapperCache.php', 'OC\\Memcache\\Redis' => __DIR__ . '/../../..' . '/lib/private/Memcache/Redis.php', 'OC\\MemoryInfo' => __DIR__ . '/../../..' . '/lib/private/MemoryInfo.php', + 'OC\\Metadata\\Capabilities' => __DIR__ . '/../../..' . '/lib/private/Metadata/Capabilities.php', + 'OC\\Metadata\\FileEventListener' => __DIR__ . '/../../..' . '/lib/private/Metadata/FileEventListener.php', + 'OC\\Metadata\\FileMetadata' => __DIR__ . '/../../..' . '/lib/private/Metadata/FileMetadata.php', + 'OC\\Metadata\\FileMetadataMapper' => __DIR__ . '/../../..' . '/lib/private/Metadata/FileMetadataMapper.php', + 'OC\\Metadata\\IMetadataManager' => __DIR__ . '/../../..' . '/lib/private/Metadata/IMetadataManager.php', + 'OC\\Metadata\\IMetadataProvider' => __DIR__ . '/../../..' . '/lib/private/Metadata/IMetadataProvider.php', + 'OC\\Metadata\\MetadataManager' => __DIR__ . '/../../..' . '/lib/private/Metadata/MetadataManager.php', + 'OC\\Metadata\\Provider\\ExifProvider' => __DIR__ . '/../../..' . '/lib/private/Metadata/Provider/ExifProvider.php', 'OC\\Migration\\BackgroundRepair' => __DIR__ . '/../../..' . '/lib/private/Migration/BackgroundRepair.php', 'OC\\Migration\\ConsoleOutput' => __DIR__ . '/../../..' . '/lib/private/Migration/ConsoleOutput.php', 'OC\\Migration\\SimpleOutput' => __DIR__ . '/../../..' . '/lib/private/Migration/SimpleOutput.php', diff --git a/lib/composer/composer/installed.php b/lib/composer/composer/installed.php index f12a8e00dbe..67a1ddfbd8c 100644 --- a/lib/composer/composer/installed.php +++ b/lib/composer/composer/installed.php @@ -5,7 +5,7 @@ 'type' => 'library', 'install_path' => __DIR__ . '/../../../', 'aliases' => array(), - 'reference' => '1225189f74d06606aafc4150d07584b90cea50dd', + 'reference' => '42c7886f80c7a5e767b192d07474114dd0848b16', 'name' => '__root__', 'dev' => false, ), @@ -16,7 +16,7 @@ 'type' => 'library', 'install_path' => __DIR__ . '/../../../', 'aliases' => array(), - 'reference' => '1225189f74d06606aafc4150d07584b90cea50dd', + 'reference' => '42c7886f80c7a5e767b192d07474114dd0848b16', 'dev_requirement' => false, ), ), diff --git a/lib/private/Files/ObjectStore/NoopScanner.php b/lib/private/Files/ObjectStore/NoopScanner.php index 42e212271d5..3b8cbdb18bb 100644 --- a/lib/private/Files/ObjectStore/NoopScanner.php +++ b/lib/private/Files/ObjectStore/NoopScanner.php @@ -31,7 +31,7 @@ use OC\Files\Storage\Storage; class NoopScanner extends Scanner { public function __construct(Storage $storage) { - //we don't need the storage, so do nothing here + // we don't need the storage, so do nothing here } /** diff --git a/lib/private/Metadata/Capabilities.php b/lib/private/Metadata/Capabilities.php new file mode 100644 index 00000000000..2fa0006f581 --- /dev/null +++ b/lib/private/Metadata/Capabilities.php @@ -0,0 +1,44 @@ +<?php + +declare(strict_types=1); + +/** + * @copyright Copyright 2022 Carl Schwan <carl@carlschwan.eu> + * @license AGPL-3.0-or-later + * + * This code is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License, version 3, + * along with this program. If not, see <http://www.gnu.org/licenses/> + * + */ + +namespace OC\Metadata; + +use OCP\Capabilities\IPublicCapability; +use OCP\IConfig; + +class Capabilities implements IPublicCapability { + private IMetadataManager $manager; + private IConfig $config; + + public function __construct(IMetadataManager $manager, IConfig $config) { + $this->manager = $manager; + $this->config = $config; + } + + public function getCapabilities() { + if ($this->config->getSystemValueBool('enable_file_metadata', true)) { + return ['metadataAvailable' => $this->manager->getCapabilities()]; + } + + return []; + } +} diff --git a/lib/private/Metadata/FileEventListener.php b/lib/private/Metadata/FileEventListener.php new file mode 100644 index 00000000000..fdec891c6e2 --- /dev/null +++ b/lib/private/Metadata/FileEventListener.php @@ -0,0 +1,84 @@ +<?php + +declare(strict_types=1); +/** + * @copyright Copyright 2022 Carl Schwan <carl@carlschwan.eu> + * @license AGPL-3.0-or-later + * + * This code is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License, version 3, + * along with this program. If not, see <http://www.gnu.org/licenses/> + * + */ + +namespace OC\Metadata; + +use OC\Files\Filesystem; +use OCP\EventDispatcher\Event; +use OCP\EventDispatcher\IEventListener; +use OCP\Files\Events\Node\NodeDeletedEvent; +use OCP\Files\Events\Node\NodeWrittenEvent; +use OCP\Files\Events\NodeRemovedFromCache; +use OCP\Files\File; +use OCP\Files\Node; +use OCP\Files\NotFoundException; +use OCP\Files\FileInfo; + +class FileEventListener implements IEventListener { + private IMetadataManager $manager; + + public function __construct(IMetadataManager $manager) { + $this->manager = $manager; + } + + private function shouldExtractMetadata(Node $node): bool { + try { + if ($node->getMimetype() === 'httpd/unix-directory') { + return false; + } + } catch (NotFoundException $e) { + return false; + } + if ($node->getSize(false) <= 0) { + return false; + } + + $path = $node->getPath(); + // TODO make this more dynamic, we have the same issue in other places + return !str_starts_with($path, 'appdata_') && !str_starts_with($path, 'files_versions/') && !str_starts_with($path, 'files_trashbin/'); + } + + public function handle(Event $event): void { + if ($event instanceof NodeRemovedFromCache) { + $view = Filesystem::getView(); + $info = $view->getFileInfo($event->getPath()); + if ($info && $info->getType() === FileInfo::TYPE_FILE) { + $this->manager->clearMetadata($info->getId()); + } + } + + if ($event instanceof NodeDeletedEvent) { + $node = $event->getNode(); + if ($this->shouldExtractMetadata($node)) { + /** @var File $node */ + $this->manager->clearMetadata($event->getNode()->getId()); + } + } + + if ($event instanceof NodeWrittenEvent) { + $node = $event->getNode(); + if ($this->shouldExtractMetadata($node)) { + /** @var File $node */ + $this->manager->generateMetadata($event->getNode(), false); + } + } + } +} diff --git a/lib/private/Metadata/FileMetadata.php b/lib/private/Metadata/FileMetadata.php new file mode 100644 index 00000000000..c53f5d7f619 --- /dev/null +++ b/lib/private/Metadata/FileMetadata.php @@ -0,0 +1,43 @@ +<?php + +declare(strict_types=1); +/** + * @copyright Copyright 2022 Carl Schwan <carl@carlschwan.eu> + * + * @license AGPL-3.0 + * + * This code is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License, version 3, + * along with this program. If not, see <http://www.gnu.org/licenses/> + * + */ + +namespace OC\Metadata; + +use OCP\AppFramework\Db\Entity; +use OCP\DB\Types; + +/** + * @method string getGroupName() + * @method void setGroupName(string $groupName) + * @method string getMetadata() + * @method void setMetadata(array $metadata) + * @see OC\Core\Migrations\Version240000Date20220404230027 + */ +class FileMetadata extends Entity { + protected ?string $groupName = null; + protected ?array $metadata = null; + + public function __construct() { + $this->addType('groupName', 'string'); + $this->addType('metadata', Types::JSON); + } +} diff --git a/lib/private/Metadata/FileMetadataMapper.php b/lib/private/Metadata/FileMetadataMapper.php new file mode 100644 index 00000000000..53f750ae540 --- /dev/null +++ b/lib/private/Metadata/FileMetadataMapper.php @@ -0,0 +1,105 @@ +<?php + +declare(strict_types=1); +/** + * @copyright Copyright 2022 Carl Schwan <carl@carlschwan.eu> + * @license AGPL-3.0-or-later + * + * This code is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License, version 3, + * along with this program. If not, see <http://www.gnu.org/licenses/> + * + */ + +namespace OC\Metadata; + +use OCP\AppFramework\Db\DoesNotExistException; +use OCP\AppFramework\Db\MultipleObjectsReturnedException; +use OCP\AppFramework\Db\QBMapper; +use OCP\DB\Exception; +use OCP\DB\QueryBuilder\IQueryBuilder; +use OCP\IDBConnection; + +class FileMetadataMapper extends QBMapper { + public function __construct(IDBConnection $db) { + parent::__construct($db, 'file_metadata', FileMetadata::class); + } + + /** + * @return FileMetadata[] + * @throws Exception + */ + public function findForFile(int $fileId): array { + $qb = $this->db->getQueryBuilder(); + $qb->select('*') + ->from($this->getTableName()) + ->where($qb->expr()->eq('id', $qb->createNamedParameter($fileId, IQueryBuilder::PARAM_INT))); + + return $this->findEntities($qb); + } + + /** + * @throws DoesNotExistException + * @throws MultipleObjectsReturnedException + * @throws Exception + */ + public function findForGroupForFile(int $fileId, string $groupName): FileMetadata { + $qb = $this->db->getQueryBuilder(); + $qb->select('*') + ->from($this->getTableName()) + ->where($qb->expr()->eq('id', $qb->createNamedParameter($fileId, IQueryBuilder::PARAM_INT_ARRAY))) + ->andWhere($qb->expr()->eq('group_name', $qb->createNamedParameter($groupName, IQueryBuilder::PARAM_STR))); + + return $this->findEntity($qb); + } + + /** + * @return array<int, FileMetadata> + * @throws Exception + */ + public function findForGroupForFiles(array $fileIds, string $groupName): array { + $qb = $this->db->getQueryBuilder(); + $qb->select('*') + ->from($this->getTableName()) + ->where($qb->expr()->in('id', $qb->createParameter('fileIds'))) + ->andWhere($qb->expr()->eq('group_name', $qb->createNamedParameter($groupName, IQueryBuilder::PARAM_STR))); + + $metadata = []; + foreach (array_chunk($fileIds, 1000) as $fileIdsChunk) { + $qb->setParameter('fileIds', $fileIdsChunk, IQueryBuilder::PARAM_INT_ARRAY); + /** @var FileMetadata[] $rawEntities */ + $rawEntities = $this->findEntities($qb); + foreach ($rawEntities as $entity) { + $metadata[$entity->getId()] = $entity; + } + } + + foreach ($fileIds as $id) { + if (isset($metadata[$id])) { + continue; + } + $empty = new FileMetadata(); + $empty->setMetadata([]); + $empty->setGroupName($groupName); + $empty->setId($id); + $metadata[$id] = $empty; + } + return $metadata; + } + + public function clear(int $fileId): void { + $qb = $this->db->getQueryBuilder(); + $qb->delete($this->getTableName()) + ->where($qb->expr()->eq('id', $qb->createNamedParameter($fileId, IQueryBuilder::PARAM_INT))); + + $qb->executeStatement(); + } +} diff --git a/lib/private/Metadata/IMetadataManager.php b/lib/private/Metadata/IMetadataManager.php new file mode 100644 index 00000000000..d2d37f15c25 --- /dev/null +++ b/lib/private/Metadata/IMetadataManager.php @@ -0,0 +1,35 @@ +<?php + +declare(strict_types=1); + +namespace OC\Metadata; + +use OCP\Files\File; + +/** + * Interface to manage additional metadata for files + */ +interface IMetadataManager { + /** + * @param class-string<IMetadataProvider> $className + */ + public function registerProvider(string $className): void; + + /** + * Generate the metadata for one file + */ + public function generateMetadata(File $file, bool $checkExisting = false): void; + + /** + * Clear the metadata for one file + */ + public function clearMetadata(int $fileId): void; + + /** @return array<int, FileMetadata> */ + public function fetchMetadataFor(string $group, array $fileIds): array; + + /** + * Get the capabilites as an array of mimetype regex to the type provided + */ + public function getCapabilities(): array; +} diff --git a/lib/private/Metadata/IMetadataProvider.php b/lib/private/Metadata/IMetadataProvider.php new file mode 100644 index 00000000000..7cbe102a538 --- /dev/null +++ b/lib/private/Metadata/IMetadataProvider.php @@ -0,0 +1,41 @@ +<?php + +namespace OC\Metadata; + +use OCP\Files\File; + +/** + * Interface for the metadata providers. If you want an application to provide + * some metadata, you can use this to store them. + */ +interface IMetadataProvider { + /** + * The list of groups that this metadata provider is able to provide. + * + * @return string[] + */ + public static function groupsProvided(): array; + + /** + * Check if the metadata provider is available. A metadata provider might be + * unavailable due to a php extension not being installed. + */ + public static function isAvailable(): bool; + + /** + * Get the mimetypes supported as a regex. + */ + public static function getMimetypesSupported(): string; + + /** + * Execute the extraction on the specified file. The metadata should be + * grouped by metadata + * + * Each group should be json serializable and the string representation + * shouldn't be longer than 4000 characters. + * + * @param File $file The file to extract the metadata from + * @param array<string, FileMetadata> An array containing all the metadata fetched. + */ + public function execute(File $file): array; +} diff --git a/lib/private/Metadata/MetadataManager.php b/lib/private/Metadata/MetadataManager.php new file mode 100644 index 00000000000..69e9cb3c852 --- /dev/null +++ b/lib/private/Metadata/MetadataManager.php @@ -0,0 +1,100 @@ +<?php +/** + * @copyright Copyright 2022 Carl Schwan <carl@carlschwan.eu> + * @license AGPL-3.0-or-later + * + * This code is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License, version 3, + * along with this program. If not, see <http://www.gnu.org/licenses/> + * + */ + +namespace OC\Metadata; + +use OC\Metadata\Provider\ExifProvider; +use OCP\Files\File; +use OCP\IConfig; +use Psr\Log\LoggerInterface; + +class MetadataManager implements IMetadataManager { + /** @var array<string, IMetadataProvider> */ + private array $providers; + private array $providerClasses; + private FileMetadataMapper $fileMetadataMapper; + private IConfig $config; + private LoggerInterface $logger; + + public function __construct( + FileMetadataMapper $fileMetadataMapper, + IConfig $config, + LoggerInterface $logger + ) { + $this->providers = []; + $this->providerClasses = []; + $this->fileMetadataMapper = $fileMetadataMapper; + $this->config = $config; + $this->logger = $logger; + + // TODO move to another place, where? + $this->registerProvider(ExifProvider::class); + } + + /** + * @param class-string<IMetadataProvider> $className + */ + public function registerProvider(string $className):void { + if (in_array($className, $this->providerClasses)) { + return; + } + + if (call_user_func([$className, 'isAvailable'])) { + $this->providers[call_user_func([$className, 'getMimetypesSupported'])] = \OC::$server->get($className); + } + } + + public function generateMetadata(File $file, bool $checkExisting = false): void { + $existingMetadataGroups = []; + + if ($checkExisting) { + $existingMetadata = $this->fileMetadataMapper->findForFile($file->getId()); + foreach ($existingMetadata as $metadata) { + $existingMetadataGroups[] = $metadata->getGroupName(); + } + } + + foreach ($this->providers as $supportedMimetype => $provider) { + if (preg_match($supportedMimetype, $file->getMimeType())) { + if (count(array_diff($provider::groupsProvided(), $existingMetadataGroups)) > 0) { + $metaDataGroup = $provider->execute($file); + foreach ($metaDataGroup as $group => $metadata) { + $this->fileMetadataMapper->insertOrUpdate($metadata); + } + } + } + } + } + + public function clearMetadata(int $fileId): void { + $this->fileMetadataMapper->clear($fileId); + } + + public function fetchMetadataFor(string $group, array $fileIds): array { + return $this->fileMetadataMapper->findForGroupForFiles($fileIds, $group); + } + + public function getCapabilities(): array { + $capabilities = []; + foreach ($this->providers as $supportedMimetype => $provider) { + $capabilities[$supportedMimetype] = $provider::groupsProvided(); + } + return $capabilities; + } +} diff --git a/lib/private/Metadata/Provider/ExifProvider.php b/lib/private/Metadata/Provider/ExifProvider.php new file mode 100644 index 00000000000..91c858f6794 --- /dev/null +++ b/lib/private/Metadata/Provider/ExifProvider.php @@ -0,0 +1,51 @@ +<?php + +namespace OC\Metadata\Provider; + +use OC\Metadata\FileMetadata; +use OC\Metadata\IMetadataProvider; +use OCP\Files\File; + +class ExifProvider implements IMetadataProvider { + public static function groupsProvided(): array { + return ['size']; + } + + public static function isAvailable(): bool { + return extension_loaded('exif'); + } + + public function execute(File $file): array { + $fileDescriptor = $file->fopen('rb'); + $data = @exif_read_data($fileDescriptor, 'ANY_TAG', true); + + $size = new FileMetadata(); + $size->setGroupName('size'); + $size->setId($file->getId()); + $size->setMetadata([]); + + if (!$data) { + return [ + 'size' => $size, + ]; + } + + if (array_key_exists('COMPUTED', $data) + && array_key_exists('Width', $data['COMPUTED']) + && array_key_exists('Height', $data['COMPUTED']) + ) { + $size->setMetadata([ + 'width' => $data['COMPUTED']['Width'], + 'height' => $data['COMPUTED']['Height'], + ]); + } + + return [ + 'size' => $size, + ]; + } + + public static function getMimetypesSupported(): string { + return '/image\/.*/'; + } +} diff --git a/lib/private/Server.php b/lib/private/Server.php index 7817d1beafe..e9d673d3746 100644 --- a/lib/private/Server.php +++ b/lib/private/Server.php @@ -122,6 +122,9 @@ use OC\Log\PsrLoggerAdapter; use OC\Mail\Mailer; use OC\Memcache\ArrayCache; use OC\Memcache\Factory; +use OC\Metadata\Capabilities as MetadataCapabilities; +use OC\Metadata\IMetadataManager; +use OC\Metadata\MetadataManager; use OC\Notification\Manager; use OC\OCS\DiscoveryService; use OC\Preview\GeneratorHelper; @@ -151,7 +154,6 @@ use OC\Template\JSCombiner; use OCA\Theming\ImageManager; use OCA\Theming\ThemingDefaults; use OCA\Theming\Util; -use OCA\WorkflowEngine\Service\Logger; use OCP\Accounts\IAccountManager; use OCP\App\IAppManager; use OCP\Authentication\LoginCredentials\IStore; @@ -241,15 +243,12 @@ use OCP\SystemTag\ISystemTagManager; use OCP\SystemTag\ISystemTagObjectMapper; use OCP\Talk\IBroker; use OCP\User\Events\BeforePasswordUpdatedEvent; -use OCP\User\Events\BeforeUserCreatedEvent; -use OCP\User\Events\BeforeUserDeletedEvent; use OCP\User\Events\BeforeUserLoggedInEvent; use OCP\User\Events\BeforeUserLoggedInWithCookieEvent; use OCP\User\Events\BeforeUserLoggedOutEvent; use OCP\User\Events\PasswordUpdatedEvent; use OCP\User\Events\PostLoginEvent; use OCP\User\Events\UserChangedEvent; -use OCP\User\Events\UserDeletedEvent; use OCP\User\Events\UserLoggedInEvent; use OCP\User\Events\UserLoggedInWithCookieEvent; use OCP\User\Events\UserLoggedOutEvent; @@ -1163,6 +1162,9 @@ class Server extends ServerContainer implements IServerContainer { $manager->registerCapability(function () use ($c) { return $c->get(\OC\Security\Bruteforce\Capabilities::class); }); + $manager->registerCapability(function () use ($c) { + return $c->get(MetadataCapabilities::class); + }); return $manager; }); /** @deprecated 19.0.0 */ @@ -1433,6 +1435,8 @@ class Server extends ServerContainer implements IServerContainer { $this->registerAlias(IBroker::class, Broker::class); + $this->registerAlias(IMetadataManager::class, MetadataManager::class); + $this->connectDispatcher(); } diff --git a/lib/public/AppFramework/Db/Entity.php b/lib/public/AppFramework/Db/Entity.php index 89e8f69859e..a059e3a27b0 100644 --- a/lib/public/AppFramework/Db/Entity.php +++ b/lib/public/AppFramework/Db/Entity.php @@ -120,6 +120,10 @@ abstract class Entity { if (!$args[0] instanceof \DateTime) { $args[0] = new \DateTime($args[0]); } + } elseif ($type === 'json') { + if (!is_array($args[0])) { + $args[0] = json_decode($args[0], true); + } } else { settype($args[0], $type); } diff --git a/lib/public/AppFramework/Db/QBMapper.php b/lib/public/AppFramework/Db/QBMapper.php index 5124650bc19..fa753a09dcf 100644 --- a/lib/public/AppFramework/Db/QBMapper.php +++ b/lib/public/AppFramework/Db/QBMapper.php @@ -253,6 +253,8 @@ abstract class QBMapper { return IQueryBuilder::PARAM_LOB; case 'datetime': return IQueryBuilder::PARAM_DATE; + case 'json': + return IQueryBuilder::PARAM_JSON; } return IQueryBuilder::PARAM_STR; diff --git a/lib/public/DB/QueryBuilder/IQueryBuilder.php b/lib/public/DB/QueryBuilder/IQueryBuilder.php index 76754f7bf41..afca9e372ee 100644 --- a/lib/public/DB/QueryBuilder/IQueryBuilder.php +++ b/lib/public/DB/QueryBuilder/IQueryBuilder.php @@ -65,6 +65,11 @@ interface IQueryBuilder { public const PARAM_DATE = 'datetime'; /** + * @since 24.0.0 + */ + public const PARAM_JSON = 'json'; + + /** * @since 9.0.0 */ public const PARAM_INT_ARRAY = Connection::PARAM_INT_ARRAY; diff --git a/lib/public/DB/Types.php b/lib/public/DB/Types.php index 4636ac3389f..31a474b03a0 100644 --- a/lib/public/DB/Types.php +++ b/lib/public/DB/Types.php @@ -110,4 +110,10 @@ final class Types { * @since 21.0.0 */ public const TIME = 'time'; + + /** + * @var string + * @since 24.0.0 + */ + public const JSON = 'json'; } diff --git a/tests/lib/AppFramework/Db/QBMapperTest.php b/tests/lib/AppFramework/Db/QBMapperTest.php index b03034056d2..96d319923b3 100644 --- a/tests/lib/AppFramework/Db/QBMapperTest.php +++ b/tests/lib/AppFramework/Db/QBMapperTest.php @@ -47,6 +47,7 @@ class QBTestEntity extends Entity { protected $stringProp; protected $integerProp; protected $booleanProp; + protected $jsonProp; public function __construct() { $this->addType('intProp', 'int'); @@ -54,11 +55,10 @@ class QBTestEntity extends Entity { $this->addType('stringProp', 'string'); $this->addType('integerProp', 'integer'); $this->addType('booleanProp', 'boolean'); + $this->addType('jsonProp', 'json'); } } -; - /** * Class QBTestMapper * @@ -69,7 +69,7 @@ class QBTestMapper extends QBMapper { parent::__construct($db, 'table'); } - public function getParameterTypeForPropertyForTest(Entity $entity, string $property): int { + public function getParameterTypeForPropertyForTest(Entity $entity, string $property) { return parent::getParameterTypeForProperty($entity, $property); } } @@ -171,6 +171,7 @@ class QBMapperTest extends \Test\TestCase { $entity->setStringProp('string'); $entity->setIntegerProp(456); $entity->setBooleanProp(false); + $entity->setJsonProp(["hello" => "world"]); $idParam = $this->qb->createNamedParameter('id', IQueryBuilder::PARAM_INT); $intParam = $this->qb->createNamedParameter('int_prop', IQueryBuilder::PARAM_INT); @@ -178,8 +179,9 @@ class QBMapperTest extends \Test\TestCase { $stringParam = $this->qb->createNamedParameter('string_prop', IQueryBuilder::PARAM_STR); $integerParam = $this->qb->createNamedParameter('integer_prop', IQueryBuilder::PARAM_INT); $booleanParam = $this->qb->createNamedParameter('boolean_prop', IQueryBuilder::PARAM_BOOL); + $jsonParam = $this->qb->createNamedParameter('json_prop', IQueryBuilder::PARAM_JSON); - $this->qb->expects($this->exactly(6)) + $this->qb->expects($this->exactly(7)) ->method('createNamedParameter') ->withConsecutive( [$this->equalTo(123), $this->equalTo(IQueryBuilder::PARAM_INT)], @@ -187,17 +189,19 @@ class QBMapperTest extends \Test\TestCase { [$this->equalTo('string'), $this->equalTo(IQueryBuilder::PARAM_STR)], [$this->equalTo(456), $this->equalTo(IQueryBuilder::PARAM_INT)], [$this->equalTo(false), $this->equalTo(IQueryBuilder::PARAM_BOOL)], - [$this->equalTo(789), $this->equalTo(IQueryBuilder::PARAM_INT)] + [$this->equalTo(["hello" => "world"]), $this->equalTo(IQueryBuilder::PARAM_JSON)], + [$this->equalTo(789), $this->equalTo(IQueryBuilder::PARAM_INT)], ); - $this->qb->expects($this->exactly(5)) + $this->qb->expects($this->exactly(6)) ->method('set') ->withConsecutive( [$this->equalTo('int_prop'), $this->equalTo($intParam)], [$this->equalTo('bool_prop'), $this->equalTo($boolParam)], [$this->equalTo('string_prop'), $this->equalTo($stringParam)], [$this->equalTo('integer_prop'), $this->equalTo($integerParam)], - [$this->equalTo('boolean_prop'), $this->equalTo($booleanParam)] + [$this->equalTo('boolean_prop'), $this->equalTo($booleanParam)], + [$this->equalTo('json_prop'), $this->equalTo($jsonParam)] ); $this->expr->expects($this->once()) @@ -227,6 +231,9 @@ class QBMapperTest extends \Test\TestCase { $stringType = $this->mapper->getParameterTypeForPropertyForTest($entity, 'stringProp'); $this->assertEquals(IQueryBuilder::PARAM_STR, $stringType, 'String type property mapping incorrect'); + $jsonType = $this->mapper->getParameterTypeForPropertyForTest($entity, 'jsonProp'); + $this->assertEquals(IQueryBuilder::PARAM_JSON, $jsonType, 'JSON type property mapping incorrect'); + $unknownType = $this->mapper->getParameterTypeForPropertyForTest($entity, 'someProp'); $this->assertEquals(IQueryBuilder::PARAM_STR, $unknownType, 'Unknown type property mapping incorrect'); } diff --git a/tests/lib/DB/MigratorTest.php b/tests/lib/DB/MigratorTest.php index 114cf03d7be..af44159efa3 100644 --- a/tests/lib/DB/MigratorTest.php +++ b/tests/lib/DB/MigratorTest.php @@ -267,6 +267,8 @@ class MigratorTest extends \Test\TestCase { [ParameterType::INTEGER, 1234, Types::INTEGER, false], [ParameterType::INTEGER, 0, Types::INTEGER, false], // Integer 0 is not stored as Null and therefor works + + [ParameterType::STRING, '{"a": 2}', Types::JSON, false], ]; } diff --git a/tests/lib/Metadata/FileMetadataMapperTest.php b/tests/lib/Metadata/FileMetadataMapperTest.php new file mode 100644 index 00000000000..8e385351be2 --- /dev/null +++ b/tests/lib/Metadata/FileMetadataMapperTest.php @@ -0,0 +1,83 @@ +<?php + +declare(strict_types=1); + +/** + * @copyright 2022 Carl Schwan <carl@carlschwan.eu> + * @license AGPL-3.0-or-later + * + * This code is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License, version 3, + * along with this program. If not, see <http://www.gnu.org/licenses/> + * + */ + +namespace Test\Metadata; + +use OC\Metadata\FileMetadataMapper; +use OC\Metadata\FileMetadata; + +/** + * @group DB + * @package Test\DB\QueryBuilder + */ +class FileMetadataMapperTest extends \Test\TestCase { + /** @var IDBConnection */ + protected $connection; + + /** @var SystemConfig|\PHPUnit\Framework\MockObject\MockObject */ + protected $config; + + protected function setUp(): void { + parent::setUp(); + + $this->connection = \OC::$server->getDatabaseConnection(); + $this->mapper = new FileMetadataMapper($this->connection); + } + + public function testFindForGroupForFiles() { + $file1 = new FileMetadata(); + $file1->setId(1); + $file1->setGroupName('size'); + $file1->setMetadata([]); + + $file2 = new FileMetadata(); + $file2->setId(2); + $file2->setGroupName('size'); + $file2->setMetadata(['width' => 293, 'height' => 23]); + + // not added, it's the default + $file3 = new FileMetadata(); + $file3->setId(3); + $file3->setGroupName('size'); + $file3->setMetadata([]); + + $file4 = new FileMetadata(); + $file4->setId(4); + $file4->setGroupName('size'); + $file4->setMetadata(['complex' => ["yes", "maybe" => 34.0]]); + + $this->mapper->insert($file1); + $this->mapper->insert($file2); + $this->mapper->insert($file4); + + $files = $this->mapper->findForGroupForFiles([1, 2, 3, 4], 'size'); + + $this->assertEquals($files[1]->getMetadata(), $file1->getMetadata()); + $this->assertEquals($files[2]->getMetadata(), $file2->getMetadata()); + $this->assertEquals($files[3]->getMetadata(), $file3->getMetadata()); + $this->assertEquals($files[4]->getMetadata(), $file4->getMetadata()); + + $this->mapper->clear(1); + $this->mapper->clear(2); + $this->mapper->clear(4); + } +} diff --git a/version.php b/version.php index 8ceb2dd58c0..d805557260d 100644 --- a/version.php +++ b/version.php @@ -30,7 +30,7 @@ // between betas, final and RCs. This is _not_ the public version number. Reset minor/patchlevel // when updating major/minor version number. -$OC_Version = [24, 0, 0, 8]; +$OC_Version = [24, 0, 0, 9]; // The human readable string $OC_VersionString = '24.0.0 beta 3'; |