From e8221303e9634b6ca4e6c0b9deb60f4dda6b3d2c Mon Sep 17 00:00:00 2001 From: Robin Appelman Date: Fri, 26 Mar 2021 17:10:25 +0100 Subject: use search query for Folder::getRecent Signed-off-by: Robin Appelman --- lib/private/Files/Node/Folder.php | 196 +++++++------------------------------- 1 file changed, 36 insertions(+), 160 deletions(-) diff --git a/lib/private/Files/Node/Folder.php b/lib/private/Files/Node/Folder.php index 14b663d4e16..bead0b19c75 100644 --- a/lib/private/Files/Node/Folder.php +++ b/lib/private/Files/Node/Folder.php @@ -31,14 +31,11 @@ namespace OC\Files\Node; -use OC\DB\QueryBuilder\Literal; use OC\Files\Search\SearchBinaryOperator; use OC\Files\Search\SearchComparison; +use OC\Files\Search\SearchOrder; use OC\Files\Search\SearchQuery; -use OC\Files\Storage\Wrapper\Jail; use OC\Files\Storage\Storage; -use OCA\Files_Sharing\SharedStorage; -use OCP\DB\QueryBuilder\IQueryBuilder; use OCP\Files\Cache\ICacheEntry; use OCP\Files\Config\ICachedMountInfo; use OCP\Files\FileInfo; @@ -48,6 +45,7 @@ use OCP\Files\NotPermittedException; use OCP\Files\Search\ISearchBinaryOperator; use OCP\Files\Search\ISearchComparison; use OCP\Files\Search\ISearchOperator; +use OCP\Files\Search\ISearchOrder; use OCP\Files\Search\ISearchQuery; use OCP\IUserManager; @@ -266,8 +264,7 @@ class Folder extends Node implements \OCP\Files\Folder { new SearchBinaryOperator(ISearchBinaryOperator::OPERATOR_AND, [ new SearchComparison(ISearchComparison::COMPARE_LIKE, 'path', $internalPath . '%'), $query->getSearchOperation(), - ] - ), + ]), $subQueryLimit, 0, $query->getOrder(), @@ -309,7 +306,7 @@ class Folder extends Node implements \OCP\Files\Folder { $order = $query->getOrder(); if ($order) { - usort($files, function (FileInfo $a,FileInfo $b) use ($order) { + usort($files, function (FileInfo $a, FileInfo $b) use ($order) { foreach ($order as $orderField) { $cmp = $orderField->sortFileInfo($a, $b); if ($cmp !== 0) { @@ -492,158 +489,37 @@ class Folder extends Node implements \OCP\Files\Folder { * @return \OCP\Files\Node[] */ public function getRecent($limit, $offset = 0) { - $mimetypeLoader = \OC::$server->getMimeTypeLoader(); - $mounts = $this->root->getMountsIn($this->path); - $mounts[] = $this->getMountPoint(); - - $mounts = array_filter($mounts, function (IMountPoint $mount) { - return $mount->getStorage() !== null; - }); - $storageIds = array_map(function (IMountPoint $mount) { - return $mount->getStorage()->getCache()->getNumericStorageId(); - }, $mounts); - /** @var IMountPoint[] $mountMap */ - $mountMap = array_combine($storageIds, $mounts); - $folderMimetype = $mimetypeLoader->getId(FileInfo::MIMETYPE_FOLDER); - - /* - * Construct an array of the storage id with their prefix path - * This helps us to filter in the final query - */ - $filters = array_map(function (IMountPoint $mount) { - $storage = $mount->getStorage(); - - $storageId = $storage->getCache()->getNumericStorageId(); - $prefix = ''; - - if ($storage->instanceOfStorage(Jail::class)) { - $prefix = $storage->getUnJailedPath(''); - } - - return [ - 'storageId' => $storageId, - 'pathPrefix' => $prefix, - ]; - }, $mounts); - - // Search in batches of 500 entries - $searchLimit = 500; - $results = []; - $searchResultCount = 0; - $count = 0; - do { - $searchResult = $this->recentSearch($searchLimit, $offset, $folderMimetype, $filters); - - // Exit condition if there are no more results - if (count($searchResult) === 0) { - break; - } - - $searchResultCount += count($searchResult); - - $parseResult = $this->recentParse($searchResult, $mountMap, $mimetypeLoader); - - foreach ($parseResult as $result) { - $results[] = $result; - } - - $offset += $searchLimit; - $count++; - } while (count($results) < $limit && ($searchResultCount < (3 * $limit) || $count < 5)); - - return array_slice($results, 0, $limit); - } - - private function recentSearch($limit, $offset, $folderMimetype, $filters) { - $dbconn = \OC::$server->getDatabaseConnection(); - $builder = $dbconn->getQueryBuilder(); - $query = $builder - ->select('f.*') - ->from('filecache', 'f'); - - /* - * Here is where we construct the filtering. - * Note that this is expensive filtering as it is a lot of like queries. - * However the alternative is we do this filtering and parsing later in php with the risk of looping endlessly - */ - $storageFilters = $builder->expr()->orX(); - foreach ($filters as $filter) { - $storageFilter = $builder->expr()->andX( - $builder->expr()->eq('f.storage', $builder->createNamedParameter($filter['storageId'])) - ); - - if ($filter['pathPrefix'] !== '') { - $storageFilter->add( - $builder->expr()->like('f.path', $builder->createNamedParameter($dbconn->escapeLikeParameter($filter['pathPrefix']) . '/%')) - ); - } - - $storageFilters->add($storageFilter); - } - - $query->andWhere($storageFilters); - - $query->andWhere($builder->expr()->orX( - // handle non empty folders separate - $builder->expr()->neq('f.mimetype', $builder->createNamedParameter($folderMimetype, IQueryBuilder::PARAM_INT)), - $builder->expr()->eq('f.size', new Literal(0)) - )) - ->andWhere($builder->expr()->notLike('f.path', $builder->createNamedParameter('files_versions/%'))) - ->andWhere($builder->expr()->notLike('f.path', $builder->createNamedParameter('files_trashbin/%'))) - ->orderBy('f.mtime', 'DESC') - ->setMaxResults($limit) - ->setFirstResult($offset); - - $result = $query->execute(); - $rows = $result->fetchAll(); - $result->closeCursor(); - - return $rows; - } - - private function recentParse($result, $mountMap, $mimetypeLoader) { - $files = array_filter(array_map(function (array $entry) use ($mountMap, $mimetypeLoader) { - $mount = $mountMap[$entry['storage']]; - $entry['internalPath'] = $entry['path']; - $entry['mimetype'] = $mimetypeLoader->getMimetypeById($entry['mimetype']); - $entry['mimepart'] = $mimetypeLoader->getMimetypeById($entry['mimepart']); - $path = $this->getAbsolutePath($mount, $entry['path']); - if (is_null($path)) { - return null; - } - $fileInfo = new \OC\Files\FileInfo($path, $mount->getStorage(), $entry['internalPath'], $entry, $mount); - return $this->root->createNode($fileInfo->getPath(), $fileInfo); - }, $result)); - - return array_values(array_filter($files, function (Node $node) { - $cacheEntry = $node->getMountPoint()->getStorage()->getCache()->get($node->getId()); - if (!$cacheEntry) { - return false; - } - $relative = $this->getRelativePath($node->getPath()); - return $relative !== null && $relative !== '/' - && ($cacheEntry->getPermissions() & \OCP\Constants::PERMISSION_READ) === \OCP\Constants::PERMISSION_READ; - })); - } - - private function getAbsolutePath(IMountPoint $mount, $path) { - $storage = $mount->getStorage(); - if ($storage->instanceOfStorage('\OC\Files\Storage\Wrapper\Jail')) { - if ($storage->instanceOfStorage(SharedStorage::class)) { - $storage->getSourceStorage(); - } - /** @var \OC\Files\Storage\Wrapper\Jail $storage */ - $jailRoot = $storage->getUnjailedPath(''); - $rootLength = strlen($jailRoot) + 1; - if ($path === $jailRoot) { - return $mount->getMountPoint(); - } elseif (substr($path, 0, $rootLength) === $jailRoot . '/') { - return $mount->getMountPoint() . substr($path, $rootLength); - } else { - return null; - } - } else { - return $mount->getMountPoint() . $path; - } + $query = new SearchQuery( + new SearchBinaryOperator( + // filter out non empty folders + ISearchBinaryOperator::OPERATOR_OR, + [ + new SearchBinaryOperator( + ISearchBinaryOperator::OPERATOR_NOT, + [ + new SearchComparison( + ISearchComparison::COMPARE_EQUAL, + 'mimetype', + FileInfo::MIMETYPE_FOLDER + ), + ] + ), + new SearchComparison( + ISearchComparison::COMPARE_EQUAL, + 'size', + 0 + ), + ] + ), + $limit, + $offset, + [ + new SearchOrder( + ISearchOrder::DIRECTION_DESCENDING, + 'mtime' + ), + ] + ); + return $this->search($query); } } -- cgit v1.2.3 From 907e997c9914c925fbf86225a98b2262ee798351 Mon Sep 17 00:00:00 2001 From: Robin Appelman Date: Fri, 26 Mar 2021 17:34:49 +0100 Subject: optimize getting share types for recent files Signed-off-by: Robin Appelman --- apps/files/lib/Controller/ApiController.php | 82 +++++++++++++++++------------ 1 file changed, 49 insertions(+), 33 deletions(-) diff --git a/apps/files/lib/Controller/ApiController.php b/apps/files/lib/Controller/ApiController.php index 0cf261af726..20f60cde30d 100644 --- a/apps/files/lib/Controller/ApiController.php +++ b/apps/files/lib/Controller/ApiController.php @@ -177,9 +177,9 @@ class ApiController extends Controller { * @return array */ private function formatNodes(array $nodes) { - return array_values(array_map(function (Node $node) { - /** @var \OC\Files\Node\Node $shareTypes */ - $shareTypes = $this->getShareTypes($node); + $shareTypesForNodes = $this->getShareTypesForNodes($nodes); + return array_values(array_map(function (Node $node) use ($shareTypesForNodes) { + $shareTypes = $shareTypesForNodes[$node->getId()] ?? []; $file = \OCA\Files\Helper::formatFileInfo($node->getFileInfo()); $file['hasPreview'] = $this->previewManager->isAvailable($node); $parts = explode('/', dirname($node->getPath()), 4); @@ -196,28 +196,13 @@ class ApiController extends Controller { } /** - * Returns a list of recently modifed files. - * - * @NoAdminRequired + * Get the share types for each node * - * @return DataResponse - */ - public function getRecentFiles() { - $nodes = $this->userFolder->getRecent(100); - $files = $this->formatNodes($nodes); - return new DataResponse(['files' => $files]); - } - - /** - * Return a list of share types for outgoing shares - * - * @param Node $node file node - * - * @return int[] array of share types + * @param \OCP\Files\Node[] $nodes + * @return array list of share types for each fileid */ - private function getShareTypes(Node $node) { + private function getShareTypesForNodes(array $nodes): array { $userId = $this->userSession->getUser()->getUID(); - $shareTypes = []; $requestedShareTypes = [ IShare::TYPE_USER, IShare::TYPE_GROUP, @@ -227,22 +212,53 @@ class ApiController extends Controller { IShare::TYPE_ROOM, IShare::TYPE_DECK, ]; - foreach ($requestedShareTypes as $requestedShareType) { - // one of each type is enough to find out about the types - $shares = $this->shareManager->getSharesBy( - $userId, - $requestedShareType, - $node, - false, - 1 - ); - if (!empty($shares)) { - $shareTypes[] = $requestedShareType; + $shareTypes = []; + + $nodeIds = array_map(function (Node $node) { + return $node->getId(); + }, $nodes); + + foreach ($requestedShareTypes as $shareType) { + $nodesLeft = array_combine($nodeIds, array_fill(0, count($nodeIds), true)); + $offset = 0; + + // fetch shares until we've either found shares for all nodes or there are no more shares left + while (count($nodesLeft) > 0) { + $shares = $this->shareManager->getSharesBy($userId, $shareType, null, false, 100, $offset); + foreach ($shares as $share) { + $fileId = $share->getNodeId(); + if (isset($nodesLeft[$fileId])) { + if (!isset($shareTypes[$fileId])) { + $shareTypes[$fileId] = []; + } + $shareTypes[$fileId][] = $shareType; + unset($nodesLeft[$fileId]); + } + } + + if (count($shares) < 100) { + break; + } else { + $offset += count($shares); + } } } return $shareTypes; } + /** + * Returns a list of recently modifed files. + * + * @NoAdminRequired + * + * @return DataResponse + */ + public function getRecentFiles() { + $nodes = $this->userFolder->getRecent(100); + $files = $this->formatNodes($nodes); + return new DataResponse(['files' => $files]); + } + /** * Change the default sort mode * -- cgit v1.2.3 From 0d5f4edc2255578b912271d997421200f7321078 Mon Sep 17 00:00:00 2001 From: Robin Appelman Date: Wed, 21 Apr 2021 14:01:44 +0200 Subject: adjust tests Signed-off-by: Robin Appelman --- lib/private/Files/Node/Folder.php | 1 - tests/lib/Files/Node/FolderTest.php | 13 +++++++++++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/lib/private/Files/Node/Folder.php b/lib/private/Files/Node/Folder.php index bead0b19c75..f77c90b65a9 100644 --- a/lib/private/Files/Node/Folder.php +++ b/lib/private/Files/Node/Folder.php @@ -35,7 +35,6 @@ use OC\Files\Search\SearchBinaryOperator; use OC\Files\Search\SearchComparison; use OC\Files\Search\SearchOrder; use OC\Files\Search\SearchQuery; -use OC\Files\Storage\Storage; use OCP\Files\Cache\ICacheEntry; use OCP\Files\Config\ICachedMountInfo; use OCP\Files\FileInfo; diff --git a/tests/lib/Files/Node/FolderTest.php b/tests/lib/Files/Node/FolderTest.php index 1d541556c0b..36398171b1b 100644 --- a/tests/lib/Files/Node/FolderTest.php +++ b/tests/lib/Files/Node/FolderTest.php @@ -773,6 +773,10 @@ class FolderTest extends NodeTest { $folderInfo->expects($this->any()) ->method('getMountPoint') ->willReturn($mount); + $root->method('getMount') + ->willReturn($mount); + $root->method('getMountsIn') + ->willReturn([]); $cache = $storage->getCache(); @@ -838,6 +842,11 @@ class FolderTest extends NodeTest { ->method('getMountPoint') ->willReturn($mount); + $root->method('getMount') + ->willReturn($mount); + $root->method('getMountsIn') + ->willReturn([]); + $cache = $storage->getCache(); $id1 = $cache->put('bar/foo/folder', [ @@ -903,6 +912,10 @@ class FolderTest extends NodeTest { $folderInfo->expects($this->any()) ->method('getMountPoint') ->willReturn($mount); + $root->method('getMount') + ->willReturn($mount); + $root->method('getMountsIn') + ->willReturn([]); $cache = $storage->getCache(); -- cgit v1.2.3